Review funkt
This commit is contained in:
5
node_modules/bullmq/dist/esm/classes/abort-controller.d.ts
generated
vendored
Normal file
5
node_modules/bullmq/dist/esm/classes/abort-controller.d.ts
generated
vendored
Normal file
@@ -0,0 +1,5 @@
|
||||
import { AbortController as AbortControllerPolyfill } from 'node-abort-controller';
|
||||
declare let AbortControllerImpl: typeof AbortControllerPolyfill;
|
||||
export declare class AbortController extends AbortControllerImpl {
|
||||
}
|
||||
export {};
|
||||
13
node_modules/bullmq/dist/esm/classes/abort-controller.js
generated
vendored
Normal file
13
node_modules/bullmq/dist/esm/classes/abort-controller.js
generated
vendored
Normal file
@@ -0,0 +1,13 @@
|
||||
// Note: this Polyfill is only needed for Node versions < 15.4.0
|
||||
import { AbortController as AbortControllerPolyfill } from 'node-abort-controller';
|
||||
let AbortControllerImpl;
|
||||
// prefer native AbortController implementation if found
|
||||
if (globalThis.AbortController) {
|
||||
AbortControllerImpl =
|
||||
globalThis.AbortController;
|
||||
}
|
||||
else {
|
||||
AbortControllerImpl = AbortControllerPolyfill;
|
||||
}
|
||||
export class AbortController extends AbortControllerImpl {
|
||||
}
|
||||
80
node_modules/bullmq/dist/esm/classes/async-fifo-queue.d.ts
generated
vendored
Normal file
80
node_modules/bullmq/dist/esm/classes/async-fifo-queue.d.ts
generated
vendored
Normal file
@@ -0,0 +1,80 @@
|
||||
/**
|
||||
* (c) 2017-2025 BullForce Labs AB, MIT Licensed.
|
||||
* @see LICENSE.md
|
||||
*
|
||||
*/
|
||||
/**
|
||||
* AsyncFifoQueue
|
||||
*
|
||||
* A minimal FIFO queue for asynchronous operations. Allows adding asynchronous operations
|
||||
* and consume them in the order they are resolved.
|
||||
*/
|
||||
export declare class AsyncFifoQueue<T> {
|
||||
private ignoreErrors;
|
||||
/**
|
||||
* A queue of completed promises. As the pending
|
||||
* promises are resolved, they are added to this queue.
|
||||
*/
|
||||
private queue;
|
||||
/**
|
||||
* A set of pending promises.
|
||||
*/
|
||||
private pending;
|
||||
/**
|
||||
* The next promise to be resolved. As soon as a pending promise
|
||||
* is resolved, this promise is resolved with the result of the
|
||||
* pending promise.
|
||||
*/
|
||||
private nextPromise;
|
||||
private resolve;
|
||||
private reject;
|
||||
constructor(ignoreErrors?: boolean);
|
||||
/**
|
||||
* Adds a promise to the queue. When it resolves, its value is enqueued
|
||||
* and, if a consumer is waiting via {@link fetch}, the next pending
|
||||
* promise is resolved with the value.
|
||||
*
|
||||
* @param promise - The asynchronous operation to add to the queue.
|
||||
*/
|
||||
add(promise: Promise<T>): void;
|
||||
/**
|
||||
* Waits for all currently pending promises to settle.
|
||||
*
|
||||
* @returns A promise that resolves once every in-flight promise has resolved.
|
||||
*/
|
||||
waitAll(): Promise<void>;
|
||||
/**
|
||||
* Returns the total number of items currently tracked by the queue,
|
||||
* including both pending (in-flight) promises and resolved items that
|
||||
* have not yet been fetched.
|
||||
*
|
||||
* @returns The sum of pending and queued items.
|
||||
*/
|
||||
numTotal(): number;
|
||||
/**
|
||||
* Returns the number of promises that are still pending (in-flight).
|
||||
*
|
||||
* @returns The number of unresolved promises currently in the queue.
|
||||
*/
|
||||
numPending(): number;
|
||||
/**
|
||||
* Returns the number of items that have already resolved but have not
|
||||
* yet been fetched by a consumer.
|
||||
*
|
||||
* @returns The number of resolved items waiting to be consumed.
|
||||
*/
|
||||
numQueued(): number;
|
||||
private resolvePromise;
|
||||
private rejectPromise;
|
||||
private newPromise;
|
||||
private wait;
|
||||
/**
|
||||
* Fetches the next resolved item from the queue in FIFO order. If no
|
||||
* items are currently resolved but pending promises exist, it waits
|
||||
* until the next one resolves.
|
||||
*
|
||||
* @returns The next resolved value, or `undefined` if there are no
|
||||
* pending or queued items.
|
||||
*/
|
||||
fetch(): Promise<T | void>;
|
||||
}
|
||||
167
node_modules/bullmq/dist/esm/classes/async-fifo-queue.js
generated
vendored
Normal file
167
node_modules/bullmq/dist/esm/classes/async-fifo-queue.js
generated
vendored
Normal file
@@ -0,0 +1,167 @@
|
||||
/**
|
||||
* (c) 2017-2025 BullForce Labs AB, MIT Licensed.
|
||||
* @see LICENSE.md
|
||||
*
|
||||
*/
|
||||
class Node {
|
||||
constructor(value) {
|
||||
this.value = undefined;
|
||||
this.next = null;
|
||||
this.value = value;
|
||||
}
|
||||
}
|
||||
class LinkedList {
|
||||
constructor() {
|
||||
this.length = 0;
|
||||
this.head = null;
|
||||
this.tail = null;
|
||||
}
|
||||
push(value) {
|
||||
const newNode = new Node(value);
|
||||
if (!this.length) {
|
||||
this.head = newNode;
|
||||
}
|
||||
else {
|
||||
this.tail.next = newNode;
|
||||
}
|
||||
this.tail = newNode;
|
||||
this.length += 1;
|
||||
return newNode;
|
||||
}
|
||||
shift() {
|
||||
if (!this.length) {
|
||||
return null;
|
||||
}
|
||||
else {
|
||||
const head = this.head;
|
||||
this.head = this.head.next;
|
||||
this.length -= 1;
|
||||
return head;
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* AsyncFifoQueue
|
||||
*
|
||||
* A minimal FIFO queue for asynchronous operations. Allows adding asynchronous operations
|
||||
* and consume them in the order they are resolved.
|
||||
*/
|
||||
export class AsyncFifoQueue {
|
||||
constructor(ignoreErrors = false) {
|
||||
this.ignoreErrors = ignoreErrors;
|
||||
/**
|
||||
* A queue of completed promises. As the pending
|
||||
* promises are resolved, they are added to this queue.
|
||||
*/
|
||||
this.queue = new LinkedList();
|
||||
/**
|
||||
* A set of pending promises.
|
||||
*/
|
||||
this.pending = new Set();
|
||||
this.newPromise();
|
||||
}
|
||||
/**
|
||||
* Adds a promise to the queue. When it resolves, its value is enqueued
|
||||
* and, if a consumer is waiting via {@link fetch}, the next pending
|
||||
* promise is resolved with the value.
|
||||
*
|
||||
* @param promise - The asynchronous operation to add to the queue.
|
||||
*/
|
||||
add(promise) {
|
||||
this.pending.add(promise);
|
||||
promise
|
||||
.then(data => {
|
||||
this.pending.delete(promise);
|
||||
if (this.queue.length === 0) {
|
||||
this.resolvePromise(data);
|
||||
}
|
||||
this.queue.push(data);
|
||||
})
|
||||
.catch(err => {
|
||||
// Ignore errors
|
||||
if (this.ignoreErrors) {
|
||||
this.queue.push(undefined);
|
||||
}
|
||||
this.pending.delete(promise);
|
||||
this.rejectPromise(err);
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Waits for all currently pending promises to settle.
|
||||
*
|
||||
* @returns A promise that resolves once every in-flight promise has resolved.
|
||||
*/
|
||||
async waitAll() {
|
||||
await Promise.all(this.pending);
|
||||
}
|
||||
/**
|
||||
* Returns the total number of items currently tracked by the queue,
|
||||
* including both pending (in-flight) promises and resolved items that
|
||||
* have not yet been fetched.
|
||||
*
|
||||
* @returns The sum of pending and queued items.
|
||||
*/
|
||||
numTotal() {
|
||||
return this.pending.size + this.queue.length;
|
||||
}
|
||||
/**
|
||||
* Returns the number of promises that are still pending (in-flight).
|
||||
*
|
||||
* @returns The number of unresolved promises currently in the queue.
|
||||
*/
|
||||
numPending() {
|
||||
return this.pending.size;
|
||||
}
|
||||
/**
|
||||
* Returns the number of items that have already resolved but have not
|
||||
* yet been fetched by a consumer.
|
||||
*
|
||||
* @returns The number of resolved items waiting to be consumed.
|
||||
*/
|
||||
numQueued() {
|
||||
return this.queue.length;
|
||||
}
|
||||
resolvePromise(data) {
|
||||
this.resolve(data);
|
||||
this.newPromise();
|
||||
}
|
||||
rejectPromise(err) {
|
||||
this.reject(err);
|
||||
this.newPromise();
|
||||
}
|
||||
newPromise() {
|
||||
this.nextPromise = new Promise((resolve, reject) => {
|
||||
this.resolve = resolve;
|
||||
this.reject = reject;
|
||||
});
|
||||
}
|
||||
async wait() {
|
||||
return this.nextPromise;
|
||||
}
|
||||
/**
|
||||
* Fetches the next resolved item from the queue in FIFO order. If no
|
||||
* items are currently resolved but pending promises exist, it waits
|
||||
* until the next one resolves.
|
||||
*
|
||||
* @returns The next resolved value, or `undefined` if there are no
|
||||
* pending or queued items.
|
||||
*/
|
||||
async fetch() {
|
||||
var _a;
|
||||
if (this.pending.size === 0 && this.queue.length === 0) {
|
||||
return;
|
||||
}
|
||||
while (this.queue.length === 0) {
|
||||
try {
|
||||
await this.wait();
|
||||
}
|
||||
catch (err) {
|
||||
// Ignore errors
|
||||
if (!this.ignoreErrors) {
|
||||
console.error('Unexpected Error in AsyncFifoQueue', err);
|
||||
}
|
||||
}
|
||||
}
|
||||
return (_a = this.queue.shift()) === null || _a === void 0 ? void 0 : _a.value;
|
||||
}
|
||||
}
|
||||
11
node_modules/bullmq/dist/esm/classes/backoffs.d.ts
generated
vendored
Normal file
11
node_modules/bullmq/dist/esm/classes/backoffs.d.ts
generated
vendored
Normal file
@@ -0,0 +1,11 @@
|
||||
import { BackoffOptions } from '../interfaces/backoff-options';
|
||||
import { MinimalJob } from '../interfaces/minimal-job';
|
||||
import { BackoffStrategy } from '../types/backoff-strategy';
|
||||
export interface BuiltInStrategies {
|
||||
[index: string]: (delay: number, jitter?: number) => BackoffStrategy;
|
||||
}
|
||||
export declare class Backoffs {
|
||||
static builtinStrategies: BuiltInStrategies;
|
||||
static normalize(backoff: number | BackoffOptions): BackoffOptions | undefined;
|
||||
static calculate(backoff: BackoffOptions, attemptsMade: number, err: Error, job: MinimalJob, customStrategy?: BackoffStrategy): Promise<number> | number | undefined;
|
||||
}
|
||||
56
node_modules/bullmq/dist/esm/classes/backoffs.js
generated
vendored
Normal file
56
node_modules/bullmq/dist/esm/classes/backoffs.js
generated
vendored
Normal file
@@ -0,0 +1,56 @@
|
||||
export class Backoffs {
|
||||
static normalize(backoff) {
|
||||
if (Number.isFinite(backoff)) {
|
||||
return {
|
||||
type: 'fixed',
|
||||
delay: backoff,
|
||||
};
|
||||
}
|
||||
else if (backoff) {
|
||||
return backoff;
|
||||
}
|
||||
}
|
||||
static calculate(backoff, attemptsMade, err, job, customStrategy) {
|
||||
if (backoff) {
|
||||
const strategy = lookupStrategy(backoff, customStrategy);
|
||||
return strategy(attemptsMade, backoff.type, err, job);
|
||||
}
|
||||
}
|
||||
}
|
||||
Backoffs.builtinStrategies = {
|
||||
fixed: function (delay, jitter = 0) {
|
||||
return function () {
|
||||
if (jitter > 0) {
|
||||
const minDelay = delay * (1 - jitter);
|
||||
return Math.floor(Math.random() * delay * jitter + minDelay);
|
||||
}
|
||||
else {
|
||||
return delay;
|
||||
}
|
||||
};
|
||||
},
|
||||
exponential: function (delay, jitter = 0) {
|
||||
return function (attemptsMade) {
|
||||
if (jitter > 0) {
|
||||
const maxDelay = Math.round(Math.pow(2, attemptsMade - 1) * delay);
|
||||
const minDelay = maxDelay * (1 - jitter);
|
||||
return Math.floor(Math.random() * maxDelay * jitter + minDelay);
|
||||
}
|
||||
else {
|
||||
return Math.round(Math.pow(2, attemptsMade - 1) * delay);
|
||||
}
|
||||
};
|
||||
},
|
||||
};
|
||||
function lookupStrategy(backoff, customStrategy) {
|
||||
if (backoff.type in Backoffs.builtinStrategies) {
|
||||
return Backoffs.builtinStrategies[backoff.type](backoff.delay, backoff.jitter);
|
||||
}
|
||||
else if (customStrategy) {
|
||||
return customStrategy;
|
||||
}
|
||||
else {
|
||||
throw new Error(`Unknown backoff strategy ${backoff.type}.
|
||||
If a custom backoff strategy is used, specify it when the queue is created.`);
|
||||
}
|
||||
}
|
||||
18
node_modules/bullmq/dist/esm/classes/bun-redis-client.d.ts
generated
vendored
Normal file
18
node_modules/bullmq/dist/esm/classes/bun-redis-client.d.ts
generated
vendored
Normal file
@@ -0,0 +1,18 @@
|
||||
import { IRedisClient } from '../interfaces/redis-client';
|
||||
export type RedisCommandArgument = string | Buffer;
|
||||
export interface BunRedisRawClient {
|
||||
connected: boolean;
|
||||
url?: string;
|
||||
onconnect?: ((...args: any[]) => void) | null;
|
||||
onclose?: ((...args: any[]) => void) | null;
|
||||
onerror?: ((...args: any[]) => void) | null;
|
||||
connect(): Promise<void>;
|
||||
close(): void;
|
||||
send<T = any>(command: string, args: RedisCommandArgument[]): Promise<T>;
|
||||
get(key: string): Promise<string | null | undefined>;
|
||||
smembers(key: string): Promise<unknown[] | null | undefined>;
|
||||
incr(key: string): Promise<number>;
|
||||
}
|
||||
export declare function createBunRedisClient<TClient extends BunRedisRawClient>(client: TClient, opts?: {
|
||||
lazyConnect?: boolean;
|
||||
}): IRedisClient;
|
||||
1029
node_modules/bullmq/dist/esm/classes/bun-redis-client.js
generated
vendored
Normal file
1029
node_modules/bullmq/dist/esm/classes/bun-redis-client.js
generated
vendored
Normal file
File diff suppressed because it is too large
Load Diff
23
node_modules/bullmq/dist/esm/classes/child-pool.d.ts
generated
vendored
Normal file
23
node_modules/bullmq/dist/esm/classes/child-pool.d.ts
generated
vendored
Normal file
@@ -0,0 +1,23 @@
|
||||
import { Child } from './child';
|
||||
import { SandboxedOptions } from '../interfaces';
|
||||
interface ChildPoolOpts extends SandboxedOptions {
|
||||
mainFile?: string;
|
||||
}
|
||||
export declare class ChildPool {
|
||||
retained: {
|
||||
[key: number]: Child;
|
||||
};
|
||||
free: {
|
||||
[key: string]: Child[];
|
||||
};
|
||||
private opts;
|
||||
constructor({ mainFile, useWorkerThreads, workerForkOptions, workerThreadsOptions, }: ChildPoolOpts);
|
||||
retain(processFile: string): Promise<Child>;
|
||||
release(child: Child): void;
|
||||
remove(child: Child): void;
|
||||
kill(child: Child, signal?: 'SIGTERM' | 'SIGKILL'): Promise<void>;
|
||||
clean(): Promise<void>;
|
||||
getFree(id: string): Child[];
|
||||
getAllFree(): Child[];
|
||||
}
|
||||
export {};
|
||||
94
node_modules/bullmq/dist/esm/classes/child-pool.js
generated
vendored
Normal file
94
node_modules/bullmq/dist/esm/classes/child-pool.js
generated
vendored
Normal file
@@ -0,0 +1,94 @@
|
||||
import * as path from 'path';
|
||||
import { Child } from './child';
|
||||
const CHILD_KILL_TIMEOUT = 30000;
|
||||
const supportCJS = () => {
|
||||
return (typeof require === 'function' &&
|
||||
typeof module === 'object' &&
|
||||
typeof module.exports === 'object');
|
||||
};
|
||||
export class ChildPool {
|
||||
constructor({ mainFile = supportCJS()
|
||||
? path.join(process.cwd(), 'dist/cjs/classes/main.js')
|
||||
: path.join(process.cwd(), 'dist/esm/classes/main.js'), useWorkerThreads, workerForkOptions, workerThreadsOptions, }) {
|
||||
this.retained = {};
|
||||
this.free = {};
|
||||
this.opts = {
|
||||
mainFile,
|
||||
useWorkerThreads,
|
||||
workerForkOptions,
|
||||
workerThreadsOptions,
|
||||
};
|
||||
}
|
||||
async retain(processFile) {
|
||||
let child = this.getFree(processFile).pop();
|
||||
if (child) {
|
||||
this.retained[child.pid] = child;
|
||||
return child;
|
||||
}
|
||||
child = new Child(this.opts.mainFile, processFile, {
|
||||
useWorkerThreads: this.opts.useWorkerThreads,
|
||||
workerForkOptions: this.opts.workerForkOptions,
|
||||
workerThreadsOptions: this.opts.workerThreadsOptions,
|
||||
});
|
||||
child.on('exit', this.remove.bind(this, child));
|
||||
try {
|
||||
await child.init();
|
||||
// Check status here as well, in case the child exited before we could
|
||||
// retain it.
|
||||
if (child.exitCode !== null || child.signalCode !== null) {
|
||||
throw new Error('Child exited before it could be retained');
|
||||
}
|
||||
this.retained[child.pid] = child;
|
||||
return child;
|
||||
}
|
||||
catch (err) {
|
||||
console.error(err);
|
||||
// A child that failed to initialize (or exited during init) must never
|
||||
// be released back into the free pool, otherwise it becomes a "zombie"
|
||||
// that is reused for every subsequent job and fails them instantly.
|
||||
// Kill and remove it so a fresh child is forked on the next retain.
|
||||
// The child also exits itself after a failed init (see ChildProcessor),
|
||||
// so this is normally a no-op; log any kill failure instead of silently
|
||||
// swallowing it so a lingering child would not go unnoticed.
|
||||
if (child.childProcess || child.worker) {
|
||||
try {
|
||||
this.kill(child, 'SIGKILL').catch(killErr => {
|
||||
console.error('Failed to kill child after init error:', killErr);
|
||||
});
|
||||
}
|
||||
catch (killErr) {
|
||||
console.error('Failed to kill child after init error:', killErr);
|
||||
}
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
release(child) {
|
||||
delete this.retained[child.pid];
|
||||
this.getFree(child.processFile).push(child);
|
||||
}
|
||||
remove(child) {
|
||||
delete this.retained[child.pid];
|
||||
const free = this.getFree(child.processFile);
|
||||
const childIndex = free.indexOf(child);
|
||||
if (childIndex > -1) {
|
||||
free.splice(childIndex, 1);
|
||||
}
|
||||
}
|
||||
async kill(child, signal = 'SIGKILL') {
|
||||
this.remove(child);
|
||||
return child.kill(signal, CHILD_KILL_TIMEOUT);
|
||||
}
|
||||
async clean() {
|
||||
const children = Object.values(this.retained).concat(this.getAllFree());
|
||||
this.retained = {};
|
||||
this.free = {};
|
||||
await Promise.all(children.map(c => this.kill(c, 'SIGTERM')));
|
||||
}
|
||||
getFree(id) {
|
||||
return (this.free[id] = this.free[id] || []);
|
||||
}
|
||||
getAllFree() {
|
||||
return Object.values(this.free).reduce((first, second) => first.concat(second), []);
|
||||
}
|
||||
}
|
||||
44
node_modules/bullmq/dist/esm/classes/child-processor.d.ts
generated
vendored
Normal file
44
node_modules/bullmq/dist/esm/classes/child-processor.d.ts
generated
vendored
Normal file
@@ -0,0 +1,44 @@
|
||||
import { Receiver, SandboxedJob } from '../interfaces';
|
||||
import { JobJsonSandbox } from '../types';
|
||||
declare enum ChildStatus {
|
||||
Idle = 0,
|
||||
Started = 1,
|
||||
Terminating = 2,
|
||||
Errored = 3
|
||||
}
|
||||
/**
|
||||
* ChildProcessor
|
||||
*
|
||||
* This class acts as the interface between a child process and it parent process
|
||||
* so that jobs can be processed in different processes.
|
||||
*
|
||||
*/
|
||||
export declare class ChildProcessor {
|
||||
private send;
|
||||
private receiver;
|
||||
status?: ChildStatus;
|
||||
processor: any;
|
||||
currentJobPromise: Promise<unknown> | undefined;
|
||||
private abortController?;
|
||||
constructor(send: (msg: any) => Promise<void>, receiver: Receiver);
|
||||
init(processorFile: string): Promise<void>;
|
||||
start(jobJson: JobJsonSandbox, token?: string): Promise<void>;
|
||||
/**
|
||||
* Cancels the currently running job by aborting its signal.
|
||||
* @param reason - Optional reason for the cancellation
|
||||
*/
|
||||
cancel(reason?: string): void;
|
||||
stop(): Promise<void>;
|
||||
waitForCurrentJobAndExit(): Promise<void>;
|
||||
/**
|
||||
* Enhance the given job argument with some functions
|
||||
* that can be called from the sandboxed job processor.
|
||||
*
|
||||
* Note, the `job` argument is a JSON deserialized message
|
||||
* from the main node process to this forked child process,
|
||||
* the functions on the original job object are not in tact.
|
||||
* The wrapped job adds back some of those original functions.
|
||||
*/
|
||||
protected wrapJob(job: JobJsonSandbox, send: (msg: any) => Promise<void>): SandboxedJob;
|
||||
}
|
||||
export {};
|
||||
264
node_modules/bullmq/dist/esm/classes/child-processor.js
generated
vendored
Normal file
264
node_modules/bullmq/dist/esm/classes/child-processor.js
generated
vendored
Normal file
@@ -0,0 +1,264 @@
|
||||
import { AbortController } from './abort-controller';
|
||||
import { ParentCommand } from '../enums';
|
||||
import { errorToJSON } from '../utils';
|
||||
var ChildStatus;
|
||||
(function (ChildStatus) {
|
||||
ChildStatus[ChildStatus["Idle"] = 0] = "Idle";
|
||||
ChildStatus[ChildStatus["Started"] = 1] = "Started";
|
||||
ChildStatus[ChildStatus["Terminating"] = 2] = "Terminating";
|
||||
ChildStatus[ChildStatus["Errored"] = 3] = "Errored";
|
||||
})(ChildStatus || (ChildStatus = {}));
|
||||
const RESPONSE_TIMEOUT = process.env.NODE_ENV === 'test' ? 500 : 5000;
|
||||
/**
|
||||
* ChildProcessor
|
||||
*
|
||||
* This class acts as the interface between a child process and it parent process
|
||||
* so that jobs can be processed in different processes.
|
||||
*
|
||||
*/
|
||||
export class ChildProcessor {
|
||||
constructor(send, receiver) {
|
||||
this.send = send;
|
||||
this.receiver = receiver;
|
||||
}
|
||||
async init(processorFile) {
|
||||
var _a;
|
||||
let processor;
|
||||
try {
|
||||
const { default: processorFn } = await import(processorFile);
|
||||
processor = processorFn;
|
||||
if (processor.default) {
|
||||
// support es2015 module.
|
||||
processor = processor.default;
|
||||
}
|
||||
if (typeof processor !== 'function') {
|
||||
throw new Error('No function is exported in processor file');
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
this.status = ChildStatus.Errored;
|
||||
try {
|
||||
await this.send({
|
||||
cmd: ParentCommand.InitFailed,
|
||||
err: errorToJSON(err),
|
||||
});
|
||||
}
|
||||
finally {
|
||||
// A child that failed to initialize cannot recover, and because the open
|
||||
// IPC channel keeps its event loop alive it would never exit on its own.
|
||||
// Exit explicitly (after attempting to send InitFailed) so the parent
|
||||
// can never reuse a half-initialized "zombie" child. This is a
|
||||
// belt-and-braces measure: ChildPool also kills the child, but exiting
|
||||
// here guarantees termination even if the parent-side kill were to fail.
|
||||
// In a worker thread this stops only the current worker, not the process.
|
||||
process.exit((_a = process.exitCode) !== null && _a !== void 0 ? _a : 1);
|
||||
}
|
||||
}
|
||||
const origProcessor = processor;
|
||||
processor = function (job, token, signal) {
|
||||
try {
|
||||
return Promise.resolve(origProcessor(job, token, signal));
|
||||
}
|
||||
catch (err) {
|
||||
return Promise.reject(err);
|
||||
}
|
||||
};
|
||||
this.processor = processor;
|
||||
this.status = ChildStatus.Idle;
|
||||
await this.send({
|
||||
cmd: ParentCommand.InitCompleted,
|
||||
});
|
||||
}
|
||||
async start(jobJson, token) {
|
||||
if (this.status !== ChildStatus.Idle) {
|
||||
return this.send({
|
||||
cmd: ParentCommand.Error,
|
||||
err: errorToJSON(new Error('cannot start a not idling child process')),
|
||||
});
|
||||
}
|
||||
this.status = ChildStatus.Started;
|
||||
this.abortController = new AbortController();
|
||||
this.currentJobPromise = (async () => {
|
||||
try {
|
||||
const job = this.wrapJob(jobJson, this.send);
|
||||
const result = await this.processor(job, token, this.abortController.signal);
|
||||
await this.send({
|
||||
cmd: ParentCommand.Completed,
|
||||
value: typeof result === 'undefined' ? null : result,
|
||||
});
|
||||
}
|
||||
catch (err) {
|
||||
await this.send({
|
||||
cmd: ParentCommand.Failed,
|
||||
value: errorToJSON(!err.message ? new Error(err) : err),
|
||||
});
|
||||
}
|
||||
finally {
|
||||
this.status = ChildStatus.Idle;
|
||||
this.currentJobPromise = undefined;
|
||||
this.abortController = undefined;
|
||||
}
|
||||
})();
|
||||
}
|
||||
/**
|
||||
* Cancels the currently running job by aborting its signal.
|
||||
* @param reason - Optional reason for the cancellation
|
||||
*/
|
||||
cancel(reason) {
|
||||
if (this.abortController) {
|
||||
this.abortController.abort(reason);
|
||||
}
|
||||
}
|
||||
async stop() { }
|
||||
async waitForCurrentJobAndExit() {
|
||||
this.status = ChildStatus.Terminating;
|
||||
try {
|
||||
await this.currentJobPromise;
|
||||
}
|
||||
finally {
|
||||
process.exit(process.exitCode || 0);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Enhance the given job argument with some functions
|
||||
* that can be called from the sandboxed job processor.
|
||||
*
|
||||
* Note, the `job` argument is a JSON deserialized message
|
||||
* from the main node process to this forked child process,
|
||||
* the functions on the original job object are not in tact.
|
||||
* The wrapped job adds back some of those original functions.
|
||||
*/
|
||||
wrapJob(job, send) {
|
||||
const wrappedJob = Object.assign(Object.assign({}, job), { queueQualifiedName: job.queueQualifiedName, data: JSON.parse(job.data || '{}'), opts: job.opts, returnValue: JSON.parse(job.returnvalue || '{}'),
|
||||
/*
|
||||
* Proxy `updateProgress` function, should works as `progress` function.
|
||||
*/
|
||||
async updateProgress(progress) {
|
||||
// Locally store reference to new progress value
|
||||
// so that we can return it from this process synchronously.
|
||||
this.progress = progress;
|
||||
// Send message to update job progress.
|
||||
await send({
|
||||
cmd: ParentCommand.Progress,
|
||||
value: progress,
|
||||
});
|
||||
},
|
||||
/*
|
||||
* Proxy job `log` function.
|
||||
*/
|
||||
log: async (row) => {
|
||||
await send({
|
||||
cmd: ParentCommand.Log,
|
||||
value: row,
|
||||
});
|
||||
},
|
||||
/*
|
||||
* Proxy `moveToDelayed` function.
|
||||
*/
|
||||
moveToDelayed: async (timestamp, token) => {
|
||||
await send({
|
||||
cmd: ParentCommand.MoveToDelayed,
|
||||
value: { timestamp, token },
|
||||
});
|
||||
},
|
||||
/*
|
||||
* Proxy `moveToWait` function.
|
||||
*/
|
||||
moveToWait: async (token) => {
|
||||
await send({
|
||||
cmd: ParentCommand.MoveToWait,
|
||||
value: { token },
|
||||
});
|
||||
},
|
||||
/*
|
||||
* Proxy `moveToWaitingChildren` function.
|
||||
*/
|
||||
moveToWaitingChildren: async (token, opts) => {
|
||||
const requestId = Math.random().toString(36).substring(2, 15);
|
||||
await send({
|
||||
requestId,
|
||||
cmd: ParentCommand.MoveToWaitingChildren,
|
||||
value: { token, opts },
|
||||
});
|
||||
return waitResponse(requestId, this.receiver, RESPONSE_TIMEOUT, 'moveToWaitingChildren');
|
||||
},
|
||||
/*
|
||||
* Proxy `updateData` function.
|
||||
*/
|
||||
updateData: async (data) => {
|
||||
await send({
|
||||
cmd: ParentCommand.Update,
|
||||
value: data,
|
||||
});
|
||||
wrappedJob.data = data;
|
||||
},
|
||||
/**
|
||||
* Proxy `getChildrenValues` function.
|
||||
*/
|
||||
getChildrenValues: async () => {
|
||||
const requestId = Math.random().toString(36).substring(2, 15);
|
||||
await send({
|
||||
requestId,
|
||||
cmd: ParentCommand.GetChildrenValues,
|
||||
});
|
||||
return waitResponse(requestId, this.receiver, RESPONSE_TIMEOUT, 'getChildrenValues');
|
||||
},
|
||||
/**
|
||||
* Proxy `getIgnoredChildrenFailures` function.
|
||||
*
|
||||
* This method sends a request to retrieve the failures of ignored children
|
||||
* and waits for a response from the parent process.
|
||||
*
|
||||
* @returns - A promise that resolves with the ignored children failures.
|
||||
* The exact structure of the returned data depends on the parent process implementation.
|
||||
*/
|
||||
getIgnoredChildrenFailures: async () => {
|
||||
const requestId = Math.random().toString(36).substring(2, 15);
|
||||
await send({
|
||||
requestId,
|
||||
cmd: ParentCommand.GetIgnoredChildrenFailures,
|
||||
});
|
||||
return waitResponse(requestId, this.receiver, RESPONSE_TIMEOUT, 'getIgnoredChildrenFailures');
|
||||
},
|
||||
/**
|
||||
* Proxy `getDependenciesCount` function.
|
||||
*/
|
||||
getDependenciesCount: async (opts) => {
|
||||
const requestId = Math.random().toString(36).substring(2, 15);
|
||||
await send({
|
||||
requestId,
|
||||
cmd: ParentCommand.GetDependenciesCount,
|
||||
value: opts,
|
||||
});
|
||||
return waitResponse(requestId, this.receiver, RESPONSE_TIMEOUT, 'getDependenciesCount');
|
||||
},
|
||||
/**
|
||||
* Proxy `getDependencies` function.
|
||||
*/
|
||||
getDependencies: async (opts) => {
|
||||
const requestId = Math.random().toString(36).substring(2, 15);
|
||||
await send({
|
||||
requestId,
|
||||
cmd: ParentCommand.GetDependencies,
|
||||
value: opts,
|
||||
});
|
||||
return waitResponse(requestId, this.receiver, RESPONSE_TIMEOUT, 'getDependencies');
|
||||
} });
|
||||
return wrappedJob;
|
||||
}
|
||||
}
|
||||
const waitResponse = async (requestId, receiver, timeout, cmd) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
const listener = (msg) => {
|
||||
if (msg.requestId === requestId) {
|
||||
resolve(msg.value);
|
||||
receiver.off('message', listener);
|
||||
}
|
||||
};
|
||||
receiver.on('message', listener);
|
||||
setTimeout(() => {
|
||||
receiver.off('message', listener);
|
||||
reject(new Error(`TimeoutError: ${cmd} timed out in (${timeout}ms)`));
|
||||
}, timeout);
|
||||
});
|
||||
};
|
||||
32
node_modules/bullmq/dist/esm/classes/child.d.ts
generated
vendored
Normal file
32
node_modules/bullmq/dist/esm/classes/child.d.ts
generated
vendored
Normal file
@@ -0,0 +1,32 @@
|
||||
import { ChildProcess } from 'child_process';
|
||||
import { Worker } from 'worker_threads';
|
||||
import { SandboxedOptions } from '../interfaces';
|
||||
import { EventEmitter } from 'events';
|
||||
/**
|
||||
* Child class
|
||||
*
|
||||
* This class is used to create a child process or worker thread, and allows using
|
||||
* isolated processes or threads for processing jobs.
|
||||
*
|
||||
*/
|
||||
export declare class Child extends EventEmitter {
|
||||
private mainFile;
|
||||
processFile: string;
|
||||
private opts;
|
||||
childProcess: ChildProcess;
|
||||
worker: Worker;
|
||||
private _exitCode;
|
||||
private _signalCode;
|
||||
private _killed;
|
||||
constructor(mainFile: string, processFile: string, opts?: SandboxedOptions);
|
||||
get pid(): number;
|
||||
get exitCode(): number;
|
||||
get signalCode(): number;
|
||||
get killed(): boolean;
|
||||
init(): Promise<void>;
|
||||
send(msg: any): Promise<void>;
|
||||
private killProcess;
|
||||
kill(signal?: 'SIGTERM' | 'SIGKILL', timeoutMs?: number): Promise<void>;
|
||||
private initChild;
|
||||
hasProcessExited(): boolean;
|
||||
}
|
||||
216
node_modules/bullmq/dist/esm/classes/child.js
generated
vendored
Normal file
216
node_modules/bullmq/dist/esm/classes/child.js
generated
vendored
Normal file
@@ -0,0 +1,216 @@
|
||||
import { fork } from 'child_process';
|
||||
import { createServer } from 'net';
|
||||
import { Worker } from 'worker_threads';
|
||||
import { ChildCommand, ParentCommand } from '../enums';
|
||||
import { EventEmitter } from 'events';
|
||||
/**
|
||||
* @see https://nodejs.org/api/process.html#process_exit_codes
|
||||
*/
|
||||
const exitCodesErrors = {
|
||||
1: 'Uncaught Fatal Exception',
|
||||
2: 'Unused',
|
||||
3: 'Internal JavaScript Parse Error',
|
||||
4: 'Internal JavaScript Evaluation Failure',
|
||||
5: 'Fatal Error',
|
||||
6: 'Non-function Internal Exception Handler',
|
||||
7: 'Internal Exception Handler Run-Time Failure',
|
||||
8: 'Unused',
|
||||
9: 'Invalid Argument',
|
||||
10: 'Internal JavaScript Run-Time Failure',
|
||||
12: 'Invalid Debug Argument',
|
||||
13: 'Unfinished Top-Level Await',
|
||||
};
|
||||
/**
|
||||
* Child class
|
||||
*
|
||||
* This class is used to create a child process or worker thread, and allows using
|
||||
* isolated processes or threads for processing jobs.
|
||||
*
|
||||
*/
|
||||
export class Child extends EventEmitter {
|
||||
constructor(mainFile, processFile, opts = {
|
||||
useWorkerThreads: false,
|
||||
}) {
|
||||
super();
|
||||
this.mainFile = mainFile;
|
||||
this.processFile = processFile;
|
||||
this.opts = opts;
|
||||
this._exitCode = null;
|
||||
this._signalCode = null;
|
||||
this._killed = false;
|
||||
}
|
||||
get pid() {
|
||||
if (this.childProcess) {
|
||||
return this.childProcess.pid;
|
||||
}
|
||||
else if (this.worker) {
|
||||
// Worker threads pids can become negative when they are terminated
|
||||
// so we need to use the absolute value to index the retained object
|
||||
return Math.abs(this.worker.threadId);
|
||||
}
|
||||
else {
|
||||
throw new Error('No child process or worker thread');
|
||||
}
|
||||
}
|
||||
get exitCode() {
|
||||
return this._exitCode;
|
||||
}
|
||||
get signalCode() {
|
||||
return this._signalCode;
|
||||
}
|
||||
get killed() {
|
||||
if (this.childProcess) {
|
||||
return this.childProcess.killed;
|
||||
}
|
||||
return this._killed;
|
||||
}
|
||||
async init() {
|
||||
var _a, _b;
|
||||
const execArgv = await convertExecArgv(process.execArgv);
|
||||
let parent;
|
||||
if (this.opts.useWorkerThreads) {
|
||||
this.worker = parent = new Worker(this.mainFile, Object.assign({ execArgv, stdin: true, stdout: true, stderr: true }, (this.opts.workerThreadsOptions
|
||||
? this.opts.workerThreadsOptions
|
||||
: {})));
|
||||
}
|
||||
else {
|
||||
this.childProcess = parent = fork(this.mainFile, [], Object.assign({ execArgv, stdio: 'pipe' }, (this.opts.workerForkOptions ? this.opts.workerForkOptions : {})));
|
||||
}
|
||||
parent.on('exit', (exitCode, signalCode) => {
|
||||
this._exitCode = exitCode;
|
||||
// Coerce to null if undefined for backwards compatibility
|
||||
signalCode = typeof signalCode === 'undefined' ? null : signalCode;
|
||||
this._signalCode = signalCode;
|
||||
this._killed = true;
|
||||
this.emit('exit', exitCode, signalCode);
|
||||
// Clean all listeners, we do not expect any more events after "exit"
|
||||
parent.removeAllListeners();
|
||||
this.removeAllListeners();
|
||||
});
|
||||
parent.on('error', (...args) => this.emit('error', ...args));
|
||||
parent.on('message', (...args) => this.emit('message', ...args));
|
||||
parent.on('close', (...args) => this.emit('close', ...args));
|
||||
// `parent.stdout`/`parent.stderr` may be null when the underlying runtime
|
||||
// does not pipe child stdio (e.g. Bun ignores `worker_threads` stdout/stderr
|
||||
// options, and Node returns null when `stdio: 'ignore'` is passed in
|
||||
// `workerForkOptions`). Guard the pipe calls so initialization does not throw.
|
||||
// See https://github.com/taskforcesh/bullmq/issues/2232
|
||||
(_a = parent.stdout) === null || _a === void 0 ? void 0 : _a.pipe(process.stdout);
|
||||
(_b = parent.stderr) === null || _b === void 0 ? void 0 : _b.pipe(process.stderr);
|
||||
await this.initChild();
|
||||
}
|
||||
async send(msg) {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (this.childProcess) {
|
||||
this.childProcess.send(msg, (err) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
}
|
||||
else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
}
|
||||
else if (this.worker) {
|
||||
resolve(this.worker.postMessage(msg));
|
||||
}
|
||||
else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
}
|
||||
killProcess(signal = 'SIGKILL') {
|
||||
if (this.childProcess) {
|
||||
this.childProcess.kill(signal);
|
||||
}
|
||||
else if (this.worker) {
|
||||
this.worker.terminate();
|
||||
}
|
||||
}
|
||||
async kill(signal = 'SIGKILL', timeoutMs) {
|
||||
if (this.hasProcessExited()) {
|
||||
return;
|
||||
}
|
||||
const onExit = onExitOnce(this.childProcess || this.worker);
|
||||
this.killProcess(signal);
|
||||
if (timeoutMs !== undefined && (timeoutMs === 0 || isFinite(timeoutMs))) {
|
||||
const timeoutHandle = setTimeout(() => {
|
||||
if (!this.hasProcessExited()) {
|
||||
this.killProcess('SIGKILL');
|
||||
}
|
||||
}, timeoutMs);
|
||||
await onExit;
|
||||
clearTimeout(timeoutHandle);
|
||||
}
|
||||
await onExit;
|
||||
}
|
||||
async initChild() {
|
||||
const onComplete = new Promise((resolve, reject) => {
|
||||
const onMessageHandler = (msg) => {
|
||||
if (!Object.values(ParentCommand).includes(msg.cmd)) {
|
||||
return;
|
||||
}
|
||||
if (msg.cmd === ParentCommand.InitCompleted) {
|
||||
resolve();
|
||||
}
|
||||
else if (msg.cmd === ParentCommand.InitFailed) {
|
||||
const err = new Error();
|
||||
err.stack = msg.err.stack;
|
||||
err.message = msg.err.message;
|
||||
reject(err);
|
||||
}
|
||||
this.off('message', onMessageHandler);
|
||||
this.off('close', onCloseHandler);
|
||||
};
|
||||
const onCloseHandler = (code, signal) => {
|
||||
if (code > 128) {
|
||||
code -= 128;
|
||||
}
|
||||
const msg = exitCodesErrors[code] || `Unknown exit code ${code}`;
|
||||
reject(new Error(`Error initializing child: ${msg} and signal ${signal}`));
|
||||
this.off('message', onMessageHandler);
|
||||
this.off('close', onCloseHandler);
|
||||
};
|
||||
this.on('message', onMessageHandler);
|
||||
this.on('close', onCloseHandler);
|
||||
});
|
||||
await this.send({
|
||||
cmd: ChildCommand.Init,
|
||||
value: this.processFile,
|
||||
});
|
||||
await onComplete;
|
||||
}
|
||||
hasProcessExited() {
|
||||
return !!(this.exitCode !== null || this.signalCode);
|
||||
}
|
||||
}
|
||||
function onExitOnce(child) {
|
||||
return new Promise(resolve => {
|
||||
child.once('exit', () => resolve());
|
||||
});
|
||||
}
|
||||
const getFreePort = async () => {
|
||||
return new Promise(resolve => {
|
||||
const server = createServer();
|
||||
server.listen(0, () => {
|
||||
const { port } = server.address();
|
||||
server.close(() => resolve(port));
|
||||
});
|
||||
});
|
||||
};
|
||||
const convertExecArgv = async (execArgv) => {
|
||||
const standard = [];
|
||||
const convertedArgs = [];
|
||||
for (let i = 0; i < execArgv.length; i++) {
|
||||
const arg = execArgv[i];
|
||||
if (arg.indexOf('--inspect') === -1) {
|
||||
standard.push(arg);
|
||||
}
|
||||
else {
|
||||
const argName = arg.split('=')[0];
|
||||
const port = await getFreePort();
|
||||
convertedArgs.push(`${argName}=${port}`);
|
||||
}
|
||||
}
|
||||
return standard.concat(convertedArgs);
|
||||
};
|
||||
20
node_modules/bullmq/dist/esm/classes/errors/connection-closed-error.d.ts
generated
vendored
Normal file
20
node_modules/bullmq/dist/esm/classes/errors/connection-closed-error.d.ts
generated
vendored
Normal file
@@ -0,0 +1,20 @@
|
||||
/**
|
||||
* Error message used when a connection is closed.
|
||||
*
|
||||
* This mirrors the constant exported by ioredis (`ioredis/built/utils`) but is
|
||||
* defined locally so that BullMQ can be used without ioredis installed (for
|
||||
* example, when using the PostgreSQL backend).
|
||||
*/
|
||||
export declare const CONNECTION_CLOSED_ERROR_MSG = "Connection is closed.";
|
||||
/**
|
||||
* Thrown by any BullMQ Redis adapter (ioredis, node-redis, Bun, …) when a
|
||||
* command fails because the connection is already closed or was closed
|
||||
* mid-flight.
|
||||
*
|
||||
* Using a single well-known class lets {@link isNotConnectionError} do a
|
||||
* structural `instanceof` check rather than fragile message-substring matching.
|
||||
*/
|
||||
export declare class ConnectionClosedError extends Error {
|
||||
readonly cause?: unknown;
|
||||
constructor(message?: string, cause?: unknown);
|
||||
}
|
||||
24
node_modules/bullmq/dist/esm/classes/errors/connection-closed-error.js
generated
vendored
Normal file
24
node_modules/bullmq/dist/esm/classes/errors/connection-closed-error.js
generated
vendored
Normal file
@@ -0,0 +1,24 @@
|
||||
/**
|
||||
* Error message used when a connection is closed.
|
||||
*
|
||||
* This mirrors the constant exported by ioredis (`ioredis/built/utils`) but is
|
||||
* defined locally so that BullMQ can be used without ioredis installed (for
|
||||
* example, when using the PostgreSQL backend).
|
||||
*/
|
||||
export const CONNECTION_CLOSED_ERROR_MSG = 'Connection is closed.';
|
||||
/**
|
||||
* Thrown by any BullMQ Redis adapter (ioredis, node-redis, Bun, …) when a
|
||||
* command fails because the connection is already closed or was closed
|
||||
* mid-flight.
|
||||
*
|
||||
* Using a single well-known class lets {@link isNotConnectionError} do a
|
||||
* structural `instanceof` check rather than fragile message-substring matching.
|
||||
*/
|
||||
export class ConnectionClosedError extends Error {
|
||||
constructor(message, cause) {
|
||||
super(message !== null && message !== void 0 ? message : CONNECTION_CLOSED_ERROR_MSG);
|
||||
this.cause = cause;
|
||||
this.name = 'ConnectionClosedError';
|
||||
Object.setPrototypeOf(this, new.target.prototype);
|
||||
}
|
||||
}
|
||||
11
node_modules/bullmq/dist/esm/classes/errors/delayed-error.d.ts
generated
vendored
Normal file
11
node_modules/bullmq/dist/esm/classes/errors/delayed-error.d.ts
generated
vendored
Normal file
@@ -0,0 +1,11 @@
|
||||
export declare const DELAYED_ERROR = "bullmq:movedToDelayed";
|
||||
/**
|
||||
* DelayedError
|
||||
*
|
||||
* Error to be thrown when job is moved to delayed state
|
||||
* from job in active state.
|
||||
*
|
||||
*/
|
||||
export declare class DelayedError extends Error {
|
||||
constructor(message?: string);
|
||||
}
|
||||
15
node_modules/bullmq/dist/esm/classes/errors/delayed-error.js
generated
vendored
Normal file
15
node_modules/bullmq/dist/esm/classes/errors/delayed-error.js
generated
vendored
Normal file
@@ -0,0 +1,15 @@
|
||||
export const DELAYED_ERROR = 'bullmq:movedToDelayed';
|
||||
/**
|
||||
* DelayedError
|
||||
*
|
||||
* Error to be thrown when job is moved to delayed state
|
||||
* from job in active state.
|
||||
*
|
||||
*/
|
||||
export class DelayedError extends Error {
|
||||
constructor(message = DELAYED_ERROR) {
|
||||
super(message);
|
||||
this.name = this.constructor.name;
|
||||
Object.setPrototypeOf(this, new.target.prototype);
|
||||
}
|
||||
}
|
||||
6
node_modules/bullmq/dist/esm/classes/errors/index.d.ts
generated
vendored
Normal file
6
node_modules/bullmq/dist/esm/classes/errors/index.d.ts
generated
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
export * from './connection-closed-error';
|
||||
export * from './delayed-error';
|
||||
export * from './rate-limit-error';
|
||||
export * from './unrecoverable-error';
|
||||
export * from './waiting-children-error';
|
||||
export * from './waiting-error';
|
||||
6
node_modules/bullmq/dist/esm/classes/errors/index.js
generated
vendored
Normal file
6
node_modules/bullmq/dist/esm/classes/errors/index.js
generated
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
export * from './connection-closed-error';
|
||||
export * from './delayed-error';
|
||||
export * from './rate-limit-error';
|
||||
export * from './unrecoverable-error';
|
||||
export * from './waiting-children-error';
|
||||
export * from './waiting-error';
|
||||
10
node_modules/bullmq/dist/esm/classes/errors/rate-limit-error.d.ts
generated
vendored
Normal file
10
node_modules/bullmq/dist/esm/classes/errors/rate-limit-error.d.ts
generated
vendored
Normal file
@@ -0,0 +1,10 @@
|
||||
export declare const RATE_LIMIT_ERROR = "bullmq:rateLimitExceeded";
|
||||
/**
|
||||
* RateLimitError
|
||||
*
|
||||
* Error to be thrown when queue reaches a rate limit.
|
||||
*
|
||||
*/
|
||||
export declare class RateLimitError extends Error {
|
||||
constructor(message?: string);
|
||||
}
|
||||
14
node_modules/bullmq/dist/esm/classes/errors/rate-limit-error.js
generated
vendored
Normal file
14
node_modules/bullmq/dist/esm/classes/errors/rate-limit-error.js
generated
vendored
Normal file
@@ -0,0 +1,14 @@
|
||||
export const RATE_LIMIT_ERROR = 'bullmq:rateLimitExceeded';
|
||||
/**
|
||||
* RateLimitError
|
||||
*
|
||||
* Error to be thrown when queue reaches a rate limit.
|
||||
*
|
||||
*/
|
||||
export class RateLimitError extends Error {
|
||||
constructor(message = RATE_LIMIT_ERROR) {
|
||||
super(message);
|
||||
this.name = this.constructor.name;
|
||||
Object.setPrototypeOf(this, new.target.prototype);
|
||||
}
|
||||
}
|
||||
11
node_modules/bullmq/dist/esm/classes/errors/unrecoverable-error.d.ts
generated
vendored
Normal file
11
node_modules/bullmq/dist/esm/classes/errors/unrecoverable-error.d.ts
generated
vendored
Normal file
@@ -0,0 +1,11 @@
|
||||
export declare const UNRECOVERABLE_ERROR = "bullmq:unrecoverable";
|
||||
/**
|
||||
* UnrecoverableError
|
||||
*
|
||||
* Error to move a job to failed even if the attemptsMade
|
||||
* are lower than the expected limit.
|
||||
*
|
||||
*/
|
||||
export declare class UnrecoverableError extends Error {
|
||||
constructor(message?: string);
|
||||
}
|
||||
15
node_modules/bullmq/dist/esm/classes/errors/unrecoverable-error.js
generated
vendored
Normal file
15
node_modules/bullmq/dist/esm/classes/errors/unrecoverable-error.js
generated
vendored
Normal file
@@ -0,0 +1,15 @@
|
||||
export const UNRECOVERABLE_ERROR = 'bullmq:unrecoverable';
|
||||
/**
|
||||
* UnrecoverableError
|
||||
*
|
||||
* Error to move a job to failed even if the attemptsMade
|
||||
* are lower than the expected limit.
|
||||
*
|
||||
*/
|
||||
export class UnrecoverableError extends Error {
|
||||
constructor(message = UNRECOVERABLE_ERROR) {
|
||||
super(message);
|
||||
this.name = this.constructor.name;
|
||||
Object.setPrototypeOf(this, new.target.prototype);
|
||||
}
|
||||
}
|
||||
11
node_modules/bullmq/dist/esm/classes/errors/waiting-children-error.d.ts
generated
vendored
Normal file
11
node_modules/bullmq/dist/esm/classes/errors/waiting-children-error.d.ts
generated
vendored
Normal file
@@ -0,0 +1,11 @@
|
||||
export declare const WAITING_CHILDREN_ERROR = "bullmq:movedToWaitingChildren";
|
||||
/**
|
||||
* WaitingChildrenError
|
||||
*
|
||||
* Error to be thrown when job is moved to waiting-children state
|
||||
* from job in active state.
|
||||
*
|
||||
*/
|
||||
export declare class WaitingChildrenError extends Error {
|
||||
constructor(message?: string);
|
||||
}
|
||||
15
node_modules/bullmq/dist/esm/classes/errors/waiting-children-error.js
generated
vendored
Normal file
15
node_modules/bullmq/dist/esm/classes/errors/waiting-children-error.js
generated
vendored
Normal file
@@ -0,0 +1,15 @@
|
||||
export const WAITING_CHILDREN_ERROR = 'bullmq:movedToWaitingChildren';
|
||||
/**
|
||||
* WaitingChildrenError
|
||||
*
|
||||
* Error to be thrown when job is moved to waiting-children state
|
||||
* from job in active state.
|
||||
*
|
||||
*/
|
||||
export class WaitingChildrenError extends Error {
|
||||
constructor(message = WAITING_CHILDREN_ERROR) {
|
||||
super(message);
|
||||
this.name = this.constructor.name;
|
||||
Object.setPrototypeOf(this, new.target.prototype);
|
||||
}
|
||||
}
|
||||
10
node_modules/bullmq/dist/esm/classes/errors/waiting-error.d.ts
generated
vendored
Normal file
10
node_modules/bullmq/dist/esm/classes/errors/waiting-error.d.ts
generated
vendored
Normal file
@@ -0,0 +1,10 @@
|
||||
export declare const WAITING_ERROR = "bullmq:movedToWait";
|
||||
/**
|
||||
* WaitingError
|
||||
*
|
||||
* Error to be thrown when job is moved to wait or prioritized state
|
||||
* from job in active state.
|
||||
*/
|
||||
export declare class WaitingError extends Error {
|
||||
constructor(message?: string);
|
||||
}
|
||||
14
node_modules/bullmq/dist/esm/classes/errors/waiting-error.js
generated
vendored
Normal file
14
node_modules/bullmq/dist/esm/classes/errors/waiting-error.js
generated
vendored
Normal file
@@ -0,0 +1,14 @@
|
||||
export const WAITING_ERROR = 'bullmq:movedToWait';
|
||||
/**
|
||||
* WaitingError
|
||||
*
|
||||
* Error to be thrown when job is moved to wait or prioritized state
|
||||
* from job in active state.
|
||||
*/
|
||||
export class WaitingError extends Error {
|
||||
constructor(message = WAITING_ERROR) {
|
||||
super(message);
|
||||
this.name = this.constructor.name;
|
||||
Object.setPrototypeOf(this, new.target.prototype);
|
||||
}
|
||||
}
|
||||
14
node_modules/bullmq/dist/esm/classes/finished-errors.d.ts
generated
vendored
Normal file
14
node_modules/bullmq/dist/esm/classes/finished-errors.d.ts
generated
vendored
Normal file
@@ -0,0 +1,14 @@
|
||||
/**
|
||||
* Builds the canonical `Error` for a negative status code returned by a backend
|
||||
* operation (Lua script, SQL function, …). Shared by every backend so that the
|
||||
* error messages a caller sees are identical regardless of the datastore.
|
||||
*
|
||||
* The resulting error carries the numeric `code` so callers can branch on it.
|
||||
*/
|
||||
export declare function finishedErrors({ code, jobId, parentKey, command, state, }: {
|
||||
code: number;
|
||||
jobId?: string;
|
||||
parentKey?: string;
|
||||
command: string;
|
||||
state?: string;
|
||||
}): Error;
|
||||
52
node_modules/bullmq/dist/esm/classes/finished-errors.js
generated
vendored
Normal file
52
node_modules/bullmq/dist/esm/classes/finished-errors.js
generated
vendored
Normal file
@@ -0,0 +1,52 @@
|
||||
import { ErrorCode } from '../enums';
|
||||
import { UnrecoverableError } from './errors/unrecoverable-error';
|
||||
/**
|
||||
* Builds the canonical `Error` for a negative status code returned by a backend
|
||||
* operation (Lua script, SQL function, …). Shared by every backend so that the
|
||||
* error messages a caller sees are identical regardless of the datastore.
|
||||
*
|
||||
* The resulting error carries the numeric `code` so callers can branch on it.
|
||||
*/
|
||||
export function finishedErrors({ code, jobId, parentKey, command, state, }) {
|
||||
let error;
|
||||
switch (code) {
|
||||
case ErrorCode.JobNotExist:
|
||||
error = new Error(`Missing key for job ${jobId}. ${command}`);
|
||||
break;
|
||||
case ErrorCode.JobLockNotExist:
|
||||
error = new Error(`Missing lock for job ${jobId}. ${command}`);
|
||||
break;
|
||||
case ErrorCode.JobNotInState:
|
||||
error = new Error(`Job ${jobId} is not in the ${state} state. ${command}`);
|
||||
break;
|
||||
case ErrorCode.JobPendingChildren:
|
||||
error = new Error(`Job ${jobId} has pending dependencies. ${command}`);
|
||||
break;
|
||||
case ErrorCode.ParentJobNotExist:
|
||||
error = new Error(`Missing key for parent job ${parentKey}. ${command}`);
|
||||
break;
|
||||
case ErrorCode.JobLockMismatch:
|
||||
error = new Error(`Lock mismatch for job ${jobId}. Cmd ${command} from ${state}`);
|
||||
break;
|
||||
case ErrorCode.ParentJobCannotBeReplaced:
|
||||
error = new Error(`The parent job ${parentKey} cannot be replaced. ${command}`);
|
||||
break;
|
||||
case ErrorCode.JobBelongsToJobScheduler:
|
||||
error = new Error(`Job ${jobId} belongs to a job scheduler and cannot be removed directly. ${command}`);
|
||||
break;
|
||||
case ErrorCode.JobHasFailedChildren:
|
||||
error = new UnrecoverableError(`Cannot complete job ${jobId} because it has at least one failed child. ${command}`);
|
||||
break;
|
||||
case ErrorCode.SchedulerJobIdCollision:
|
||||
error = new Error(`Cannot create job scheduler iteration - job ID already exists. ${command}`);
|
||||
break;
|
||||
case ErrorCode.SchedulerJobSlotsBusy:
|
||||
error = new Error(`Cannot create job scheduler iteration - current and next time slots already have jobs. ${command}`);
|
||||
break;
|
||||
default:
|
||||
error = new Error(`Unknown code ${code} error for ${jobId}. ${command}`);
|
||||
}
|
||||
// Add the code property to the error object
|
||||
error.code = code;
|
||||
return error;
|
||||
}
|
||||
201
node_modules/bullmq/dist/esm/classes/flow-producer.d.ts
generated
vendored
Normal file
201
node_modules/bullmq/dist/esm/classes/flow-producer.d.ts
generated
vendored
Normal file
@@ -0,0 +1,201 @@
|
||||
import { EventEmitter } from 'events';
|
||||
import { BackendFactory, FlowJob, FlowProducerOptions, FlowQueuesOpts, FlowOpts, IoredisListener, IQueueBackend, JobJson, ParentKeyOpts, ParentOptions, Tracer, ContextManager } from '../interfaces';
|
||||
import { Job } from './job';
|
||||
import { RedisQueueBackend } from './redis-queue-backend';
|
||||
import { KeysMap } from './queue-keys';
|
||||
/**
|
||||
* A single job insert collected while walking a flow tree, ready to be handed
|
||||
* to the backend's atomic {@link IQueueBackend.addFlow} operation.
|
||||
*/
|
||||
export interface FlowJobEntry {
|
||||
jobData: JobJson;
|
||||
jobId: string;
|
||||
parentKeyOpts: ParentKeyOpts;
|
||||
prefix: string;
|
||||
queueName: string;
|
||||
}
|
||||
export interface AddNodeOpts {
|
||||
entries: FlowJobEntry[];
|
||||
node: FlowJob;
|
||||
parent?: {
|
||||
parentOpts: ParentOptions;
|
||||
parentDependenciesKey: string;
|
||||
};
|
||||
/**
|
||||
* Queues options that will be applied in each node depending on queue name presence.
|
||||
*/
|
||||
queuesOpts?: FlowQueuesOpts;
|
||||
}
|
||||
export interface AddChildrenOpts {
|
||||
entries: FlowJobEntry[];
|
||||
nodes: FlowJob[];
|
||||
parent: {
|
||||
parentOpts: ParentOptions;
|
||||
parentDependenciesKey: string;
|
||||
};
|
||||
queuesOpts?: FlowQueuesOpts;
|
||||
}
|
||||
export interface NodeOpts {
|
||||
/**
|
||||
* Root job queue name.
|
||||
*/
|
||||
queueName: string;
|
||||
/**
|
||||
* Prefix included in job key.
|
||||
*/
|
||||
prefix?: string;
|
||||
/**
|
||||
* Root job id.
|
||||
*/
|
||||
id: string;
|
||||
/**
|
||||
* Maximum depth or levels to visit in the tree.
|
||||
*/
|
||||
depth?: number;
|
||||
/**
|
||||
* Maximum quantity of children per type (processed, unprocessed).
|
||||
*/
|
||||
maxChildren?: number;
|
||||
}
|
||||
export interface JobNode {
|
||||
job: Job;
|
||||
children?: JobNode[];
|
||||
}
|
||||
export interface FlowProducerListener extends IoredisListener {
|
||||
/**
|
||||
* Listen to 'error' event.
|
||||
*
|
||||
* This event is triggered when an error is throw.
|
||||
*/
|
||||
error: (failedReason: Error) => void;
|
||||
}
|
||||
/**
|
||||
* This class allows to add jobs with dependencies between them in such
|
||||
* a way that it is possible to build complex flows.
|
||||
* Note: A flow is a tree-like structure of jobs that depend on each other.
|
||||
* Whenever the children of a given parent are completed, the parent
|
||||
* will be processed, being able to access the children's result data.
|
||||
* All Jobs can be in different queues, either children or parents,
|
||||
*/
|
||||
export declare class FlowProducer<B extends IQueueBackend = RedisQueueBackend> extends EventEmitter {
|
||||
opts: FlowProducerOptions;
|
||||
toKey: (name: string, type: string) => string;
|
||||
keys: KeysMap;
|
||||
closing: Promise<void> | undefined;
|
||||
protected backend: B;
|
||||
protected telemetry: {
|
||||
tracer: Tracer | undefined;
|
||||
contextManager: ContextManager | undefined;
|
||||
};
|
||||
constructor(opts?: FlowProducerOptions, backendFactory?: BackendFactory<B>);
|
||||
emit<U extends keyof FlowProducerListener>(event: U, ...args: Parameters<FlowProducerListener[U]>): boolean;
|
||||
off<U extends keyof FlowProducerListener>(eventName: U, listener: FlowProducerListener[U]): this;
|
||||
on<U extends keyof FlowProducerListener>(event: U, listener: FlowProducerListener[U]): this;
|
||||
once<U extends keyof FlowProducerListener>(event: U, listener: FlowProducerListener[U]): this;
|
||||
/**
|
||||
* Helper to easily extend Job class calls.
|
||||
*/
|
||||
protected get Job(): typeof Job;
|
||||
waitUntilReady(): Promise<void>;
|
||||
/**
|
||||
* Returns the datastore backend that powers this flow producer.
|
||||
*
|
||||
* The backend owns its connection and exposes every datastore-agnostic
|
||||
* operation through {@link IQueueBackend}. Datastore-specific escape hatches
|
||||
* (e.g. the raw Redis client) live on the concrete backend implementation,
|
||||
* and are exposed here when the flow producer is parameterized on that
|
||||
* concrete backend type (the default is the Redis backend).
|
||||
*/
|
||||
getBackend(): B;
|
||||
/**
|
||||
* Adds a flow.
|
||||
*
|
||||
* This call would be atomic, either it fails and no jobs will
|
||||
* be added to the queues, or it succeeds and all jobs will be added.
|
||||
*
|
||||
* @param flow - an object with a tree-like structure where children jobs
|
||||
* will be processed before their parents.
|
||||
* @param opts - options that will be applied to the flow object.
|
||||
*/
|
||||
add(flow: FlowJob, opts?: FlowOpts): Promise<JobNode>;
|
||||
/**
|
||||
* Get a flow.
|
||||
*
|
||||
* @param opts - an object with options for getting a JobNode.
|
||||
*/
|
||||
getFlow(opts: NodeOpts): Promise<JobNode>;
|
||||
/**
|
||||
* Adds multiple flows.
|
||||
*
|
||||
* A flow is a tree-like structure of jobs that depend on each other.
|
||||
* Whenever the children of a given parent are completed, the parent
|
||||
* will be processed, being able to access the children's result data.
|
||||
*
|
||||
* All Jobs can be in different queues, either children or parents,
|
||||
* however this call would be atomic, either it fails and no jobs will
|
||||
* be added to the queues, or it succeeds and all jobs will be added.
|
||||
*
|
||||
* @param flows - an array of objects with a tree-like structure where children jobs
|
||||
* will be processed before their parents.
|
||||
*/
|
||||
addBulk(flows: FlowJob[]): Promise<JobNode[]>;
|
||||
/**
|
||||
* Add a node (job) of a flow to the queue. This method will recursively
|
||||
* add all its children as well. Note that a given job can potentially be
|
||||
* a parent and a child job at the same time depending on where it is located
|
||||
* in the tree hierarchy.
|
||||
*
|
||||
* @param multi - IRedisTransaction
|
||||
* @param node - the node representing a job to be added to some queue
|
||||
* @param parent - parent data sent to children to create the "links" to their parent
|
||||
* @returns
|
||||
*/
|
||||
protected addNode({ entries, node, parent, queuesOpts, }: AddNodeOpts): Promise<JobNode>;
|
||||
/**
|
||||
* Adds nodes (jobs) of multiple flows to the queue. This method will recursively
|
||||
* add all its children as well. Note that a given job can potentially be
|
||||
* a parent and a child job at the same time depending on where it is located
|
||||
* in the tree hierarchy.
|
||||
*
|
||||
* @param multi - IRedisTransaction
|
||||
* @param nodes - the nodes representing jobs to be added to some queue
|
||||
* @returns
|
||||
*/
|
||||
/**
|
||||
* Collects a single job insert for a flow, preserving the same await point
|
||||
* as the previous transaction-based insert so that the relative order of
|
||||
* entries (in particular, roots before their descendants) is unchanged.
|
||||
*/
|
||||
private collectFlowEntry;
|
||||
protected addNodes(entries: FlowJobEntry[], nodes: FlowJob[]): Promise<JobNode[]>;
|
||||
private getNode;
|
||||
private validateFlowJobs;
|
||||
private addChildren;
|
||||
private getChildren;
|
||||
/**
|
||||
* Helper factory method that creates a queue-like object
|
||||
* required to create jobs in any queue.
|
||||
*
|
||||
* @param node - The flow node containing the queue name and other job options.
|
||||
* @param prefix - The key prefix for the queue (honored by the Redis backend only).
|
||||
* @returns A queue-like object with the keys, identity and backend needed to create jobs.
|
||||
*/
|
||||
private queueFromNode;
|
||||
/**
|
||||
* Translates numeric addJob Lua error codes returned by root flow exec.
|
||||
*
|
||||
* @param code - Numeric error code returned from Redis.
|
||||
* @param parentKey - Parent key for contextual error messages.
|
||||
*/
|
||||
private toFlowError;
|
||||
/**
|
||||
*
|
||||
* Closes the connection and returns a promise that resolves when the connection is closed.
|
||||
*/
|
||||
close(): Promise<void>;
|
||||
/**
|
||||
*
|
||||
* Force disconnects a connection.
|
||||
*/
|
||||
disconnect(): Promise<void>;
|
||||
}
|
||||
431
node_modules/bullmq/dist/esm/classes/flow-producer.js
generated
vendored
Normal file
431
node_modules/bullmq/dist/esm/classes/flow-producer.js
generated
vendored
Normal file
@@ -0,0 +1,431 @@
|
||||
import { EventEmitter } from 'events';
|
||||
import { getParentKey, randomUUID, trace } from '../utils';
|
||||
import { getDefaultBackendFactory } from '../utils/create-backend';
|
||||
import { Job } from './job';
|
||||
import { ErrorCode, SpanKind, TelemetryAttributes } from '../enums';
|
||||
/**
|
||||
* This class allows to add jobs with dependencies between them in such
|
||||
* a way that it is possible to build complex flows.
|
||||
* Note: A flow is a tree-like structure of jobs that depend on each other.
|
||||
* Whenever the children of a given parent are completed, the parent
|
||||
* will be processed, being able to access the children's result data.
|
||||
* All Jobs can be in different queues, either children or parents,
|
||||
*/
|
||||
export class FlowProducer extends EventEmitter {
|
||||
constructor(opts = { connection: {} }, backendFactory = getDefaultBackendFactory()) {
|
||||
super();
|
||||
this.opts = opts;
|
||||
this.opts = Object.assign({}, opts);
|
||||
// The flow producer is not bound to a single queue: each flow entry carries
|
||||
// its own queue identity, so the backend is created with an empty name.
|
||||
this.backend = backendFactory('', this.opts);
|
||||
this.backend.on('error', (error) => {
|
||||
if (this.listenerCount('error') > 0) {
|
||||
this.emit('error', error);
|
||||
}
|
||||
});
|
||||
this.backend.on('close', () => {
|
||||
if (!this.closing) {
|
||||
this.emit('ioredis:close');
|
||||
}
|
||||
});
|
||||
if (opts === null || opts === void 0 ? void 0 : opts.telemetry) {
|
||||
this.telemetry = opts.telemetry;
|
||||
}
|
||||
}
|
||||
emit(event, ...args) {
|
||||
return super.emit(event, ...args);
|
||||
}
|
||||
off(eventName, listener) {
|
||||
super.off(eventName, listener);
|
||||
return this;
|
||||
}
|
||||
on(event, listener) {
|
||||
super.on(event, listener);
|
||||
return this;
|
||||
}
|
||||
once(event, listener) {
|
||||
super.once(event, listener);
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* Helper to easily extend Job class calls.
|
||||
*/
|
||||
get Job() {
|
||||
return Job;
|
||||
}
|
||||
waitUntilReady() {
|
||||
return this.backend.waitUntilReady();
|
||||
}
|
||||
/**
|
||||
* Returns the datastore backend that powers this flow producer.
|
||||
*
|
||||
* The backend owns its connection and exposes every datastore-agnostic
|
||||
* operation through {@link IQueueBackend}. Datastore-specific escape hatches
|
||||
* (e.g. the raw Redis client) live on the concrete backend implementation,
|
||||
* and are exposed here when the flow producer is parameterized on that
|
||||
* concrete backend type (the default is the Redis backend).
|
||||
*/
|
||||
getBackend() {
|
||||
return this.backend;
|
||||
}
|
||||
/**
|
||||
* Adds a flow.
|
||||
*
|
||||
* This call would be atomic, either it fails and no jobs will
|
||||
* be added to the queues, or it succeeds and all jobs will be added.
|
||||
*
|
||||
* @param flow - an object with a tree-like structure where children jobs
|
||||
* will be processed before their parents.
|
||||
* @param opts - options that will be applied to the flow object.
|
||||
*/
|
||||
async add(flow, opts) {
|
||||
if (this.closing) {
|
||||
return;
|
||||
}
|
||||
this.validateFlowJobs([flow]);
|
||||
// Ensure the backend (and thus the connection) is ready before building
|
||||
// the per-node queue contexts used to create jobs.
|
||||
await this.backend.waitUntilReady();
|
||||
const flowOpts = flow === null || flow === void 0 ? void 0 : flow.opts;
|
||||
const parentOpts = flowOpts && 'parent' in flowOpts ? flowOpts.parent : undefined;
|
||||
const parentKey = getParentKey(parentOpts);
|
||||
const parentDependenciesKey = parentKey
|
||||
? `${parentKey}:dependencies`
|
||||
: undefined;
|
||||
return trace(this.telemetry, SpanKind.PRODUCER, flow.queueName, 'addFlow', flow.queueName, async (span) => {
|
||||
span === null || span === void 0 ? void 0 : span.setAttributes({
|
||||
[TelemetryAttributes.FlowName]: flow.name,
|
||||
});
|
||||
const entries = [];
|
||||
const jobsTree = await this.addNode({
|
||||
entries,
|
||||
node: flow,
|
||||
queuesOpts: opts === null || opts === void 0 ? void 0 : opts.queuesOptions,
|
||||
parent: {
|
||||
parentOpts,
|
||||
parentDependenciesKey,
|
||||
},
|
||||
});
|
||||
const results = await this.backend.addFlow(entries);
|
||||
const [result] = results || [];
|
||||
if (result) {
|
||||
const [err, jobId] = result;
|
||||
if (err) {
|
||||
throw err;
|
||||
}
|
||||
if (typeof jobId === 'number' && jobId < 0) {
|
||||
throw this.toFlowError(jobId, parentKey);
|
||||
}
|
||||
if (typeof jobId === 'string') {
|
||||
jobsTree.job.id = jobId;
|
||||
}
|
||||
}
|
||||
return jobsTree;
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Get a flow.
|
||||
*
|
||||
* @param opts - an object with options for getting a JobNode.
|
||||
*/
|
||||
async getFlow(opts) {
|
||||
if (this.closing) {
|
||||
return;
|
||||
}
|
||||
await this.backend.waitUntilReady();
|
||||
const updatedOpts = Object.assign({
|
||||
depth: 10,
|
||||
maxChildren: 20,
|
||||
prefix: this.opts.prefix,
|
||||
}, opts);
|
||||
const jobsTree = this.getNode(updatedOpts);
|
||||
return jobsTree;
|
||||
}
|
||||
/**
|
||||
* Adds multiple flows.
|
||||
*
|
||||
* A flow is a tree-like structure of jobs that depend on each other.
|
||||
* Whenever the children of a given parent are completed, the parent
|
||||
* will be processed, being able to access the children's result data.
|
||||
*
|
||||
* All Jobs can be in different queues, either children or parents,
|
||||
* however this call would be atomic, either it fails and no jobs will
|
||||
* be added to the queues, or it succeeds and all jobs will be added.
|
||||
*
|
||||
* @param flows - an array of objects with a tree-like structure where children jobs
|
||||
* will be processed before their parents.
|
||||
*/
|
||||
async addBulk(flows) {
|
||||
if (this.closing) {
|
||||
return;
|
||||
}
|
||||
this.validateFlowJobs(flows);
|
||||
// Ensure the backend (and thus the connection) is ready before building
|
||||
// the per-node queue contexts used to create jobs.
|
||||
await this.backend.waitUntilReady();
|
||||
return trace(this.telemetry, SpanKind.PRODUCER, '', 'addBulkFlows', '', async (span) => {
|
||||
span === null || span === void 0 ? void 0 : span.setAttributes({
|
||||
[TelemetryAttributes.BulkCount]: flows.length,
|
||||
[TelemetryAttributes.BulkNames]: flows
|
||||
.map(flow => flow.name)
|
||||
.join(','),
|
||||
});
|
||||
const entries = [];
|
||||
const jobsTrees = await this.addNodes(entries, flows);
|
||||
const results = await this.backend.addFlow(entries);
|
||||
for (let index = 0; index < jobsTrees.length; ++index) {
|
||||
const result = results === null || results === void 0 ? void 0 : results[index];
|
||||
if (!result) {
|
||||
continue;
|
||||
}
|
||||
const [err, jobId] = result;
|
||||
if (!err && typeof jobId === 'string') {
|
||||
jobsTrees[index].job.id = jobId;
|
||||
}
|
||||
}
|
||||
return jobsTrees;
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Add a node (job) of a flow to the queue. This method will recursively
|
||||
* add all its children as well. Note that a given job can potentially be
|
||||
* a parent and a child job at the same time depending on where it is located
|
||||
* in the tree hierarchy.
|
||||
*
|
||||
* @param multi - IRedisTransaction
|
||||
* @param node - the node representing a job to be added to some queue
|
||||
* @param parent - parent data sent to children to create the "links" to their parent
|
||||
* @returns
|
||||
*/
|
||||
async addNode({ entries, node, parent, queuesOpts, }) {
|
||||
var _a, _b;
|
||||
const prefix = node.prefix || this.opts.prefix;
|
||||
const queue = this.queueFromNode(node, prefix);
|
||||
const queueOpts = queuesOpts && queuesOpts[node.queueName];
|
||||
const jobsOpts = (_a = queueOpts === null || queueOpts === void 0 ? void 0 : queueOpts.defaultJobOptions) !== null && _a !== void 0 ? _a : {};
|
||||
const jobId = ((_b = node.opts) === null || _b === void 0 ? void 0 : _b.jobId) || randomUUID();
|
||||
return trace(this.telemetry, SpanKind.PRODUCER, node.queueName, 'addNode', node.queueName, async (span, srcPropagationMetadata) => {
|
||||
var _a, _b;
|
||||
span === null || span === void 0 ? void 0 : span.setAttributes({
|
||||
[TelemetryAttributes.JobName]: node.name,
|
||||
[TelemetryAttributes.JobId]: jobId,
|
||||
});
|
||||
const opts = node.opts;
|
||||
let telemetry = opts === null || opts === void 0 ? void 0 : opts.telemetry;
|
||||
if (srcPropagationMetadata && opts) {
|
||||
const omitContext = (_a = opts.telemetry) === null || _a === void 0 ? void 0 : _a.omitContext;
|
||||
const telemetryMetadata = ((_b = opts.telemetry) === null || _b === void 0 ? void 0 : _b.metadata) ||
|
||||
(!omitContext && srcPropagationMetadata);
|
||||
if (telemetryMetadata || omitContext) {
|
||||
telemetry = {
|
||||
metadata: telemetryMetadata,
|
||||
omitContext,
|
||||
};
|
||||
}
|
||||
}
|
||||
const job = new this.Job(queue, node.name, node.data, Object.assign(Object.assign(Object.assign({}, jobsOpts), opts), { parent: parent === null || parent === void 0 ? void 0 : parent.parentOpts, telemetry }), jobId);
|
||||
const parentKey = getParentKey(parent === null || parent === void 0 ? void 0 : parent.parentOpts);
|
||||
if (node.children && node.children.length > 0) {
|
||||
// Create the parent job, it will be a job in status "waiting-children".
|
||||
const parentId = jobId;
|
||||
await this.collectFlowEntry(entries, job, {
|
||||
parentDependenciesKey: parent === null || parent === void 0 ? void 0 : parent.parentDependenciesKey,
|
||||
addToWaitingChildren: true,
|
||||
parentKey,
|
||||
});
|
||||
// Queue identity is owned by the backend (the `queue` object above is
|
||||
// bound to this node's queue via the backend's `forQueue`).
|
||||
const parentDependenciesKey = `${queue.toKey(parentId)}:dependencies`;
|
||||
const children = await this.addChildren({
|
||||
entries,
|
||||
nodes: node.children,
|
||||
parent: {
|
||||
parentOpts: {
|
||||
id: parentId,
|
||||
queue: queue.qualifiedName,
|
||||
},
|
||||
parentDependenciesKey,
|
||||
},
|
||||
queuesOpts,
|
||||
});
|
||||
return { job, children };
|
||||
}
|
||||
else {
|
||||
await this.collectFlowEntry(entries, job, {
|
||||
parentDependenciesKey: parent === null || parent === void 0 ? void 0 : parent.parentDependenciesKey,
|
||||
parentKey,
|
||||
});
|
||||
return { job };
|
||||
}
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Adds nodes (jobs) of multiple flows to the queue. This method will recursively
|
||||
* add all its children as well. Note that a given job can potentially be
|
||||
* a parent and a child job at the same time depending on where it is located
|
||||
* in the tree hierarchy.
|
||||
*
|
||||
* @param multi - IRedisTransaction
|
||||
* @param nodes - the nodes representing jobs to be added to some queue
|
||||
* @returns
|
||||
*/
|
||||
/**
|
||||
* Collects a single job insert for a flow, preserving the same await point
|
||||
* as the previous transaction-based insert so that the relative order of
|
||||
* entries (in particular, roots before their descendants) is unchanged.
|
||||
*/
|
||||
async collectFlowEntry(entries, job, parentOpts) {
|
||||
entries.push(job.toFlowEntry(parentOpts));
|
||||
}
|
||||
addNodes(entries, nodes) {
|
||||
return Promise.all(nodes.map(node => {
|
||||
const nodeOpts = node === null || node === void 0 ? void 0 : node.opts;
|
||||
const parentOpts = nodeOpts && 'parent' in nodeOpts ? nodeOpts.parent : undefined;
|
||||
const parentKey = getParentKey(parentOpts);
|
||||
const parentDependenciesKey = parentKey
|
||||
? `${parentKey}:dependencies`
|
||||
: undefined;
|
||||
return this.addNode({
|
||||
entries,
|
||||
node,
|
||||
parent: {
|
||||
parentOpts,
|
||||
parentDependenciesKey,
|
||||
},
|
||||
});
|
||||
}));
|
||||
}
|
||||
async getNode(node) {
|
||||
const queue = this.queueFromNode(node, node.prefix);
|
||||
const job = await this.Job.fromId(queue, node.id);
|
||||
if (job) {
|
||||
const { processed = {}, unprocessed = [], failed = [], ignored = {}, } = await job.getDependencies({
|
||||
failed: {
|
||||
count: node.maxChildren,
|
||||
},
|
||||
processed: {
|
||||
count: node.maxChildren,
|
||||
},
|
||||
unprocessed: {
|
||||
count: node.maxChildren,
|
||||
},
|
||||
ignored: {
|
||||
count: node.maxChildren,
|
||||
},
|
||||
});
|
||||
const processedKeys = Object.keys(processed);
|
||||
const ignoredKeys = Object.keys(ignored);
|
||||
const childrenCount = processedKeys.length +
|
||||
unprocessed.length +
|
||||
ignoredKeys.length +
|
||||
failed.length;
|
||||
const newDepth = node.depth - 1;
|
||||
if (childrenCount > 0 && newDepth) {
|
||||
const children = await this.getChildren([...processedKeys, ...unprocessed, ...failed, ...ignoredKeys], newDepth, node.maxChildren);
|
||||
return { job, children };
|
||||
}
|
||||
else {
|
||||
return { job };
|
||||
}
|
||||
}
|
||||
}
|
||||
validateFlowJobs(nodes) {
|
||||
for (const node of nodes) {
|
||||
const children = node.children;
|
||||
if (children && children.length > 0) {
|
||||
const nodeOpts = node.opts;
|
||||
const hasDeduplication = nodeOpts && 'deduplication' in nodeOpts && nodeOpts.deduplication;
|
||||
if (hasDeduplication) {
|
||||
throw new Error('Deduplication options cannot be used on flow nodes with children');
|
||||
}
|
||||
this.validateFlowJobs(children);
|
||||
}
|
||||
}
|
||||
}
|
||||
addChildren({ entries, nodes, parent, queuesOpts }) {
|
||||
return Promise.all(nodes.map(node => this.addNode({ entries, node, parent, queuesOpts })));
|
||||
}
|
||||
getChildren(childrenKeys, depth, maxChildren) {
|
||||
const getChild = (key) => {
|
||||
const { prefix, queueName, id } = this.backend.parseNodeKey(key);
|
||||
return this.getNode({
|
||||
id,
|
||||
queueName,
|
||||
prefix,
|
||||
depth,
|
||||
maxChildren,
|
||||
});
|
||||
};
|
||||
return Promise.all([...childrenKeys.map(getChild)]);
|
||||
}
|
||||
/**
|
||||
* Helper factory method that creates a queue-like object
|
||||
* required to create jobs in any queue.
|
||||
*
|
||||
* @param node - The flow node containing the queue name and other job options.
|
||||
* @param prefix - The key prefix for the queue (honored by the Redis backend only).
|
||||
* @returns A queue-like object with the keys, identity and backend needed to create jobs.
|
||||
*/
|
||||
queueFromNode(node, prefix) {
|
||||
// Queue identity and key building are owned by the backend (the Redis
|
||||
// backend encodes the key `prefix`; other backends format their own
|
||||
// identity). The flow's own backend is queue-agnostic, so we ask it for a
|
||||
// sibling bound to this node's queue.
|
||||
const backend = this.backend.forQueue(node.queueName, prefix);
|
||||
return {
|
||||
name: node.queueName,
|
||||
keys: backend.keys,
|
||||
toKey: (type) => backend.toKey(type),
|
||||
opts: { prefix, connection: {} },
|
||||
qualifiedName: backend.qualifiedName,
|
||||
closing: this.closing,
|
||||
backend,
|
||||
waitUntilReady: async () => {
|
||||
await this.backend.waitUntilReady();
|
||||
},
|
||||
removeListener: this.removeListener.bind(this),
|
||||
emit: this.emit.bind(this),
|
||||
on: this.on.bind(this),
|
||||
trace: async () => { },
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Translates numeric addJob Lua error codes returned by root flow exec.
|
||||
*
|
||||
* @param code - Numeric error code returned from Redis.
|
||||
* @param parentKey - Parent key for contextual error messages.
|
||||
*/
|
||||
toFlowError(code, parentKey) {
|
||||
let error;
|
||||
switch (code) {
|
||||
case ErrorCode.ParentJobNotExist:
|
||||
error = new Error(`Missing key for parent job ${parentKey}. addJob`);
|
||||
break;
|
||||
case ErrorCode.ParentJobCannotBeReplaced:
|
||||
error = new Error(`The parent job ${parentKey} cannot be replaced. addJob`);
|
||||
break;
|
||||
default:
|
||||
error = new Error(`Unknown code ${code} error for addJob`);
|
||||
}
|
||||
error.code = code;
|
||||
return error;
|
||||
}
|
||||
/**
|
||||
*
|
||||
* Closes the connection and returns a promise that resolves when the connection is closed.
|
||||
*/
|
||||
async close() {
|
||||
if (!this.closing) {
|
||||
this.closing = this.backend.close();
|
||||
}
|
||||
await this.closing;
|
||||
}
|
||||
/**
|
||||
*
|
||||
* Force disconnects a connection.
|
||||
*/
|
||||
disconnect() {
|
||||
return this.backend.disconnect();
|
||||
}
|
||||
}
|
||||
24
node_modules/bullmq/dist/esm/classes/index.d.ts
generated
vendored
Normal file
24
node_modules/bullmq/dist/esm/classes/index.d.ts
generated
vendored
Normal file
@@ -0,0 +1,24 @@
|
||||
export * from './async-fifo-queue';
|
||||
export * from './backoffs';
|
||||
export * from './child';
|
||||
export * from './child-pool';
|
||||
export * from './child-processor';
|
||||
export * from './errors';
|
||||
export * from './flow-producer';
|
||||
export * from './ioredis-client';
|
||||
export * from './node-redis-client';
|
||||
export * from './bun-redis-client';
|
||||
export * from './valkey-glide-client';
|
||||
export * from './job';
|
||||
export * from './job-scheduler';
|
||||
export * from './lock-manager';
|
||||
export * from './queue-base';
|
||||
export * from './queue-events';
|
||||
export * from './queue-events-producer';
|
||||
export * from './queue-getters';
|
||||
export * from './queue-keys';
|
||||
export * from './queue';
|
||||
export * from './redis-connection';
|
||||
export * from './sandbox';
|
||||
export * from './redis-queue-backend';
|
||||
export * from './worker';
|
||||
26
node_modules/bullmq/dist/esm/classes/index.js
generated
vendored
Normal file
26
node_modules/bullmq/dist/esm/classes/index.js
generated
vendored
Normal file
@@ -0,0 +1,26 @@
|
||||
export * from './async-fifo-queue';
|
||||
export * from './backoffs';
|
||||
export * from './child';
|
||||
export * from './child-pool';
|
||||
export * from './child-processor';
|
||||
export * from './errors';
|
||||
export * from './flow-producer';
|
||||
export * from './ioredis-client';
|
||||
export * from './node-redis-client';
|
||||
export * from './bun-redis-client';
|
||||
export * from './valkey-glide-client';
|
||||
export * from './job';
|
||||
export * from './job-scheduler';
|
||||
// export * from './main'; this file must not be exported
|
||||
// export * from './main-worker'; this file must not be exported
|
||||
export * from './lock-manager';
|
||||
export * from './queue-base';
|
||||
export * from './queue-events';
|
||||
export * from './queue-events-producer';
|
||||
export * from './queue-getters';
|
||||
export * from './queue-keys';
|
||||
export * from './queue';
|
||||
export * from './redis-connection';
|
||||
export * from './sandbox';
|
||||
export * from './redis-queue-backend';
|
||||
export * from './worker';
|
||||
27
node_modules/bullmq/dist/esm/classes/ioredis-client.d.ts
generated
vendored
Normal file
27
node_modules/bullmq/dist/esm/classes/ioredis-client.d.ts
generated
vendored
Normal file
@@ -0,0 +1,27 @@
|
||||
import { Cluster, Redis } from 'ioredis';
|
||||
import { IRedisClient } from '../interfaces/redis-client';
|
||||
/**
|
||||
* Wraps an ioredis `Redis` / `Cluster` instance with a `Proxy` so it conforms
|
||||
* to {@link IRedisClient}.
|
||||
*
|
||||
* For backwards compatibility BullMQ continues to accept a raw `IORedis`
|
||||
* instance through tehe `connection` option, even though internally it relies
|
||||
* on the `IRedisClient` adapter interface. The returned proxy:
|
||||
*
|
||||
* - exposes `runCommand` (Lua script dispatch by name)
|
||||
* - exposes structured-options variants of `hset`, `set`, `zrange`,
|
||||
* `zrevrange`, `xadd`, `xread`, `xtrim`, `scan` (backward-compatible:
|
||||
* they still accept native ioredis varargs if called that way)
|
||||
* - returns augmented {@link IRedisTransaction}s from `pipeline()` / `multi()`
|
||||
* - wraps the result of `duplicate()` in a new proxy
|
||||
*
|
||||
* The underlying ioredis instance is **not** mutated. Properties and methods
|
||||
* not in the override table are forwarded to the raw client via the proxy
|
||||
* traps, with `this === target` so EventEmitter / Commander internals work
|
||||
* normally.
|
||||
*/
|
||||
export declare function createIORedisClient<TClient extends Redis | Cluster>(client: TClient): TClient & IRedisClient;
|
||||
/**
|
||||
* Check if an object already implements {@link IRedisClient}.
|
||||
*/
|
||||
export declare function isIRedisClient(obj: any): obj is IRedisClient;
|
||||
331
node_modules/bullmq/dist/esm/classes/ioredis-client.js
generated
vendored
Normal file
331
node_modules/bullmq/dist/esm/classes/ioredis-client.js
generated
vendored
Normal file
@@ -0,0 +1,331 @@
|
||||
/**
|
||||
* Per-raw-client cache so repeated calls to `createIORedisClient` with the
|
||||
* same underlying ioredis instance return the same proxy. This preserves
|
||||
* event-listener identity for the BullMQ-facing client.
|
||||
*/
|
||||
const proxyCache = new WeakMap();
|
||||
/**
|
||||
* Wraps an ioredis `Redis` / `Cluster` instance with a `Proxy` so it conforms
|
||||
* to {@link IRedisClient}.
|
||||
*
|
||||
* For backwards compatibility BullMQ continues to accept a raw `IORedis`
|
||||
* instance through tehe `connection` option, even though internally it relies
|
||||
* on the `IRedisClient` adapter interface. The returned proxy:
|
||||
*
|
||||
* - exposes `runCommand` (Lua script dispatch by name)
|
||||
* - exposes structured-options variants of `hset`, `set`, `zrange`,
|
||||
* `zrevrange`, `xadd`, `xread`, `xtrim`, `scan` (backward-compatible:
|
||||
* they still accept native ioredis varargs if called that way)
|
||||
* - returns augmented {@link IRedisTransaction}s from `pipeline()` / `multi()`
|
||||
* - wraps the result of `duplicate()` in a new proxy
|
||||
*
|
||||
* The underlying ioredis instance is **not** mutated. Properties and methods
|
||||
* not in the override table are forwarded to the raw client via the proxy
|
||||
* traps, with `this === target` so EventEmitter / Commander internals work
|
||||
* normally.
|
||||
*/
|
||||
export function createIORedisClient(client) {
|
||||
// If the caller already passed a proxy produced by this function, return
|
||||
// it as-is. Wrapping a proxy in a second proxy would defeat the WeakMap
|
||||
// cache (the inner raw client is no longer reachable from the outer
|
||||
// argument) and break listener-identity / equality checks for callers
|
||||
// that hold on to the original wrapper.
|
||||
if (client.__bullmq_iredis === true) {
|
||||
return client;
|
||||
}
|
||||
const cached = proxyCache.get(client);
|
||||
if (cached) {
|
||||
return cached;
|
||||
}
|
||||
const isCluster = client.isCluster === true;
|
||||
// Cache bound prototype methods so the returned function identity is
|
||||
// stable across accesses (important for `once`/`removeListener` patterns).
|
||||
const boundCache = new Map();
|
||||
// Override table — properties returned by the proxy without touching the
|
||||
// underlying ioredis instance. The arrow functions close over `client`
|
||||
// directly so ioredis internals always see the raw instance as `this`.
|
||||
const overrides = Object.create(null);
|
||||
overrides.__bullmq_iredis = true;
|
||||
overrides.isCluster = isCluster;
|
||||
// Lua script engine.
|
||||
overrides.runCommand = (name, args) => {
|
||||
return client[name](args);
|
||||
};
|
||||
// Pipeline / Multi — wrap the ChainableCommander with structured overrides.
|
||||
overrides.pipeline = (...args) => {
|
||||
return augmentTransaction(client.pipeline(...args));
|
||||
};
|
||||
overrides.multi = (...args) => {
|
||||
return augmentTransaction(client.multi(...args));
|
||||
};
|
||||
// duplicate — wrap the new raw client with a fresh proxy.
|
||||
// ioredis Cluster.duplicate(startupNodes?, options?) expects connection
|
||||
// options under `redisOptions`, while Redis.duplicate(options?) takes them
|
||||
// at the top level. Normalise so callers can always pass `{ connectionName }`.
|
||||
if (typeof client.duplicate === 'function') {
|
||||
overrides.duplicate = (opts) => {
|
||||
var _a;
|
||||
if (isCluster) {
|
||||
const existingRedisOpts = ((_a = client.options) === null || _a === void 0 ? void 0 : _a.redisOptions) || {};
|
||||
const mergedRedisOpts = opts
|
||||
? Object.assign(Object.assign({}, existingRedisOpts), opts) : existingRedisOpts;
|
||||
return createIORedisClient(client.duplicate(undefined, {
|
||||
redisOptions: mergedRedisOpts,
|
||||
}));
|
||||
}
|
||||
return createIORedisClient(client.duplicate(opts));
|
||||
};
|
||||
}
|
||||
// --- Structured → ioredis varargs translations ---
|
||||
// Each override accepts both the IRedisClient structured-options form and
|
||||
// the native ioredis varargs form, dispatching by argument shape.
|
||||
// hset: structured { f1: v1 } → ioredis hset(key, f1, v1, …)
|
||||
overrides.hset = (key, dataOrField, ...rest) => {
|
||||
if (typeof dataOrField === 'string') {
|
||||
return client.hset(key, dataOrField, ...rest);
|
||||
}
|
||||
const args = [key];
|
||||
for (const [f, v] of Object.entries(dataOrField)) {
|
||||
args.push(f, v);
|
||||
}
|
||||
return client.hset(...args);
|
||||
};
|
||||
// set: structured { PX?: n } → ioredis set(key, value, 'PX', n)
|
||||
overrides.set = (key, value, optionsOrModifier, ...rest) => {
|
||||
if (typeof optionsOrModifier === 'string' || optionsOrModifier == null) {
|
||||
return client.set(key, value, ...(optionsOrModifier != null ? [optionsOrModifier, ...rest] : []));
|
||||
}
|
||||
const args = [key, value];
|
||||
if (optionsOrModifier.PX != null) {
|
||||
args.push('PX', optionsOrModifier.PX);
|
||||
}
|
||||
else if (optionsOrModifier.EX != null) {
|
||||
args.push('EX', optionsOrModifier.EX);
|
||||
}
|
||||
return client.set(...args);
|
||||
};
|
||||
// zrange: structured { WITHSCORES? } → ioredis zrange(key, start, end, 'WITHSCORES')
|
||||
overrides.zrange = (key, start, end, optionsOrStr, ...rest) => {
|
||||
if (typeof optionsOrStr === 'string') {
|
||||
return client.zrange(key, start, end, optionsOrStr, ...rest);
|
||||
}
|
||||
if (optionsOrStr === null || optionsOrStr === void 0 ? void 0 : optionsOrStr.WITHSCORES) {
|
||||
return client.zrange(key, start, end, 'WITHSCORES');
|
||||
}
|
||||
return client.zrange(key, start, end);
|
||||
};
|
||||
// zrevrange: structured { WITHSCORES? } → ioredis zrevrange(key, start, end, 'WITHSCORES')
|
||||
overrides.zrevrange = (key, start, end, optionsOrStr, ...rest) => {
|
||||
if (typeof optionsOrStr === 'string') {
|
||||
return client.zrevrange(key, start, end, optionsOrStr, ...rest);
|
||||
}
|
||||
if (optionsOrStr === null || optionsOrStr === void 0 ? void 0 : optionsOrStr.WITHSCORES) {
|
||||
return client.zrevrange(key, start, end, 'WITHSCORES');
|
||||
}
|
||||
return client.zrevrange(key, start, end);
|
||||
};
|
||||
// xadd: structured (key, id, { field: value }, { MAXLEN? }) → ioredis varargs
|
||||
overrides.xadd = (key, idOrModifier, fieldsOrArg, ...rest) => {
|
||||
if (typeof fieldsOrArg === 'string') {
|
||||
return client.xadd(key, idOrModifier, fieldsOrArg, ...rest);
|
||||
}
|
||||
const options = rest[0];
|
||||
const args = [key];
|
||||
if ((options === null || options === void 0 ? void 0 : options.MAXLEN) != null) {
|
||||
args.push('MAXLEN');
|
||||
if (options.approximate !== false) {
|
||||
args.push('~');
|
||||
}
|
||||
args.push(options.MAXLEN);
|
||||
}
|
||||
args.push(idOrModifier);
|
||||
for (const [f, v] of Object.entries(fieldsOrArg)) {
|
||||
args.push(f, v);
|
||||
}
|
||||
return client.xadd(...args);
|
||||
};
|
||||
// xread: structured ([{ key, id }], { BLOCK?, COUNT? }) → ioredis varargs
|
||||
overrides.xread = (streamsOrModifier, ...rest) => {
|
||||
if (typeof streamsOrModifier === 'string') {
|
||||
return client.xread(streamsOrModifier, ...rest);
|
||||
}
|
||||
const options = rest[0];
|
||||
const args = [];
|
||||
if ((options === null || options === void 0 ? void 0 : options.BLOCK) != null) {
|
||||
args.push('BLOCK', options.BLOCK);
|
||||
}
|
||||
if ((options === null || options === void 0 ? void 0 : options.COUNT) != null) {
|
||||
args.push('COUNT', options.COUNT);
|
||||
}
|
||||
args.push('STREAMS');
|
||||
for (const s of streamsOrModifier) {
|
||||
args.push(s.key);
|
||||
}
|
||||
for (const s of streamsOrModifier) {
|
||||
args.push(s.id);
|
||||
}
|
||||
return client.xread(...args);
|
||||
};
|
||||
// xtrim: structured (key, 'MAXLEN', threshold, { approximate? })
|
||||
overrides.xtrim = (key, strategy, thresholdOrApprox, ...rest) => {
|
||||
if (typeof thresholdOrApprox === 'string' || rest.length === 0) {
|
||||
return client.xtrim(key, strategy, thresholdOrApprox, ...rest);
|
||||
}
|
||||
const options = rest[0];
|
||||
const args = [key, strategy];
|
||||
if ((options === null || options === void 0 ? void 0 : options.approximate) !== false) {
|
||||
args.push('~');
|
||||
}
|
||||
args.push(thresholdOrApprox);
|
||||
return client.xtrim(...args);
|
||||
};
|
||||
// bzpopmin is not overridden — ioredis already returns
|
||||
// `[key, member, score]`, which matches IRedisClient.
|
||||
// clientSetName / clientList helpers that forward to CLIENT subcommands.
|
||||
overrides.clientSetName = (name) => client.client('SETNAME', name);
|
||||
overrides.clientList = () => client.client('LIST');
|
||||
// scan(cursor, { MATCH?, COUNT? }) — accepts either structured options or
|
||||
// ioredis varargs (used internally by `scanStream`).
|
||||
overrides.scan = (cursor, ...rest) => {
|
||||
if (rest.length === 0 ||
|
||||
typeof rest[0] === 'string' ||
|
||||
typeof rest[0] === 'function') {
|
||||
return client.scan(cursor, ...rest);
|
||||
}
|
||||
const options = rest[0];
|
||||
const args = [cursor];
|
||||
if ((options === null || options === void 0 ? void 0 : options.MATCH) != null) {
|
||||
args.push('MATCH', options.MATCH);
|
||||
}
|
||||
if ((options === null || options === void 0 ? void 0 : options.COUNT) != null) {
|
||||
args.push('COUNT', options.COUNT);
|
||||
}
|
||||
return client.scan(...args);
|
||||
};
|
||||
const proxy = new Proxy(client, {
|
||||
get(target, prop) {
|
||||
if (prop in overrides) {
|
||||
return overrides[prop];
|
||||
}
|
||||
// Read against the raw target so getters on the prototype (e.g.
|
||||
// ioredis' EventEmitter internals) see `this === target` rather than
|
||||
// the proxy. This avoids infinite recursion through the proxy traps.
|
||||
const value = Reflect.get(target, prop, target);
|
||||
if (typeof value !== 'function') {
|
||||
return value;
|
||||
}
|
||||
// Own properties (including ioredis commands installed via
|
||||
// `defineCommand` and test-time spies set via `obj.method = spy`)
|
||||
// are bound fresh on each access so reassignment is honoured.
|
||||
if (Object.prototype.hasOwnProperty.call(target, prop)) {
|
||||
return value.bind(target);
|
||||
}
|
||||
// Prototype methods (EventEmitter, Commander, ...) are cached so
|
||||
// identity is stable across accesses.
|
||||
const cachedBound = boundCache.get(prop);
|
||||
if (cachedBound !== undefined) {
|
||||
return cachedBound;
|
||||
}
|
||||
const bound = value.bind(target);
|
||||
boundCache.set(prop, bound);
|
||||
return bound;
|
||||
},
|
||||
set(target, prop, value) {
|
||||
// Two assignment paths:
|
||||
// - Properties present in the override table are reassigned in the
|
||||
// table itself, so subsequent `get` traps return the new value
|
||||
// (used by sinon-style spies that stub `runCommand`, `pipeline`,
|
||||
// etc. on the proxy).
|
||||
// - All other properties are written through to the raw ioredis
|
||||
// instance via `Reflect.set`, and any stale bound-method entry is
|
||||
// invalidated so the next access rebinds the new function.
|
||||
if (prop in overrides) {
|
||||
overrides[prop] = value;
|
||||
return true;
|
||||
}
|
||||
boundCache.delete(prop);
|
||||
return Reflect.set(target, prop, value);
|
||||
},
|
||||
deleteProperty(target, prop) {
|
||||
if (prop in overrides) {
|
||||
return false;
|
||||
}
|
||||
boundCache.delete(prop);
|
||||
return Reflect.deleteProperty(target, prop);
|
||||
},
|
||||
has(target, prop) {
|
||||
return prop in overrides || Reflect.has(target, prop);
|
||||
},
|
||||
});
|
||||
proxyCache.set(client, proxy);
|
||||
return proxy;
|
||||
}
|
||||
/**
|
||||
* Adds `runCommand` and structured overrides to an ioredis ChainableCommander
|
||||
* so it satisfies {@link IRedisTransaction}.
|
||||
*/
|
||||
function augmentTransaction(commander) {
|
||||
const transaction = commander;
|
||||
transaction.runCommand = function (name, args) {
|
||||
transaction[name](args);
|
||||
return transaction;
|
||||
};
|
||||
// hset(key, { f1: v1 }) → ioredis pipeline.hset(key, f1, v1, …)
|
||||
const origHset = transaction.hset.bind(transaction);
|
||||
transaction.hset = function (key, data) {
|
||||
const args = [key];
|
||||
for (const [f, v] of Object.entries(data)) {
|
||||
args.push(f, v);
|
||||
}
|
||||
origHset(...args);
|
||||
return transaction;
|
||||
};
|
||||
// hscan(key, cursor, { COUNT? }) → ioredis hscan(key, cursor, 'COUNT', n)
|
||||
const origHscan = transaction.hscan.bind(transaction);
|
||||
transaction.hscan = function (key, cursor, options) {
|
||||
if ((options === null || options === void 0 ? void 0 : options.COUNT) != null) {
|
||||
origHscan(key, cursor, 'COUNT', options.COUNT);
|
||||
}
|
||||
else {
|
||||
origHscan(key, cursor);
|
||||
}
|
||||
return transaction;
|
||||
};
|
||||
// sscan(key, cursor, { COUNT? })
|
||||
const origSscan = transaction.sscan.bind(transaction);
|
||||
transaction.sscan = function (key, cursor, options) {
|
||||
if ((options === null || options === void 0 ? void 0 : options.COUNT) != null) {
|
||||
origSscan(key, cursor, 'COUNT', options.COUNT);
|
||||
}
|
||||
else {
|
||||
origSscan(key, cursor);
|
||||
}
|
||||
return transaction;
|
||||
};
|
||||
return transaction;
|
||||
}
|
||||
/**
|
||||
* Check if an object already implements {@link IRedisClient}.
|
||||
*/
|
||||
export function isIRedisClient(obj) {
|
||||
if (!obj || typeof obj !== 'object') {
|
||||
return false;
|
||||
}
|
||||
// Fast path for ioredis instances already wrapped by `createIORedisClient`.
|
||||
if (obj.__bullmq_iredis === true) {
|
||||
return true;
|
||||
}
|
||||
// Fallback structural check for wrapper-based adapters
|
||||
// (node-redis, Bun, or custom IRedisClient implementations).
|
||||
return (typeof obj.runCommand === 'function' &&
|
||||
typeof obj.defineCommand === 'function' &&
|
||||
typeof obj.pipeline === 'function' &&
|
||||
typeof obj.multi === 'function' &&
|
||||
typeof obj.duplicate === 'function' &&
|
||||
typeof obj.scanStream === 'function' &&
|
||||
typeof obj.connect === 'function' &&
|
||||
typeof obj.disconnect === 'function' &&
|
||||
typeof obj.on === 'function' &&
|
||||
typeof obj.status === 'string' &&
|
||||
typeof obj.isCluster === 'boolean');
|
||||
}
|
||||
51
node_modules/bullmq/dist/esm/classes/job-scheduler.d.ts
generated
vendored
Normal file
51
node_modules/bullmq/dist/esm/classes/job-scheduler.d.ts
generated
vendored
Normal file
@@ -0,0 +1,51 @@
|
||||
import { BackendFactory, JobSchedulerJson, RepeatBaseOptions, RepeatOptions } from '../interfaces';
|
||||
import { JobSchedulerTemplateOptions } from '../types';
|
||||
import { Job } from './job';
|
||||
import { QueueBase } from './queue-base';
|
||||
export declare const LEGACY_REPEATABLE_JOBS_MIGRATION_URL = "https://docs.bullmq.io/guide/migrations/migrate-from-v5-to-v6";
|
||||
export declare function hasLegacyRepeatableKeyShape(key: string): boolean;
|
||||
export declare const isLegacyRepeatableJobKey: typeof hasLegacyRepeatableKeyShape;
|
||||
export declare function getLegacyRepeatableJobError(key: string): Error;
|
||||
export declare class JobScheduler extends QueueBase {
|
||||
private repeatStrategy;
|
||||
constructor(name: string, opts: RepeatBaseOptions, backendFactory?: BackendFactory);
|
||||
upsertJobScheduler<T = any, R = any, N extends string = string>(jobSchedulerId: string, repeatOpts: Omit<RepeatOptions, 'key' | 'prevMillis'>, jobName: N, jobData: T, opts: JobSchedulerTemplateOptions, { override, producerId }: {
|
||||
override: boolean;
|
||||
producerId?: string;
|
||||
}): Promise<Job<T, R, N> | undefined>;
|
||||
private getNextJobOpts;
|
||||
removeJobScheduler(jobSchedulerId: string): Promise<number>;
|
||||
private getSchedulerData;
|
||||
private transformSchedulerData;
|
||||
/**
|
||||
* Checks if a given id corresponds to a registered job scheduler.
|
||||
*
|
||||
* This is used to disambiguate between new job scheduler ids (which may
|
||||
* contain any number of colon segments) and legacy repeatable job keys
|
||||
* (which always contain 5+ colon segments). Relying purely on segment
|
||||
* count is not safe because a user-provided jobSchedulerId may itself
|
||||
* contain 5+ colon segments, which would otherwise be misclassified as
|
||||
* a legacy repeatable key.
|
||||
*
|
||||
* We cannot use ZSCORE on the shared `repeat` sorted set because legacy
|
||||
* repeatable jobs are stored in the same sorted set and would be reported
|
||||
* as schedulers. Instead, we probe the per-id metadata hash (`repeat:<id>`)
|
||||
* for the `ic` (iteration count) field, which is written exclusively by
|
||||
* `storeJobScheduler` and is never set by the legacy `addRepeatableJob`
|
||||
* flow.
|
||||
*/
|
||||
isJobScheduler(id: string): Promise<boolean>;
|
||||
getScheduler<D = any>(id: string): Promise<JobSchedulerJson<D> | undefined>;
|
||||
private getTemplateFromJSON;
|
||||
getJobSchedulers<D = any>(start?: number, end?: number, asc?: boolean): Promise<JobSchedulerJson<D>[]>;
|
||||
getSchedulersCount(): Promise<number>;
|
||||
private getSchedulerNextJobId;
|
||||
}
|
||||
export declare const defaultRepeatStrategy: (millis: number, opts: RepeatOptions) => number | undefined;
|
||||
/**
|
||||
* Computes the next execution time (in ms since epoch) for the given repeat
|
||||
* options, supporting both `.every` (fixed interval) and `.pattern` (cron)
|
||||
* strategies. This is the default repeat strategy used to schedule the next
|
||||
* iteration of a job scheduler.
|
||||
*/
|
||||
export declare const getNextMillis: (millis: number, opts: RepeatOptions) => number | undefined;
|
||||
339
node_modules/bullmq/dist/esm/classes/job-scheduler.js
generated
vendored
Normal file
339
node_modules/bullmq/dist/esm/classes/job-scheduler.js
generated
vendored
Normal file
@@ -0,0 +1,339 @@
|
||||
import { __rest } from "tslib";
|
||||
import { CronExpressionParser } from 'cron-parser';
|
||||
import { Job } from './job';
|
||||
import { QueueBase } from './queue-base';
|
||||
import { SpanKind, TelemetryAttributes } from '../enums';
|
||||
import { array2obj } from '../utils';
|
||||
export const LEGACY_REPEATABLE_JOBS_MIGRATION_URL = 'https://docs.bullmq.io/guide/migrations/migrate-from-v5-to-v6';
|
||||
/**
|
||||
* Legacy repeatable job keys use the format `name:id:endDate:tz:pattern`.
|
||||
* The suffix after `name:id:endDate:tz:` is either:
|
||||
* - a cron pattern (contains spaces), or
|
||||
* - an `every` interval (purely numeric).
|
||||
*/
|
||||
/**
|
||||
* Returns true when key[from..to) is composed only of ASCII digits.
|
||||
*/
|
||||
function isNumericSegment(key, from, to) {
|
||||
if (from >= to) {
|
||||
return false;
|
||||
}
|
||||
for (let i = from; i < to; i++) {
|
||||
const charCode = key.charCodeAt(i);
|
||||
if (charCode < 48 || charCode > 57) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
export function hasLegacyRepeatableKeyShape(key) {
|
||||
const firstColon = key.indexOf(':');
|
||||
if (firstColon === -1) {
|
||||
return false;
|
||||
}
|
||||
const secondColon = key.indexOf(':', firstColon + 1);
|
||||
if (secondColon === -1) {
|
||||
return false;
|
||||
}
|
||||
const thirdColon = key.indexOf(':', secondColon + 1);
|
||||
if (thirdColon === -1) {
|
||||
return false;
|
||||
}
|
||||
const fourthColon = key.indexOf(':', thirdColon + 1);
|
||||
if (fourthColon === -1) {
|
||||
return false;
|
||||
}
|
||||
// endDate can be empty or numeric in legacy keys.
|
||||
if (secondColon + 1 < thirdColon &&
|
||||
!isNumericSegment(key, secondColon + 1, thirdColon)) {
|
||||
return false;
|
||||
}
|
||||
const suffixStart = fourthColon + 1;
|
||||
if (suffixStart >= key.length) {
|
||||
return false;
|
||||
}
|
||||
if (key.indexOf(' ', suffixStart) !== -1) {
|
||||
return true;
|
||||
}
|
||||
return isNumericSegment(key, suffixStart, key.length);
|
||||
}
|
||||
export const isLegacyRepeatableJobKey = hasLegacyRepeatableKeyShape;
|
||||
export function getLegacyRepeatableJobError(key) {
|
||||
return new Error(`Legacy repeatable job metadata is not supported in BullMQ v6 ` +
|
||||
`(key: "${key}"). Migrate legacy repeatable jobs to Job Schedulers ` +
|
||||
`before upgrading. See ${LEGACY_REPEATABLE_JOBS_MIGRATION_URL}`);
|
||||
}
|
||||
export class JobScheduler extends QueueBase {
|
||||
constructor(name, opts, backendFactory) {
|
||||
super(name, opts, backendFactory);
|
||||
this.repeatStrategy =
|
||||
(opts.settings && opts.settings.repeatStrategy) || defaultRepeatStrategy;
|
||||
}
|
||||
async upsertJobScheduler(jobSchedulerId, repeatOpts, jobName, jobData, opts, { override, producerId }) {
|
||||
const { every, limit, pattern, offset } = repeatOpts;
|
||||
if (pattern && every) {
|
||||
throw new Error('Both .pattern and .every options are defined for this repeatable job');
|
||||
}
|
||||
if (!pattern && !every) {
|
||||
throw new Error('Either .pattern or .every options must be defined for this repeatable job');
|
||||
}
|
||||
if (repeatOpts.immediately && repeatOpts.startDate) {
|
||||
throw new Error('Both .immediately and .startDate options are defined for this repeatable job');
|
||||
}
|
||||
if (repeatOpts.immediately && repeatOpts.every) {
|
||||
console.warn("Using option immediately with every does not affect the job's schedule. Job will run immediately anyway.");
|
||||
}
|
||||
if (Object.prototype.hasOwnProperty.call(opts, 'debounce')) {
|
||||
throw new Error('Debounce option has been removed. Use deduplication option instead');
|
||||
}
|
||||
// Check if we reached the limit of the repeatable job's iterations
|
||||
const iterationCount = repeatOpts.count ? repeatOpts.count + 1 : 1;
|
||||
if (typeof repeatOpts.limit !== 'undefined' &&
|
||||
iterationCount > repeatOpts.limit) {
|
||||
return;
|
||||
}
|
||||
// Check if we reached the end date of the repeatable job
|
||||
let now = Date.now();
|
||||
const { endDate } = repeatOpts;
|
||||
if (endDate && now > new Date(endDate).getTime()) {
|
||||
return;
|
||||
}
|
||||
const prevMillis = opts.prevMillis || 0;
|
||||
now = prevMillis < now ? now : prevMillis;
|
||||
// Check if we have a start date for the repeatable job
|
||||
const { immediately } = repeatOpts, filteredRepeatOpts = __rest(repeatOpts, ["immediately"]);
|
||||
let nextMillis;
|
||||
const newOffset = every && offset ? offset : null;
|
||||
if (pattern) {
|
||||
nextMillis = await this.repeatStrategy(now, repeatOpts, jobName);
|
||||
if (nextMillis < now) {
|
||||
nextMillis = now;
|
||||
}
|
||||
}
|
||||
if (nextMillis || every) {
|
||||
return this.trace(SpanKind.PRODUCER, 'add', `${this.name}.${jobName}`, async (span, srcPropagationMetadata) => {
|
||||
var _a, _b;
|
||||
let telemetry = opts.telemetry;
|
||||
if (srcPropagationMetadata) {
|
||||
const omitContext = (_a = opts.telemetry) === null || _a === void 0 ? void 0 : _a.omitContext;
|
||||
const telemetryMetadata = ((_b = opts.telemetry) === null || _b === void 0 ? void 0 : _b.metadata) ||
|
||||
(!omitContext && srcPropagationMetadata);
|
||||
if (telemetryMetadata || omitContext) {
|
||||
telemetry = {
|
||||
metadata: telemetryMetadata,
|
||||
omitContext,
|
||||
};
|
||||
}
|
||||
}
|
||||
const mergedOpts = this.getNextJobOpts(nextMillis, jobSchedulerId, Object.assign(Object.assign({}, opts), { repeat: filteredRepeatOpts, telemetry }), iterationCount, newOffset);
|
||||
if (override) {
|
||||
// Clamp nextMillis to now if it's in the past
|
||||
if (nextMillis < now) {
|
||||
nextMillis = now;
|
||||
}
|
||||
const [jobId, delay] = await this.backend.addJobScheduler(jobSchedulerId, nextMillis, JSON.stringify(typeof jobData === 'undefined' ? {} : jobData), opts, {
|
||||
name: jobName,
|
||||
startDate: repeatOpts.startDate
|
||||
? new Date(repeatOpts.startDate).getTime()
|
||||
: undefined,
|
||||
endDate: endDate ? new Date(endDate).getTime() : undefined,
|
||||
tz: repeatOpts.tz,
|
||||
pattern,
|
||||
every,
|
||||
limit,
|
||||
offset: newOffset,
|
||||
}, mergedOpts, producerId);
|
||||
// Ensure delay is a number (Dragonflydb may return it as a string)
|
||||
const numericDelay = typeof delay === 'string' ? parseInt(delay, 10) : delay;
|
||||
const job = new this.Job(this, jobName, jobData, Object.assign(Object.assign({}, mergedOpts), { delay: numericDelay }), jobId);
|
||||
job.id = jobId;
|
||||
span === null || span === void 0 ? void 0 : span.setAttributes({
|
||||
[TelemetryAttributes.JobSchedulerId]: jobSchedulerId,
|
||||
[TelemetryAttributes.JobId]: job.id,
|
||||
});
|
||||
return job;
|
||||
}
|
||||
else {
|
||||
const jobId = await this.backend.updateJobSchedulerNextMillis(jobSchedulerId, nextMillis, JSON.stringify(typeof jobData === 'undefined' ? {} : jobData), mergedOpts, producerId);
|
||||
if (jobId) {
|
||||
const job = new this.Job(this, jobName, jobData, mergedOpts, jobId);
|
||||
job.id = jobId;
|
||||
span === null || span === void 0 ? void 0 : span.setAttributes({
|
||||
[TelemetryAttributes.JobSchedulerId]: jobSchedulerId,
|
||||
[TelemetryAttributes.JobId]: job.id,
|
||||
});
|
||||
return job;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
getNextJobOpts(nextMillis, jobSchedulerId, opts, currentCount, offset) {
|
||||
var _a, _b;
|
||||
//
|
||||
// Generate unique job id for this iteration.
|
||||
//
|
||||
const jobId = this.getSchedulerNextJobId({
|
||||
jobSchedulerId,
|
||||
nextMillis,
|
||||
});
|
||||
const now = Date.now();
|
||||
const delay = nextMillis + offset - now;
|
||||
const mergedOpts = Object.assign(Object.assign({}, opts), { jobId, delay: delay < 0 ? 0 : delay, timestamp: now, prevMillis: nextMillis, repeatJobKey: jobSchedulerId });
|
||||
mergedOpts.repeat = Object.assign(Object.assign({}, opts.repeat), { offset, count: currentCount, startDate: ((_a = opts.repeat) === null || _a === void 0 ? void 0 : _a.startDate)
|
||||
? new Date(opts.repeat.startDate).getTime()
|
||||
: undefined, endDate: ((_b = opts.repeat) === null || _b === void 0 ? void 0 : _b.endDate)
|
||||
? new Date(opts.repeat.endDate).getTime()
|
||||
: undefined });
|
||||
return mergedOpts;
|
||||
}
|
||||
async removeJobScheduler(jobSchedulerId) {
|
||||
return this.backend.removeJobScheduler(jobSchedulerId);
|
||||
}
|
||||
async getSchedulerData(key, next) {
|
||||
const jobData = await this.backend.getJobSchedulerData(key);
|
||||
const scheduler = this.transformSchedulerData(key, jobData, next);
|
||||
if (!scheduler) {
|
||||
await this.backend.removeJobScheduler(key);
|
||||
}
|
||||
return scheduler;
|
||||
}
|
||||
transformSchedulerData(key, jobData, next) {
|
||||
if (jobData && Object.keys(jobData).length > 0) {
|
||||
const jobSchedulerData = {
|
||||
key,
|
||||
name: jobData.name,
|
||||
next,
|
||||
};
|
||||
if (jobData.ic) {
|
||||
jobSchedulerData.iterationCount = parseInt(jobData.ic);
|
||||
}
|
||||
if (jobData.limit) {
|
||||
jobSchedulerData.limit = parseInt(jobData.limit);
|
||||
}
|
||||
if (jobData.startDate) {
|
||||
jobSchedulerData.startDate = parseInt(jobData.startDate);
|
||||
}
|
||||
if (jobData.endDate) {
|
||||
jobSchedulerData.endDate = parseInt(jobData.endDate);
|
||||
}
|
||||
if (jobData.tz) {
|
||||
jobSchedulerData.tz = jobData.tz;
|
||||
}
|
||||
if (jobData.pattern) {
|
||||
jobSchedulerData.pattern = jobData.pattern;
|
||||
}
|
||||
if (jobData.every) {
|
||||
jobSchedulerData.every = parseInt(jobData.every);
|
||||
}
|
||||
if (jobData.offset) {
|
||||
jobSchedulerData.offset = parseInt(jobData.offset);
|
||||
}
|
||||
if (jobData.data || jobData.opts) {
|
||||
jobSchedulerData.template = this.getTemplateFromJSON(jobData.data, jobData.opts);
|
||||
}
|
||||
return jobSchedulerData;
|
||||
}
|
||||
if (hasLegacyRepeatableKeyShape(key)) {
|
||||
throw getLegacyRepeatableJobError(key);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
/**
|
||||
* Checks if a given id corresponds to a registered job scheduler.
|
||||
*
|
||||
* This is used to disambiguate between new job scheduler ids (which may
|
||||
* contain any number of colon segments) and legacy repeatable job keys
|
||||
* (which always contain 5+ colon segments). Relying purely on segment
|
||||
* count is not safe because a user-provided jobSchedulerId may itself
|
||||
* contain 5+ colon segments, which would otherwise be misclassified as
|
||||
* a legacy repeatable key.
|
||||
*
|
||||
* We cannot use ZSCORE on the shared `repeat` sorted set because legacy
|
||||
* repeatable jobs are stored in the same sorted set and would be reported
|
||||
* as schedulers. Instead, we probe the per-id metadata hash (`repeat:<id>`)
|
||||
* for the `ic` (iteration count) field, which is written exclusively by
|
||||
* `storeJobScheduler` and is never set by the legacy `addRepeatableJob`
|
||||
* flow.
|
||||
*/
|
||||
async isJobScheduler(id) {
|
||||
return this.backend.isJobScheduler(id);
|
||||
}
|
||||
async getScheduler(id) {
|
||||
const [rawJobData, next] = await this.backend.getJobScheduler(id);
|
||||
return this.transformSchedulerData(id, rawJobData ? array2obj(rawJobData) : null, next ? parseInt(next) : null);
|
||||
}
|
||||
getTemplateFromJSON(rawData, rawOpts) {
|
||||
const template = {};
|
||||
if (rawData) {
|
||||
template.data = JSON.parse(rawData);
|
||||
}
|
||||
if (rawOpts) {
|
||||
template.opts = Job.optsFromJSON(rawOpts);
|
||||
}
|
||||
return template;
|
||||
}
|
||||
async getJobSchedulers(start = 0, end = -1, asc = false) {
|
||||
const result = await this.backend.getJobSchedulersRange(start, end, asc);
|
||||
const jobs = [];
|
||||
for (let i = 0; i < result.length; i += 2) {
|
||||
jobs.push(this.getSchedulerData(result[i], parseInt(result[i + 1])));
|
||||
}
|
||||
return (await Promise.all(jobs)).filter((job) => !!job);
|
||||
}
|
||||
async getSchedulersCount() {
|
||||
return this.backend.getJobSchedulersCount();
|
||||
}
|
||||
getSchedulerNextJobId({ nextMillis, jobSchedulerId, }) {
|
||||
return `repeat:${jobSchedulerId}:${nextMillis}`;
|
||||
}
|
||||
}
|
||||
export const defaultRepeatStrategy = (millis, opts) => {
|
||||
const { pattern } = opts;
|
||||
const dateFromMillis = new Date(millis);
|
||||
const startDate = opts.startDate && new Date(opts.startDate);
|
||||
const currentDate = startDate > dateFromMillis ? startDate : dateFromMillis;
|
||||
const interval = CronExpressionParser.parse(pattern, Object.assign(Object.assign({}, opts), { currentDate }));
|
||||
try {
|
||||
if (opts.immediately) {
|
||||
return new Date().getTime();
|
||||
}
|
||||
else {
|
||||
return interval.next().getTime();
|
||||
}
|
||||
}
|
||||
catch (e) {
|
||||
// Ignore error
|
||||
}
|
||||
};
|
||||
/**
|
||||
* Computes the next execution time (in ms since epoch) for the given repeat
|
||||
* options, supporting both `.every` (fixed interval) and `.pattern` (cron)
|
||||
* strategies. This is the default repeat strategy used to schedule the next
|
||||
* iteration of a job scheduler.
|
||||
*/
|
||||
export const getNextMillis = (millis, opts) => {
|
||||
const pattern = opts.pattern;
|
||||
if (pattern && opts.every) {
|
||||
throw new Error('Both .pattern and .every options are defined for this repeatable job');
|
||||
}
|
||||
if (opts.every) {
|
||||
return (Math.floor(millis / opts.every) * opts.every +
|
||||
(opts.immediately ? 0 : opts.every));
|
||||
}
|
||||
const currentDate = opts.startDate && new Date(opts.startDate) > new Date(millis)
|
||||
? new Date(opts.startDate)
|
||||
: new Date(millis);
|
||||
const interval = CronExpressionParser.parse(pattern, Object.assign(Object.assign({}, opts), { currentDate }));
|
||||
try {
|
||||
if (opts.immediately) {
|
||||
return new Date().getTime();
|
||||
}
|
||||
else {
|
||||
return interval.next().getTime();
|
||||
}
|
||||
}
|
||||
catch (e) {
|
||||
// Ignore error
|
||||
}
|
||||
};
|
||||
491
node_modules/bullmq/dist/esm/classes/job.d.ts
generated
vendored
Normal file
491
node_modules/bullmq/dist/esm/classes/job.d.ts
generated
vendored
Normal file
@@ -0,0 +1,491 @@
|
||||
import { BulkJobOptions, DependenciesOpts, IRedisTransaction, IQueueBackend, JobJson, MinimalJob, MinimalQueue, MoveToWaitingChildrenOpts, ParentKeys, ParentKeyOpts, RedisClient, RetryOptions } from '../interfaces';
|
||||
import { FinishedStatus, JobsOptions, JobState, JobJsonSandbox, JobProgress } from '../types';
|
||||
import type { QueueEvents } from './queue-events';
|
||||
export declare const PRIORITY_LIMIT: number;
|
||||
/**
|
||||
* Job
|
||||
*
|
||||
* This class represents a Job in the queue. Normally job are implicitly created when
|
||||
* you add a job to the queue with methods such as Queue.addJob( ... )
|
||||
*
|
||||
* A Job instance is also passed to the Worker's process function.
|
||||
*
|
||||
*/
|
||||
export declare class Job<DataType = any, ReturnType = any, NameType extends string = string> implements MinimalJob<DataType, ReturnType, NameType> {
|
||||
protected queue: MinimalQueue;
|
||||
/**
|
||||
* The name of the Job
|
||||
*/
|
||||
name: NameType;
|
||||
/**
|
||||
* The payload for this job.
|
||||
*/
|
||||
data: DataType;
|
||||
/**
|
||||
* The options object for this job.
|
||||
*/
|
||||
opts: JobsOptions;
|
||||
id?: string;
|
||||
/**
|
||||
* It includes the prefix, the namespace separator :, and queue name.
|
||||
* @see {@link https://www.gnu.org/software/gawk/manual/html_node/Qualified-Names.html}
|
||||
*/
|
||||
readonly queueQualifiedName: string;
|
||||
/**
|
||||
* The progress a job has performed so far.
|
||||
* @defaultValue 0
|
||||
*/
|
||||
progress: JobProgress;
|
||||
/**
|
||||
* The value returned by the processor when processing this job.
|
||||
* @defaultValue null
|
||||
*/
|
||||
returnvalue: ReturnType;
|
||||
/**
|
||||
* Stacktrace for the error (for failed jobs).
|
||||
* @defaultValue null
|
||||
*/
|
||||
stacktrace: string[] | null;
|
||||
/**
|
||||
* An amount of milliseconds to wait until this job can be processed.
|
||||
* @defaultValue 0
|
||||
*/
|
||||
delay: number;
|
||||
/**
|
||||
* Ranges from 0 to 2 097 151. `0` means no explicit priority, and jobs with
|
||||
* no explicit priority are processed before prioritized jobs. For prioritized
|
||||
* jobs, lower numbers are processed before higher numbers. Note that using
|
||||
* priorities has a slight impact on performance, so do not use it if not
|
||||
* required.
|
||||
* @defaultValue 0
|
||||
*/
|
||||
priority: number;
|
||||
/**
|
||||
* Timestamp when the job was created (unless overridden with job options).
|
||||
*/
|
||||
timestamp: number;
|
||||
/**
|
||||
* Number of attempts when job is moved to active.
|
||||
* @defaultValue 0
|
||||
*/
|
||||
attemptsStarted: number;
|
||||
/**
|
||||
* Number of attempts after the job has failed.
|
||||
* @defaultValue 0
|
||||
*/
|
||||
attemptsMade: number;
|
||||
/**
|
||||
* Number of times where job has stalled.
|
||||
* @defaultValue 0
|
||||
*/
|
||||
stalledCounter: number;
|
||||
/**
|
||||
* Reason for failing.
|
||||
*/
|
||||
failedReason: string;
|
||||
/**
|
||||
* Deferred failure. Stores a failed message and marks this job to be failed directly
|
||||
* as soon as the job is picked up by a worker, and using this string as the failed reason.
|
||||
*/
|
||||
deferredFailure: string;
|
||||
/**
|
||||
* Timestamp for when the job finished (completed or failed).
|
||||
*/
|
||||
finishedOn?: number;
|
||||
/**
|
||||
* Timestamp for when the job was processed.
|
||||
*/
|
||||
processedOn?: number;
|
||||
/**
|
||||
* Fully qualified key (including the queue prefix) pointing to the parent of this job.
|
||||
*/
|
||||
parentKey?: string;
|
||||
/**
|
||||
* Object that contains parentId (id) and parent queueKey.
|
||||
*/
|
||||
parent?: ParentKeys;
|
||||
/**
|
||||
* Deduplication identifier.
|
||||
*/
|
||||
deduplicationId?: string;
|
||||
/**
|
||||
* Base repeat job key.
|
||||
*/
|
||||
repeatJobKey?: string;
|
||||
/**
|
||||
* The token used for locking this job.
|
||||
*/
|
||||
token?: string;
|
||||
/**
|
||||
* The worker name that is processing or processed this job.
|
||||
*/
|
||||
processedBy?: string;
|
||||
protected toKey: (type: string) => string;
|
||||
protected backend: IQueueBackend;
|
||||
constructor(queue: MinimalQueue,
|
||||
/**
|
||||
* The name of the Job
|
||||
*/
|
||||
name: NameType,
|
||||
/**
|
||||
* The payload for this job.
|
||||
*/
|
||||
data: DataType,
|
||||
/**
|
||||
* The options object for this job.
|
||||
*/
|
||||
opts?: JobsOptions, id?: string);
|
||||
/**
|
||||
* Creates a new job and adds it to the queue.
|
||||
*
|
||||
* @param queue - the queue where to add the job.
|
||||
* @param name - the name of the job.
|
||||
* @param data - the payload of the job.
|
||||
* @param opts - the options bag for this job.
|
||||
* @returns The created Job instance
|
||||
*/
|
||||
static create<T = any, R = any, N extends string = string>(queue: MinimalQueue, name: N, data: T, opts?: JobsOptions): Promise<Job<T, R, N>>;
|
||||
/**
|
||||
* Creates a bulk of jobs and adds them atomically to the given queue.
|
||||
*
|
||||
* @param queue - the queue where to add the jobs.
|
||||
* @param jobs - an array of jobs to be added to the queue.
|
||||
* @returns The created Job instances
|
||||
*/
|
||||
static createBulk<T = any, R = any, N extends string = string>(queue: MinimalQueue, jobs: {
|
||||
name: N;
|
||||
data: T;
|
||||
opts?: BulkJobOptions;
|
||||
}[]): Promise<Job<T, R, N>[]>;
|
||||
/**
|
||||
* Instantiates a Job from a {@link JobJson} object (the public, decoded
|
||||
* representation of a stored job).
|
||||
*
|
||||
* @param queue - the queue where the job belongs to.
|
||||
* @param json - the plain object containing the job.
|
||||
* @param jobId - an optional job id (overrides the id coming from the JSON object)
|
||||
* @returns A Job instance reconstructed from the JSON data
|
||||
*/
|
||||
static fromJSON<T = any, R = any, N extends string = string>(queue: MinimalQueue, json: JobJson, jobId?: string): Job<T, R, N>;
|
||||
protected createBackend(): void;
|
||||
static optsFromJSON(rawOpts?: string, optsDecode?: Record<string, string>): JobsOptions;
|
||||
/**
|
||||
* Fetches a Job from the queue given the passed job id.
|
||||
*
|
||||
* @param queue - the queue where the job belongs to.
|
||||
* @param jobId - the job id.
|
||||
* @returns
|
||||
*/
|
||||
static fromId<T = any, R = any, N extends string = string>(queue: MinimalQueue, jobId: string): Promise<Job<T, R, N> | undefined>;
|
||||
/**
|
||||
* addJobLog
|
||||
*
|
||||
* @param queue - A minimal queue instance
|
||||
* @param jobId - Job id
|
||||
* @param logRow - String with a row of log data to be logged
|
||||
* @param keepLogs - The optional amount of log entries to preserve
|
||||
*
|
||||
* @returns The total number of log entries for this job so far.
|
||||
*/
|
||||
static addJobLog(queue: MinimalQueue, jobId: string, logRow: string, keepLogs?: number): Promise<number>;
|
||||
toJSON(): Omit<this, "toJSON" | "prefix" | "queueName" | "moveToWaitingChildren" | "getChildrenValues" | "getIgnoredChildrenFailures" | "getDependenciesCount" | "getDependencies" | "changeDelay" | "changePriority" | "extendLock" | "getState" | "moveToDelayed" | "promote" | "removeChildDependency" | "removeDeduplicationKey" | "removeUnprocessedChildren" | "updateData" | "updateProgress" | "addJob" | "queue" | "backend" | "asJSON" | "asJSONSandbox" | "log" | "clearLogs" | "remove" | "moveToCompleted" | "moveToWait" | "moveToFailed" | "isCompleted" | "isFailed" | "isDelayed" | "isWaitingChildren" | "isActive" | "isWaiting" | "getFailedChildrenValues" | "waitUntilFinished" | "retry" | "toFlowEntry">;
|
||||
/**
|
||||
* Prepares a job to be serialized for storage in Redis.
|
||||
* @returns
|
||||
*/
|
||||
asJSON(): JobJson;
|
||||
/**
|
||||
* Prepares a job to be passed to Sandbox.
|
||||
* @returns
|
||||
*/
|
||||
asJSONSandbox(): JobJsonSandbox;
|
||||
/**
|
||||
* Updates a job's data
|
||||
*
|
||||
* @param data - the data that will replace the current jobs data.
|
||||
*/
|
||||
updateData(data: DataType): Promise<void>;
|
||||
/**
|
||||
* Updates a job's progress
|
||||
*
|
||||
* @param progress - number or object to be saved as progress.
|
||||
*/
|
||||
updateProgress(progress: JobProgress): Promise<void>;
|
||||
/**
|
||||
* Logs one row of log data.
|
||||
*
|
||||
* @param logRow - string with log data to be logged.
|
||||
* @returns The total number of log entries for this job so far.
|
||||
*/
|
||||
log(logRow: string): Promise<number>;
|
||||
/**
|
||||
* Removes child dependency from parent when child is not yet finished
|
||||
*
|
||||
* @returns True if the relationship existed and if it was removed.
|
||||
*/
|
||||
removeChildDependency(): Promise<boolean>;
|
||||
/**
|
||||
* Clears job's logs
|
||||
*
|
||||
* @param keepLogs - the amount of log entries to preserve
|
||||
*/
|
||||
clearLogs(keepLogs?: number): Promise<void>;
|
||||
/**
|
||||
* Completely remove the job from the queue.
|
||||
* Note, this call will throw an exception if the job
|
||||
* is being processed when the call is performed.
|
||||
*
|
||||
* @param opts - Options to remove a job
|
||||
*/
|
||||
remove({ removeChildren }?: {
|
||||
removeChildren?: boolean;
|
||||
}): Promise<void>;
|
||||
/**
|
||||
* Remove all children from this job that are not yet processed,
|
||||
* in other words that are in any other state than completed, failed or active.
|
||||
*
|
||||
* @remarks
|
||||
* - Jobs with locks (most likely active) are ignored.
|
||||
* - This method can be slow if the number of children is large (\> 1000).
|
||||
*/
|
||||
removeUnprocessedChildren(): Promise<void>;
|
||||
/**
|
||||
* Extend the lock for this job.
|
||||
*
|
||||
* @param token - unique token for the lock
|
||||
* @param duration - lock duration in milliseconds
|
||||
*/
|
||||
extendLock(token: string, duration: number): Promise<number>;
|
||||
/**
|
||||
* Moves a job to the completed queue.
|
||||
* Returned job to be used with Queue.prototype.nextJobFromJobData.
|
||||
*
|
||||
* @param returnValue - The jobs success message.
|
||||
* @param token - Worker token used to acquire completed job.
|
||||
* @param fetchNext - True when wanting to fetch the next job.
|
||||
* @returns Returns the jobData of the next job in the waiting queue or void.
|
||||
*/
|
||||
moveToCompleted(returnValue: ReturnType, token: string, fetchNext?: boolean): Promise<void | any[]>;
|
||||
/**
|
||||
* Moves a job to the wait or prioritized state.
|
||||
*
|
||||
* @param token - Worker token used to acquire completed job.
|
||||
* @returns Returns pttl.
|
||||
*/
|
||||
moveToWait(token?: string): Promise<number>;
|
||||
private shouldRetryJob;
|
||||
/**
|
||||
* Moves a job to the failed queue.
|
||||
*
|
||||
* @param err - the jobs error message.
|
||||
* @param token - token to check job is locked by current worker
|
||||
* @param fetchNext - true when wanting to fetch the next job
|
||||
* @returns Returns the jobData of the next job in the waiting queue or void.
|
||||
*/
|
||||
moveToFailed<E extends Error>(err: E, token: string, fetchNext?: boolean): Promise<void | any[]>;
|
||||
private getSpanOperation;
|
||||
/**
|
||||
* Records job metrics if a meter is configured in telemetry options.
|
||||
*
|
||||
* @param state - The job state
|
||||
*/
|
||||
private recordJobMetrics;
|
||||
/**
|
||||
* @returns true if the job has completed.
|
||||
*/
|
||||
isCompleted(): Promise<boolean>;
|
||||
/**
|
||||
* @returns true if the job has failed.
|
||||
*/
|
||||
isFailed(): Promise<boolean>;
|
||||
/**
|
||||
* @returns true if the job is delayed.
|
||||
*/
|
||||
isDelayed(): Promise<boolean>;
|
||||
/**
|
||||
* @returns true if the job is waiting for children.
|
||||
*/
|
||||
isWaitingChildren(): Promise<boolean>;
|
||||
/**
|
||||
* @returns true if the job is active.
|
||||
*/
|
||||
isActive(): Promise<boolean>;
|
||||
/**
|
||||
* @returns true if the job is waiting.
|
||||
*/
|
||||
isWaiting(): Promise<boolean>;
|
||||
/**
|
||||
* @returns the queue name this job belongs to.
|
||||
*/
|
||||
get queueName(): string;
|
||||
/**
|
||||
* @returns the prefix that is used.
|
||||
*/
|
||||
get prefix(): string;
|
||||
/**
|
||||
* Get current state.
|
||||
*
|
||||
* @returns Returns one of these values:
|
||||
* 'completed', 'failed', 'delayed', 'active', 'waiting', 'waiting-children', 'unknown'.
|
||||
*/
|
||||
getState(): Promise<JobState | 'unknown'>;
|
||||
/**
|
||||
* Change delay of a delayed job.
|
||||
*
|
||||
* Reschedules a delayed job by setting a new delay from the current time.
|
||||
* For example, calling changeDelay(5000) will reschedule the job to execute
|
||||
* 5000 milliseconds (5 seconds) from now, regardless of the original delay.
|
||||
*
|
||||
* @param delay - milliseconds from now when the job should be processed.
|
||||
* @returns void
|
||||
* @throws JobNotExist
|
||||
* This exception is thrown if jobId is missing.
|
||||
* @throws JobNotInState
|
||||
* This exception is thrown if job is not in delayed state.
|
||||
*/
|
||||
changeDelay(delay: number): Promise<void>;
|
||||
/**
|
||||
* Change job priority.
|
||||
*
|
||||
* @param opts - options containing priority and lifo values.
|
||||
* @returns void
|
||||
*/
|
||||
changePriority(opts: {
|
||||
priority?: number;
|
||||
lifo?: boolean;
|
||||
}): Promise<void>;
|
||||
/**
|
||||
* Get this jobs children result values if any.
|
||||
*
|
||||
* @returns Object mapping children job keys with their values.
|
||||
*/
|
||||
getChildrenValues<CT = any>(): Promise<{
|
||||
[jobKey: string]: CT;
|
||||
}>;
|
||||
/**
|
||||
* Retrieves the failures of child jobs that were explicitly ignored while using ignoreDependencyOnFailure option.
|
||||
* This method is useful for inspecting which child jobs were intentionally ignored when an error occurred.
|
||||
* @see {@link https://docs.bullmq.io/guide/flows/ignore-dependency}
|
||||
*
|
||||
* @returns Object mapping children job keys with their failure values.
|
||||
*/
|
||||
getIgnoredChildrenFailures(): Promise<{
|
||||
[jobKey: string]: string;
|
||||
}>;
|
||||
/**
|
||||
* Get job's children failure values that were ignored if any.
|
||||
*
|
||||
* @deprecated This method is deprecated and will be removed in v6. Use getIgnoredChildrenFailures instead.
|
||||
*
|
||||
* @returns Object mapping children job keys with their failure values.
|
||||
*/
|
||||
getFailedChildrenValues(): Promise<{
|
||||
[jobKey: string]: string;
|
||||
}>;
|
||||
/**
|
||||
* Get children job keys if this job is a parent and has children.
|
||||
* @remarks
|
||||
* Count options before Redis v7.2 works as expected with any quantity of entries
|
||||
* on processed/unprocessed dependencies, since v7.2 you must consider that count
|
||||
* won't have any effect until processed/unprocessed dependencies have a length
|
||||
* greater than 127
|
||||
* @see {@link https://redis.io/docs/management/optimization/memory-optimization/#redis--72}
|
||||
* @see {@link https://docs.bullmq.io/guide/flows#getters}
|
||||
* @returns dependencies separated by processed, unprocessed, ignored and failed.
|
||||
*/
|
||||
getDependencies(opts?: DependenciesOpts): Promise<{
|
||||
nextFailedCursor?: number;
|
||||
failed?: string[];
|
||||
nextIgnoredCursor?: number;
|
||||
ignored?: Record<string, any>;
|
||||
nextProcessedCursor?: number;
|
||||
processed?: Record<string, any>;
|
||||
nextUnprocessedCursor?: number;
|
||||
unprocessed?: string[];
|
||||
}>;
|
||||
/**
|
||||
* Get children job counts if this job is a parent and has children.
|
||||
*
|
||||
* @returns dependencies count separated by processed, unprocessed, ignored and failed.
|
||||
*/
|
||||
getDependenciesCount(opts?: {
|
||||
failed?: boolean;
|
||||
ignored?: boolean;
|
||||
processed?: boolean;
|
||||
unprocessed?: boolean;
|
||||
}): Promise<{
|
||||
failed?: number;
|
||||
ignored?: number;
|
||||
processed?: number;
|
||||
unprocessed?: number;
|
||||
}>;
|
||||
/**
|
||||
* Returns a promise the resolves when the job has completed (containing the return value of the job),
|
||||
* or rejects when the job has failed (containing the failedReason).
|
||||
*
|
||||
* @param queueEvents - Instance of QueueEvents.
|
||||
* @param ttl - Time in milliseconds to wait for job to finish before timing out.
|
||||
*/
|
||||
waitUntilFinished(queueEvents: QueueEvents, ttl?: number): Promise<ReturnType>;
|
||||
/**
|
||||
* Moves the job to the delay set.
|
||||
*
|
||||
* @param timestamp - timestamp when the job should be moved back to "wait"
|
||||
* @param token - token to check job is locked by current worker
|
||||
* @returns
|
||||
*/
|
||||
moveToDelayed(timestamp: number, token?: string): Promise<void>;
|
||||
/**
|
||||
* Moves the job to the waiting-children set.
|
||||
*
|
||||
* @param token - Token to check job is locked by current worker
|
||||
* @param opts - The options bag for moving a job to waiting-children.
|
||||
* @returns true if the job was moved
|
||||
*/
|
||||
moveToWaitingChildren(token: string, opts?: MoveToWaitingChildrenOpts): Promise<boolean>;
|
||||
/**
|
||||
* Promotes a delayed job so that it starts to be processed as soon as possible.
|
||||
*/
|
||||
promote(): Promise<void>;
|
||||
/**
|
||||
* Attempts to retry the job. Only a job that has failed or completed can be retried.
|
||||
*
|
||||
* @param state - completed / failed
|
||||
* @param opts - options to retry a job
|
||||
* @returns A promise that resolves when the job has been successfully moved to the wait queue.
|
||||
* The queue emits a waiting event when the job is successfully moved.
|
||||
* @throws Will throw an error if the job does not exist, is locked, or is not in the expected state.
|
||||
*/
|
||||
retry(state?: FinishedStatus, opts?: RetryOptions): Promise<void>;
|
||||
private isInState;
|
||||
/**
|
||||
* Adds the job to Redis.
|
||||
*
|
||||
* @param client - The Redis client to use for adding the job.
|
||||
* @param parentOpts - Options for the parent-child relationship.
|
||||
* @returns The job ID
|
||||
*/
|
||||
addJob(client: RedisClient | IRedisTransaction, parentOpts?: ParentKeyOpts): Promise<string>;
|
||||
/**
|
||||
* Builds the data needed to insert this job as part of a flow, without
|
||||
* touching the datastore. Used by FlowProducer to collect a whole flow tree
|
||||
* and hand it to the backend's atomic `addFlow` operation.
|
||||
*
|
||||
* @param parentOpts - parent-link options for this node.
|
||||
*/
|
||||
toFlowEntry(parentOpts?: ParentKeyOpts): {
|
||||
jobData: JobJson;
|
||||
jobId: string;
|
||||
parentKeyOpts: ParentKeyOpts;
|
||||
prefix: string;
|
||||
queueName: string;
|
||||
};
|
||||
/**
|
||||
* Removes a deduplication key if job is still the cause of deduplication.
|
||||
* @returns true if the deduplication key was removed.
|
||||
*/
|
||||
removeDeduplicationKey(): Promise<boolean>;
|
||||
protected validateOptions(jobData: JobJson): void;
|
||||
protected updateStacktrace(err: Error): void;
|
||||
private setSpanJobAttributes;
|
||||
}
|
||||
977
node_modules/bullmq/dist/esm/classes/job.js
generated
vendored
Normal file
977
node_modules/bullmq/dist/esm/classes/job.js
generated
vendored
Normal file
@@ -0,0 +1,977 @@
|
||||
import { __rest } from "tslib";
|
||||
import { debuglog } from 'util';
|
||||
import { errorObject, getParentKey, lengthInUtf8Bytes, optsDecodeMap, parseObjectValues, tryCatch, removeUndefinedFields, } from '../utils';
|
||||
import { Backoffs } from './backoffs';
|
||||
import { UnrecoverableError } from './errors/unrecoverable-error';
|
||||
import { SpanKind, TelemetryAttributes, MetricNames } from '../enums';
|
||||
const logger = debuglog('bull');
|
||||
export const PRIORITY_LIMIT = 2 ** 21 - 1;
|
||||
/**
|
||||
* Job
|
||||
*
|
||||
* This class represents a Job in the queue. Normally job are implicitly created when
|
||||
* you add a job to the queue with methods such as Queue.addJob( ... )
|
||||
*
|
||||
* A Job instance is also passed to the Worker's process function.
|
||||
*
|
||||
*/
|
||||
export class Job {
|
||||
constructor(queue,
|
||||
/**
|
||||
* The name of the Job
|
||||
*/
|
||||
name,
|
||||
/**
|
||||
* The payload for this job.
|
||||
*/
|
||||
data,
|
||||
/**
|
||||
* The options object for this job.
|
||||
*/
|
||||
opts = {}, id) {
|
||||
var _a;
|
||||
this.queue = queue;
|
||||
this.name = name;
|
||||
this.data = data;
|
||||
this.opts = opts;
|
||||
this.id = id;
|
||||
/**
|
||||
* The progress a job has performed so far.
|
||||
* @defaultValue 0
|
||||
*/
|
||||
this.progress = 0;
|
||||
/**
|
||||
* The value returned by the processor when processing this job.
|
||||
* @defaultValue null
|
||||
*/
|
||||
this.returnvalue = null;
|
||||
/**
|
||||
* Stacktrace for the error (for failed jobs).
|
||||
* @defaultValue null
|
||||
*/
|
||||
this.stacktrace = null;
|
||||
/**
|
||||
* An amount of milliseconds to wait until this job can be processed.
|
||||
* @defaultValue 0
|
||||
*/
|
||||
this.delay = 0;
|
||||
/**
|
||||
* Ranges from 0 to 2 097 151. `0` means no explicit priority, and jobs with
|
||||
* no explicit priority are processed before prioritized jobs. For prioritized
|
||||
* jobs, lower numbers are processed before higher numbers. Note that using
|
||||
* priorities has a slight impact on performance, so do not use it if not
|
||||
* required.
|
||||
* @defaultValue 0
|
||||
*/
|
||||
this.priority = 0;
|
||||
/**
|
||||
* Number of attempts when job is moved to active.
|
||||
* @defaultValue 0
|
||||
*/
|
||||
this.attemptsStarted = 0;
|
||||
/**
|
||||
* Number of attempts after the job has failed.
|
||||
* @defaultValue 0
|
||||
*/
|
||||
this.attemptsMade = 0;
|
||||
/**
|
||||
* Number of times where job has stalled.
|
||||
* @defaultValue 0
|
||||
*/
|
||||
this.stalledCounter = 0;
|
||||
const _b = this.opts, { repeatJobKey } = _b, restOpts = __rest(_b, ["repeatJobKey"]);
|
||||
this.opts = Object.assign({
|
||||
attempts: 0,
|
||||
}, restOpts);
|
||||
this.delay = this.opts.delay;
|
||||
this.priority = this.opts.priority || 0;
|
||||
this.repeatJobKey = repeatJobKey;
|
||||
this.timestamp = opts.timestamp ? opts.timestamp : Date.now();
|
||||
this.opts.backoff = Backoffs.normalize(opts.backoff);
|
||||
this.parentKey = getParentKey(opts.parent);
|
||||
if (opts.parent) {
|
||||
this.parent = { id: opts.parent.id, queueKey: opts.parent.queue };
|
||||
if (opts.failParentOnFailure) {
|
||||
this.parent.fpof = true;
|
||||
}
|
||||
if (opts.removeDependencyOnFailure) {
|
||||
this.parent.rdof = true;
|
||||
}
|
||||
if (opts.ignoreDependencyOnFailure) {
|
||||
this.parent.idof = true;
|
||||
}
|
||||
if (opts.continueParentOnFailure) {
|
||||
this.parent.cpof = true;
|
||||
}
|
||||
}
|
||||
this.deduplicationId = (_a = this.opts.deduplication) === null || _a === void 0 ? void 0 : _a.id;
|
||||
this.toKey = queue.toKey.bind(queue);
|
||||
this.createBackend();
|
||||
this.queueQualifiedName = queue.qualifiedName;
|
||||
}
|
||||
/**
|
||||
* Creates a new job and adds it to the queue.
|
||||
*
|
||||
* @param queue - the queue where to add the job.
|
||||
* @param name - the name of the job.
|
||||
* @param data - the payload of the job.
|
||||
* @param opts - the options bag for this job.
|
||||
* @returns The created Job instance
|
||||
*/
|
||||
static async create(queue, name, data, opts) {
|
||||
const job = new this(queue, name, data, opts, opts && opts.jobId);
|
||||
const jobData = job.asJSON();
|
||||
job.validateOptions(jobData);
|
||||
job.id = await job.backend.addJob(jobData, job.id, {
|
||||
parentKey: job.parentKey,
|
||||
parentDependenciesKey: job.parentKey
|
||||
? `${job.parentKey}:dependencies`
|
||||
: '',
|
||||
});
|
||||
return job;
|
||||
}
|
||||
/**
|
||||
* Creates a bulk of jobs and adds them atomically to the given queue.
|
||||
*
|
||||
* @param queue - the queue where to add the jobs.
|
||||
* @param jobs - an array of jobs to be added to the queue.
|
||||
* @returns The created Job instances
|
||||
*/
|
||||
static async createBulk(queue, jobs) {
|
||||
const jobInstances = jobs.map(job => { var _a; return new this(queue, job.name, job.data, job.opts, (_a = job.opts) === null || _a === void 0 ? void 0 : _a.jobId); });
|
||||
const scripts = queue.backend;
|
||||
const entries = jobInstances.map(job => {
|
||||
const jobData = job.asJSON();
|
||||
job.validateOptions(jobData);
|
||||
return {
|
||||
job: jobData,
|
||||
jobId: job.id,
|
||||
parentKeyOpts: {
|
||||
parentKey: job.parentKey,
|
||||
parentDependenciesKey: job.parentKey
|
||||
? `${job.parentKey}:dependencies`
|
||||
: '',
|
||||
},
|
||||
};
|
||||
});
|
||||
const ids = await scripts.addJobs(entries);
|
||||
jobInstances.forEach((job, index) => {
|
||||
job.id = ids[index];
|
||||
});
|
||||
return jobInstances;
|
||||
}
|
||||
/**
|
||||
* Instantiates a Job from a {@link JobJson} object (the public, decoded
|
||||
* representation of a stored job).
|
||||
*
|
||||
* @param queue - the queue where the job belongs to.
|
||||
* @param json - the plain object containing the job.
|
||||
* @param jobId - an optional job id (overrides the id coming from the JSON object)
|
||||
* @returns A Job instance reconstructed from the JSON data
|
||||
*/
|
||||
static fromJSON(queue, json, jobId) {
|
||||
var _a, _b, _c, _d;
|
||||
const data = JSON.parse(json.data || '{}');
|
||||
const job = new this(queue, json.name, data, json.opts, json.id || jobId);
|
||||
job.progress = (_a = json.progress) !== null && _a !== void 0 ? _a : 0;
|
||||
job.delay = json.delay;
|
||||
job.priority = json.priority;
|
||||
job.timestamp = json.timestamp;
|
||||
if (json.finishedOn) {
|
||||
job.finishedOn = json.finishedOn;
|
||||
}
|
||||
if (json.processedOn) {
|
||||
job.processedOn = json.processedOn;
|
||||
}
|
||||
if (json.repeatJobKey) {
|
||||
job.repeatJobKey = json.repeatJobKey;
|
||||
}
|
||||
if (json.deduplicationId) {
|
||||
job.deduplicationId = json.deduplicationId;
|
||||
}
|
||||
if (json.failedReason) {
|
||||
job.failedReason = json.failedReason;
|
||||
}
|
||||
job.attemptsStarted = (_b = json.attemptsStarted) !== null && _b !== void 0 ? _b : 0;
|
||||
job.attemptsMade = (_c = json.attemptsMade) !== null && _c !== void 0 ? _c : 0;
|
||||
job.stalledCounter = (_d = json.stalledCounter) !== null && _d !== void 0 ? _d : 0;
|
||||
if (json.deferredFailure) {
|
||||
job.deferredFailure = json.deferredFailure;
|
||||
}
|
||||
job.stacktrace = getTraces(json.stacktrace);
|
||||
if (typeof json.returnvalue === 'string') {
|
||||
job.returnvalue = getReturnValue(json.returnvalue);
|
||||
}
|
||||
if (json.parentKey) {
|
||||
job.parentKey = json.parentKey;
|
||||
}
|
||||
else {
|
||||
job.parentKey = undefined;
|
||||
}
|
||||
if (json.parent) {
|
||||
job.parent = json.parent;
|
||||
}
|
||||
else {
|
||||
job.parent = undefined;
|
||||
}
|
||||
if (json.processedBy) {
|
||||
job.processedBy = json.processedBy;
|
||||
}
|
||||
return job;
|
||||
}
|
||||
createBackend() {
|
||||
this.backend = this.queue.backend;
|
||||
}
|
||||
static optsFromJSON(rawOpts, optsDecode = optsDecodeMap) {
|
||||
const opts = JSON.parse(rawOpts || '{}');
|
||||
const optionEntries = Object.entries(opts);
|
||||
const options = {};
|
||||
for (const item of optionEntries) {
|
||||
const [attributeName, value] = item;
|
||||
if (optsDecode[attributeName]) {
|
||||
options[optsDecode[attributeName]] =
|
||||
value;
|
||||
}
|
||||
else {
|
||||
if (attributeName === 'tm') {
|
||||
options.telemetry = Object.assign(Object.assign({}, options.telemetry), { metadata: value });
|
||||
}
|
||||
else if (attributeName === 'omc') {
|
||||
options.telemetry = Object.assign(Object.assign({}, options.telemetry), { omitContext: value });
|
||||
}
|
||||
else {
|
||||
options[attributeName] = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
return options;
|
||||
}
|
||||
/**
|
||||
* Fetches a Job from the queue given the passed job id.
|
||||
*
|
||||
* @param queue - the queue where the job belongs to.
|
||||
* @param jobId - the job id.
|
||||
* @returns
|
||||
*/
|
||||
static async fromId(queue, jobId) {
|
||||
// jobId can be undefined if moveJob returns undefined
|
||||
if (jobId) {
|
||||
const scripts = queue.backend;
|
||||
const jobData = await scripts.getJobData(jobId);
|
||||
return jobData
|
||||
? this.fromJSON(queue, jobData, jobId)
|
||||
: undefined;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* addJobLog
|
||||
*
|
||||
* @param queue - A minimal queue instance
|
||||
* @param jobId - Job id
|
||||
* @param logRow - String with a row of log data to be logged
|
||||
* @param keepLogs - The optional amount of log entries to preserve
|
||||
*
|
||||
* @returns The total number of log entries for this job so far.
|
||||
*/
|
||||
static addJobLog(queue, jobId, logRow, keepLogs) {
|
||||
const scripts = queue.backend;
|
||||
return scripts.addLog(jobId, logRow, keepLogs);
|
||||
}
|
||||
toJSON() {
|
||||
const _a = this, { queue, backend: scripts } = _a, withoutQueueAndScripts = __rest(_a, ["queue", "backend"]);
|
||||
return withoutQueueAndScripts;
|
||||
}
|
||||
/**
|
||||
* Prepares a job to be serialized for storage in Redis.
|
||||
* @returns
|
||||
*/
|
||||
asJSON() {
|
||||
return removeUndefinedFields({
|
||||
id: this.id,
|
||||
name: this.name,
|
||||
data: JSON.stringify(typeof this.data === 'undefined' ? {} : this.data),
|
||||
opts: this.opts,
|
||||
parent: this.parent ? Object.assign({}, this.parent) : undefined,
|
||||
parentKey: this.parentKey,
|
||||
progress: this.progress,
|
||||
attemptsMade: this.attemptsMade,
|
||||
attemptsStarted: this.attemptsStarted,
|
||||
stalledCounter: this.stalledCounter,
|
||||
finishedOn: this.finishedOn,
|
||||
processedOn: this.processedOn,
|
||||
timestamp: this.timestamp,
|
||||
failedReason: JSON.stringify(this.failedReason),
|
||||
stacktrace: JSON.stringify(this.stacktrace),
|
||||
deduplicationId: this.deduplicationId,
|
||||
repeatJobKey: this.repeatJobKey,
|
||||
returnvalue: JSON.stringify(this.returnvalue),
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Prepares a job to be passed to Sandbox.
|
||||
* @returns
|
||||
*/
|
||||
asJSONSandbox() {
|
||||
return Object.assign(Object.assign({}, this.asJSON()), { queueName: this.queueName, queueQualifiedName: this.queueQualifiedName, prefix: this.prefix });
|
||||
}
|
||||
/**
|
||||
* Updates a job's data
|
||||
*
|
||||
* @param data - the data that will replace the current jobs data.
|
||||
*/
|
||||
updateData(data) {
|
||||
this.data = data;
|
||||
return this.backend.updateData(this, data);
|
||||
}
|
||||
/**
|
||||
* Updates a job's progress
|
||||
*
|
||||
* @param progress - number or object to be saved as progress.
|
||||
*/
|
||||
async updateProgress(progress) {
|
||||
this.progress = progress;
|
||||
await this.backend.updateProgress(this.id, progress);
|
||||
this.queue.emit('progress', this, progress);
|
||||
}
|
||||
/**
|
||||
* Logs one row of log data.
|
||||
*
|
||||
* @param logRow - string with log data to be logged.
|
||||
* @returns The total number of log entries for this job so far.
|
||||
*/
|
||||
async log(logRow) {
|
||||
return Job.addJobLog(this.queue, this.id, logRow, this.opts.keepLogs);
|
||||
}
|
||||
/**
|
||||
* Removes child dependency from parent when child is not yet finished
|
||||
*
|
||||
* @returns True if the relationship existed and if it was removed.
|
||||
*/
|
||||
async removeChildDependency() {
|
||||
const childDependencyIsRemoved = await this.backend.removeChildDependency(this.id, this.parentKey);
|
||||
if (childDependencyIsRemoved) {
|
||||
this.parent = undefined;
|
||||
this.parentKey = undefined;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
/**
|
||||
* Clears job's logs
|
||||
*
|
||||
* @param keepLogs - the amount of log entries to preserve
|
||||
*/
|
||||
async clearLogs(keepLogs) {
|
||||
await this.backend.clearLogs(this.id, keepLogs);
|
||||
}
|
||||
/**
|
||||
* Completely remove the job from the queue.
|
||||
* Note, this call will throw an exception if the job
|
||||
* is being processed when the call is performed.
|
||||
*
|
||||
* @param opts - Options to remove a job
|
||||
*/
|
||||
async remove({ removeChildren = true } = {}) {
|
||||
await this.queue.waitUntilReady();
|
||||
const queue = this.queue;
|
||||
const job = this;
|
||||
const removed = await this.backend.remove(job.id, removeChildren);
|
||||
if (removed) {
|
||||
queue.emit('removed', job);
|
||||
}
|
||||
else {
|
||||
throw new Error(`Job ${this.id} could not be removed because it is locked by another worker`);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Remove all children from this job that are not yet processed,
|
||||
* in other words that are in any other state than completed, failed or active.
|
||||
*
|
||||
* @remarks
|
||||
* - Jobs with locks (most likely active) are ignored.
|
||||
* - This method can be slow if the number of children is large (\> 1000).
|
||||
*/
|
||||
async removeUnprocessedChildren() {
|
||||
const jobId = this.id;
|
||||
await this.backend.removeUnprocessedChildren(jobId);
|
||||
}
|
||||
/**
|
||||
* Extend the lock for this job.
|
||||
*
|
||||
* @param token - unique token for the lock
|
||||
* @param duration - lock duration in milliseconds
|
||||
*/
|
||||
extendLock(token, duration) {
|
||||
return this.backend.extendLock(this.id, token, duration);
|
||||
}
|
||||
/**
|
||||
* Moves a job to the completed queue.
|
||||
* Returned job to be used with Queue.prototype.nextJobFromJobData.
|
||||
*
|
||||
* @param returnValue - The jobs success message.
|
||||
* @param token - Worker token used to acquire completed job.
|
||||
* @param fetchNext - True when wanting to fetch the next job.
|
||||
* @returns Returns the jobData of the next job in the waiting queue or void.
|
||||
*/
|
||||
async moveToCompleted(returnValue, token, fetchNext = true) {
|
||||
return this.queue.trace(SpanKind.INTERNAL, 'complete', this.queue.name, async (span) => {
|
||||
this.setSpanJobAttributes(span);
|
||||
await this.queue.waitUntilReady();
|
||||
this.returnvalue = returnValue || void 0;
|
||||
const stringifiedReturnValue = tryCatch(JSON.stringify, JSON, [
|
||||
returnValue,
|
||||
]);
|
||||
if (stringifiedReturnValue === errorObject) {
|
||||
throw errorObject.value;
|
||||
}
|
||||
const { result, finishedOn } = await this.backend.moveToCompleted(this, returnValue, this.opts.removeOnComplete, token, fetchNext);
|
||||
this.finishedOn = finishedOn;
|
||||
this.attemptsMade += 1;
|
||||
this.recordJobMetrics('completed');
|
||||
return result;
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Moves a job to the wait or prioritized state.
|
||||
*
|
||||
* @param token - Worker token used to acquire completed job.
|
||||
* @returns Returns pttl.
|
||||
*/
|
||||
async moveToWait(token) {
|
||||
const result = await this.backend.moveJobFromActiveToWait(this.id, token);
|
||||
this.recordJobMetrics('waiting');
|
||||
return result;
|
||||
}
|
||||
async shouldRetryJob(err) {
|
||||
if (this.attemptsMade + 1 < this.opts.attempts &&
|
||||
!(err instanceof UnrecoverableError || err.name == 'UnrecoverableError')) {
|
||||
const opts = this.queue.opts;
|
||||
const delay = await Backoffs.calculate(this.opts.backoff, this.attemptsMade + 1, err, this, opts.settings && opts.settings.backoffStrategy);
|
||||
return [delay == -1 ? false : true, delay == -1 ? 0 : delay];
|
||||
}
|
||||
else {
|
||||
return [false, 0];
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Moves a job to the failed queue.
|
||||
*
|
||||
* @param err - the jobs error message.
|
||||
* @param token - token to check job is locked by current worker
|
||||
* @param fetchNext - true when wanting to fetch the next job
|
||||
* @returns Returns the jobData of the next job in the waiting queue or void.
|
||||
*/
|
||||
async moveToFailed(err, token, fetchNext = false) {
|
||||
this.failedReason = err === null || err === void 0 ? void 0 : err.message;
|
||||
// Check if an automatic retry should be performed
|
||||
const [shouldRetry, retryDelay] = await this.shouldRetryJob(err);
|
||||
return this.queue.trace(SpanKind.INTERNAL, this.getSpanOperation(shouldRetry, retryDelay), this.queue.name, async (span, dstPropagationMetadata) => {
|
||||
var _a, _b;
|
||||
this.setSpanJobAttributes(span);
|
||||
let tm;
|
||||
if (!((_b = (_a = this.opts) === null || _a === void 0 ? void 0 : _a.telemetry) === null || _b === void 0 ? void 0 : _b.omitContext) && dstPropagationMetadata) {
|
||||
tm = dstPropagationMetadata;
|
||||
}
|
||||
let result;
|
||||
this.updateStacktrace(err);
|
||||
const fieldsToUpdate = {
|
||||
failedReason: this.failedReason,
|
||||
stacktrace: JSON.stringify(this.stacktrace),
|
||||
tm,
|
||||
};
|
||||
let finishedOn;
|
||||
if (shouldRetry) {
|
||||
if (retryDelay) {
|
||||
// Retry with delay
|
||||
result = await this.backend.moveToDelayed(this.id, Date.now(), retryDelay, token, { fieldsToUpdate, fetchNext });
|
||||
this.recordJobMetrics('delayed');
|
||||
}
|
||||
else {
|
||||
// Retry immediately
|
||||
result = await this.backend.retryJob(this.id, this.opts.lifo, token, {
|
||||
fieldsToUpdate,
|
||||
});
|
||||
this.recordJobMetrics('retried');
|
||||
}
|
||||
}
|
||||
else {
|
||||
const moved = await this.backend.moveToFailed(this, this.failedReason, this.opts.removeOnFail, token, fetchNext, fieldsToUpdate);
|
||||
result = moved.result;
|
||||
finishedOn = moved.finishedOn;
|
||||
// Only record failed metrics when job is not retrying
|
||||
this.recordJobMetrics('failed');
|
||||
}
|
||||
if (finishedOn && typeof finishedOn === 'number') {
|
||||
this.finishedOn = finishedOn;
|
||||
}
|
||||
if (retryDelay && typeof retryDelay === 'number') {
|
||||
this.delay = retryDelay;
|
||||
}
|
||||
this.attemptsMade += 1;
|
||||
return result;
|
||||
});
|
||||
}
|
||||
getSpanOperation(shouldRetry, retryDelay) {
|
||||
if (shouldRetry) {
|
||||
if (retryDelay) {
|
||||
return 'delay';
|
||||
}
|
||||
return 'retry';
|
||||
}
|
||||
return 'fail';
|
||||
}
|
||||
/**
|
||||
* Records job metrics if a meter is configured in telemetry options.
|
||||
*
|
||||
* @param state - The job state
|
||||
*/
|
||||
recordJobMetrics(state) {
|
||||
var _a, _b;
|
||||
const meter = (_b = (_a = this.queue.opts) === null || _a === void 0 ? void 0 : _a.telemetry) === null || _b === void 0 ? void 0 : _b.meter;
|
||||
if (!meter) {
|
||||
return;
|
||||
}
|
||||
const attributes = {
|
||||
[TelemetryAttributes.QueueName]: this.queue.name,
|
||||
[TelemetryAttributes.JobName]: this.name,
|
||||
[TelemetryAttributes.JobState]: state,
|
||||
};
|
||||
// Record counter metric based on state
|
||||
const stateToCounterName = {
|
||||
completed: MetricNames.JobsCompleted,
|
||||
failed: MetricNames.JobsFailed,
|
||||
delayed: MetricNames.JobsDelayed,
|
||||
retried: MetricNames.JobsRetried,
|
||||
waiting: MetricNames.JobsWaiting,
|
||||
'waiting-children': MetricNames.JobsWaitingChildren,
|
||||
};
|
||||
const counterName = stateToCounterName[state];
|
||||
const counter = meter.createCounter(counterName, {
|
||||
description: `Number of jobs ${state}`,
|
||||
unit: '1',
|
||||
});
|
||||
counter.add(1, attributes);
|
||||
// Record duration histogram if processedOn is available
|
||||
if (this.processedOn) {
|
||||
const duration = Date.now() - this.processedOn;
|
||||
const histogram = meter.createHistogram(MetricNames.JobDuration, {
|
||||
description: 'Job processing duration',
|
||||
unit: 'ms',
|
||||
});
|
||||
histogram.record(duration, attributes);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* @returns true if the job has completed.
|
||||
*/
|
||||
isCompleted() {
|
||||
return this.isInState('completed');
|
||||
}
|
||||
/**
|
||||
* @returns true if the job has failed.
|
||||
*/
|
||||
isFailed() {
|
||||
return this.isInState('failed');
|
||||
}
|
||||
/**
|
||||
* @returns true if the job is delayed.
|
||||
*/
|
||||
isDelayed() {
|
||||
return this.isInState('delayed');
|
||||
}
|
||||
/**
|
||||
* @returns true if the job is waiting for children.
|
||||
*/
|
||||
isWaitingChildren() {
|
||||
return this.isInState('waiting-children');
|
||||
}
|
||||
/**
|
||||
* @returns true if the job is active.
|
||||
*/
|
||||
isActive() {
|
||||
return this.isInState('active');
|
||||
}
|
||||
/**
|
||||
* @returns true if the job is waiting.
|
||||
*/
|
||||
async isWaiting() {
|
||||
return this.isInState('waiting');
|
||||
}
|
||||
/**
|
||||
* @returns the queue name this job belongs to.
|
||||
*/
|
||||
get queueName() {
|
||||
return this.queue.name;
|
||||
}
|
||||
/**
|
||||
* @returns the prefix that is used.
|
||||
*/
|
||||
get prefix() {
|
||||
// The key `prefix` is a Redis concept; derive it from the qualified name
|
||||
// (`"<prefix>:<queue>"`) rather than from queue options. Backends without a
|
||||
// prefix (where the qualified name is just the queue name) yield `''`.
|
||||
const qualified = this.queueQualifiedName;
|
||||
const name = this.queueName;
|
||||
return qualified.length > name.length + 1
|
||||
? qualified.slice(0, qualified.length - name.length - 1)
|
||||
: '';
|
||||
}
|
||||
/**
|
||||
* Get current state.
|
||||
*
|
||||
* @returns Returns one of these values:
|
||||
* 'completed', 'failed', 'delayed', 'active', 'waiting', 'waiting-children', 'unknown'.
|
||||
*/
|
||||
getState() {
|
||||
return this.backend.getState(this.id);
|
||||
}
|
||||
/**
|
||||
* Change delay of a delayed job.
|
||||
*
|
||||
* Reschedules a delayed job by setting a new delay from the current time.
|
||||
* For example, calling changeDelay(5000) will reschedule the job to execute
|
||||
* 5000 milliseconds (5 seconds) from now, regardless of the original delay.
|
||||
*
|
||||
* @param delay - milliseconds from now when the job should be processed.
|
||||
* @returns void
|
||||
* @throws JobNotExist
|
||||
* This exception is thrown if jobId is missing.
|
||||
* @throws JobNotInState
|
||||
* This exception is thrown if job is not in delayed state.
|
||||
*/
|
||||
async changeDelay(delay) {
|
||||
await this.backend.changeDelay(this.id, delay);
|
||||
this.delay = delay;
|
||||
}
|
||||
/**
|
||||
* Change job priority.
|
||||
*
|
||||
* @param opts - options containing priority and lifo values.
|
||||
* @returns void
|
||||
*/
|
||||
async changePriority(opts) {
|
||||
await this.backend.changePriority(this.id, opts.priority, opts.lifo);
|
||||
this.priority = opts.priority || 0;
|
||||
}
|
||||
/**
|
||||
* Get this jobs children result values if any.
|
||||
*
|
||||
* @returns Object mapping children job keys with their values.
|
||||
*/
|
||||
async getChildrenValues() {
|
||||
const result = await this.backend.getProcessedChildrenValues(this.id);
|
||||
if (result) {
|
||||
return parseObjectValues(result);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Retrieves the failures of child jobs that were explicitly ignored while using ignoreDependencyOnFailure option.
|
||||
* This method is useful for inspecting which child jobs were intentionally ignored when an error occurred.
|
||||
* @see {@link https://docs.bullmq.io/guide/flows/ignore-dependency}
|
||||
*
|
||||
* @returns Object mapping children job keys with their failure values.
|
||||
*/
|
||||
async getIgnoredChildrenFailures() {
|
||||
return this.backend.getIgnoredChildrenFailures(this.id);
|
||||
}
|
||||
/**
|
||||
* Get job's children failure values that were ignored if any.
|
||||
*
|
||||
* @deprecated This method is deprecated and will be removed in v6. Use getIgnoredChildrenFailures instead.
|
||||
*
|
||||
* @returns Object mapping children job keys with their failure values.
|
||||
*/
|
||||
async getFailedChildrenValues() {
|
||||
return this.backend.getIgnoredChildrenFailures(this.id);
|
||||
}
|
||||
/**
|
||||
* Get children job keys if this job is a parent and has children.
|
||||
* @remarks
|
||||
* Count options before Redis v7.2 works as expected with any quantity of entries
|
||||
* on processed/unprocessed dependencies, since v7.2 you must consider that count
|
||||
* won't have any effect until processed/unprocessed dependencies have a length
|
||||
* greater than 127
|
||||
* @see {@link https://redis.io/docs/management/optimization/memory-optimization/#redis--72}
|
||||
* @see {@link https://docs.bullmq.io/guide/flows#getters}
|
||||
* @returns dependencies separated by processed, unprocessed, ignored and failed.
|
||||
*/
|
||||
async getDependencies(opts = {}) {
|
||||
return this.backend.getDependencies(this.id, opts);
|
||||
}
|
||||
/**
|
||||
* Get children job counts if this job is a parent and has children.
|
||||
*
|
||||
* @returns dependencies count separated by processed, unprocessed, ignored and failed.
|
||||
*/
|
||||
async getDependenciesCount(opts = {}) {
|
||||
const types = [];
|
||||
Object.entries(opts).forEach(([key, value]) => {
|
||||
if (value) {
|
||||
types.push(key);
|
||||
}
|
||||
});
|
||||
const finalTypes = types.length
|
||||
? types
|
||||
: ['processed', 'unprocessed', 'ignored', 'failed'];
|
||||
const responses = await this.backend.getDependencyCounts(this.id, finalTypes);
|
||||
const counts = {};
|
||||
responses.forEach((res, index) => {
|
||||
counts[`${finalTypes[index]}`] = res || 0;
|
||||
});
|
||||
return counts;
|
||||
}
|
||||
/**
|
||||
* Returns a promise the resolves when the job has completed (containing the return value of the job),
|
||||
* or rejects when the job has failed (containing the failedReason).
|
||||
*
|
||||
* @param queueEvents - Instance of QueueEvents.
|
||||
* @param ttl - Time in milliseconds to wait for job to finish before timing out.
|
||||
*/
|
||||
async waitUntilFinished(queueEvents, ttl) {
|
||||
await this.queue.waitUntilReady();
|
||||
const jobId = this.id;
|
||||
return new Promise(async (resolve, reject) => {
|
||||
let timeout;
|
||||
if (ttl) {
|
||||
timeout = setTimeout(() => onFailed(
|
||||
/* eslint-disable max-len */
|
||||
`Job wait ${this.name} timed out before finishing, no finish notification arrived after ${ttl}ms (id=${jobId})`), ttl);
|
||||
}
|
||||
function onCompleted(args) {
|
||||
removeListeners();
|
||||
resolve(args.returnvalue);
|
||||
}
|
||||
function onFailed(args) {
|
||||
removeListeners();
|
||||
reject(new Error(args.failedReason || args));
|
||||
}
|
||||
const completedEvent = `completed:${jobId}`;
|
||||
const failedEvent = `failed:${jobId}`;
|
||||
queueEvents.on(completedEvent, onCompleted);
|
||||
queueEvents.on(failedEvent, onFailed);
|
||||
this.queue.on('closing', onFailed);
|
||||
const removeListeners = () => {
|
||||
clearInterval(timeout);
|
||||
queueEvents.removeListener(completedEvent, onCompleted);
|
||||
queueEvents.removeListener(failedEvent, onFailed);
|
||||
this.queue.removeListener('closing', onFailed);
|
||||
};
|
||||
// Poll once right now to see if the job has already finished. The job may have been completed before we were able
|
||||
// to register the event handlers on the QueueEvents, so we check here to make sure we're not waiting for an event
|
||||
// that has already happened. We block checking the job until the queue events object is actually listening to
|
||||
// Redis so there's no chance that it will miss events.
|
||||
await queueEvents.waitUntilReady();
|
||||
const [status, result] = (await this.backend.isFinished(jobId, true));
|
||||
const finished = status != 0;
|
||||
if (finished) {
|
||||
if (status == -1 || status == 2) {
|
||||
onFailed({ failedReason: result });
|
||||
}
|
||||
else {
|
||||
onCompleted({ returnvalue: getReturnValue(result) });
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Moves the job to the delay set.
|
||||
*
|
||||
* @param timestamp - timestamp when the job should be moved back to "wait"
|
||||
* @param token - token to check job is locked by current worker
|
||||
* @returns
|
||||
*/
|
||||
async moveToDelayed(timestamp, token) {
|
||||
const now = Date.now();
|
||||
const delay = timestamp - now;
|
||||
const finalDelay = delay > 0 ? delay : 0;
|
||||
await this.backend.moveToDelayed(this.id, now, finalDelay, token, {
|
||||
skipAttempt: true,
|
||||
});
|
||||
this.delay = finalDelay;
|
||||
this.recordJobMetrics('delayed');
|
||||
}
|
||||
/**
|
||||
* Moves the job to the waiting-children set.
|
||||
*
|
||||
* @param token - Token to check job is locked by current worker
|
||||
* @param opts - The options bag for moving a job to waiting-children.
|
||||
* @returns true if the job was moved
|
||||
*/
|
||||
async moveToWaitingChildren(token, opts = {}) {
|
||||
const movedToWaitingChildren = await this.backend.moveToWaitingChildren(this.id, token, opts);
|
||||
if (movedToWaitingChildren) {
|
||||
this.recordJobMetrics('waiting-children');
|
||||
}
|
||||
return movedToWaitingChildren;
|
||||
}
|
||||
/**
|
||||
* Promotes a delayed job so that it starts to be processed as soon as possible.
|
||||
*/
|
||||
async promote() {
|
||||
const jobId = this.id;
|
||||
await this.backend.promote(jobId);
|
||||
this.delay = 0;
|
||||
}
|
||||
/**
|
||||
* Attempts to retry the job. Only a job that has failed or completed can be retried.
|
||||
*
|
||||
* @param state - completed / failed
|
||||
* @param opts - options to retry a job
|
||||
* @returns A promise that resolves when the job has been successfully moved to the wait queue.
|
||||
* The queue emits a waiting event when the job is successfully moved.
|
||||
* @throws Will throw an error if the job does not exist, is locked, or is not in the expected state.
|
||||
*/
|
||||
async retry(state = 'failed', opts = {}) {
|
||||
await this.backend.retryFinishedJob(this, state, opts);
|
||||
this.failedReason = null;
|
||||
this.finishedOn = null;
|
||||
this.processedOn = null;
|
||||
this.returnvalue = null;
|
||||
if (opts.resetAttemptsMade) {
|
||||
this.attemptsMade = 0;
|
||||
}
|
||||
if (opts.resetAttemptsStarted) {
|
||||
this.attemptsStarted = 0;
|
||||
}
|
||||
}
|
||||
async isInState(state) {
|
||||
return this.backend.isJobInState(state, this.id);
|
||||
}
|
||||
/**
|
||||
* Adds the job to Redis.
|
||||
*
|
||||
* @param client - The Redis client to use for adding the job.
|
||||
* @param parentOpts - Options for the parent-child relationship.
|
||||
* @returns The job ID
|
||||
*/
|
||||
addJob(client, parentOpts) {
|
||||
const jobData = this.asJSON();
|
||||
this.validateOptions(jobData);
|
||||
// `addJobToTransaction` operates on a raw Redis client/transaction and is
|
||||
// therefore specific to the Redis backend implementation.
|
||||
return this.backend.addJobToTransaction(client, jobData, this.id, parentOpts);
|
||||
}
|
||||
/**
|
||||
* Builds the data needed to insert this job as part of a flow, without
|
||||
* touching the datastore. Used by FlowProducer to collect a whole flow tree
|
||||
* and hand it to the backend's atomic `addFlow` operation.
|
||||
*
|
||||
* @param parentOpts - parent-link options for this node.
|
||||
*/
|
||||
toFlowEntry(parentOpts = {}) {
|
||||
const jobData = this.asJSON();
|
||||
this.validateOptions(jobData);
|
||||
return {
|
||||
jobData,
|
||||
jobId: this.id,
|
||||
parentKeyOpts: parentOpts,
|
||||
prefix: this.prefix,
|
||||
queueName: this.queueName,
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Removes a deduplication key if job is still the cause of deduplication.
|
||||
* @returns true if the deduplication key was removed.
|
||||
*/
|
||||
async removeDeduplicationKey() {
|
||||
if (this.deduplicationId) {
|
||||
const result = await this.backend.removeDeduplicationKey(this.deduplicationId, this.id);
|
||||
return result > 0;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
validateOptions(jobData) {
|
||||
var _a, _b, _c, _d, _e, _f;
|
||||
const exclusiveOptions = [
|
||||
'removeDependencyOnFailure',
|
||||
'failParentOnFailure',
|
||||
'continueParentOnFailure',
|
||||
'ignoreDependencyOnFailure',
|
||||
];
|
||||
const exceedLimit = this.opts.sizeLimit &&
|
||||
lengthInUtf8Bytes(jobData.data) > this.opts.sizeLimit;
|
||||
if (exceedLimit) {
|
||||
throw new Error(`The size of job ${this.name} exceeds the limit ${this.opts.sizeLimit} bytes`);
|
||||
}
|
||||
const enabledExclusiveOptions = exclusiveOptions.filter(opt => this.opts[opt]);
|
||||
if (enabledExclusiveOptions.length > 1) {
|
||||
const optionsList = enabledExclusiveOptions.join(', ');
|
||||
throw new Error(`The following options cannot be used together: ${optionsList}`);
|
||||
}
|
||||
if ((_a = this.opts) === null || _a === void 0 ? void 0 : _a.jobId) {
|
||||
if (`${parseInt(this.opts.jobId, 10)}` === ((_b = this.opts) === null || _b === void 0 ? void 0 : _b.jobId)) {
|
||||
throw new Error('Custom Id cannot be integers');
|
||||
}
|
||||
// TODO: replace this check in next breaking check with include(':')
|
||||
// By using split we are still keeping compatibility with old repeatable jobs
|
||||
if (((_c = this.opts) === null || _c === void 0 ? void 0 : _c.jobId.includes(':')) &&
|
||||
((_e = (_d = this.opts) === null || _d === void 0 ? void 0 : _d.jobId) === null || _e === void 0 ? void 0 : _e.split(':').length) !== 3) {
|
||||
throw new Error('Custom Id cannot contain :');
|
||||
}
|
||||
}
|
||||
if (this.opts.priority) {
|
||||
if (Math.trunc(this.opts.priority) !== this.opts.priority) {
|
||||
throw new Error(`Priority should not be float`);
|
||||
}
|
||||
if (this.opts.priority > PRIORITY_LIMIT) {
|
||||
throw new Error(`Priority should be between 0 and ${PRIORITY_LIMIT}`);
|
||||
}
|
||||
}
|
||||
if (this.opts.deduplication) {
|
||||
if (!((_f = this.opts.deduplication) === null || _f === void 0 ? void 0 : _f.id)) {
|
||||
throw new Error('Deduplication id must be provided');
|
||||
}
|
||||
if (this.parentKey) {
|
||||
throw new Error('Deduplication and parent options cannot be used together');
|
||||
}
|
||||
}
|
||||
if (Object.prototype.hasOwnProperty.call(this.opts, 'debounce')) {
|
||||
throw new Error('Debounce option has been removed. Use deduplication option instead');
|
||||
}
|
||||
if (typeof this.opts.backoff === 'object' &&
|
||||
typeof this.opts.backoff.jitter === 'number') {
|
||||
if (this.opts.backoff.jitter < 0 || this.opts.backoff.jitter > 1) {
|
||||
throw new Error(`Jitter should be between 0 and 1`);
|
||||
}
|
||||
}
|
||||
}
|
||||
updateStacktrace(err) {
|
||||
this.stacktrace = this.stacktrace || [];
|
||||
if (err === null || err === void 0 ? void 0 : err.stack) {
|
||||
this.stacktrace.push(err.stack);
|
||||
if (this.opts.stackTraceLimit === 0) {
|
||||
this.stacktrace = [];
|
||||
}
|
||||
else if (this.opts.stackTraceLimit) {
|
||||
this.stacktrace = this.stacktrace.slice(-this.opts.stackTraceLimit);
|
||||
}
|
||||
}
|
||||
}
|
||||
setSpanJobAttributes(span) {
|
||||
span === null || span === void 0 ? void 0 : span.setAttributes({
|
||||
[TelemetryAttributes.JobName]: this.name,
|
||||
[TelemetryAttributes.JobId]: this.id,
|
||||
});
|
||||
}
|
||||
}
|
||||
function getTraces(stacktrace) {
|
||||
if (!stacktrace) {
|
||||
return [];
|
||||
}
|
||||
const traces = tryCatch(JSON.parse, JSON, [stacktrace]);
|
||||
if (traces === errorObject || !(traces instanceof Array)) {
|
||||
return [];
|
||||
}
|
||||
else {
|
||||
return traces;
|
||||
}
|
||||
}
|
||||
function getReturnValue(_value) {
|
||||
const value = tryCatch(JSON.parse, JSON, [_value]);
|
||||
if (value !== errorObject) {
|
||||
return value;
|
||||
}
|
||||
else {
|
||||
logger('corrupted returnvalue: ' + _value, value);
|
||||
}
|
||||
}
|
||||
69
node_modules/bullmq/dist/esm/classes/lock-manager.d.ts
generated
vendored
Normal file
69
node_modules/bullmq/dist/esm/classes/lock-manager.d.ts
generated
vendored
Normal file
@@ -0,0 +1,69 @@
|
||||
import { AbortController } from './abort-controller';
|
||||
import { LockManagerWorkerContext } from '../interfaces';
|
||||
export interface LockManagerOptions {
|
||||
lockRenewTime: number;
|
||||
lockDuration: number;
|
||||
workerId: string;
|
||||
workerName?: string;
|
||||
}
|
||||
/**
|
||||
* Manages lock renewal for BullMQ workers.
|
||||
* It periodically extends locks for active jobs to prevent them from being
|
||||
* considered stalled by other workers.
|
||||
*/
|
||||
export declare class LockManager {
|
||||
protected worker: LockManagerWorkerContext;
|
||||
protected opts: LockManagerOptions;
|
||||
protected lockRenewalTimer?: NodeJS.Timeout;
|
||||
protected trackedJobs: Map<string, {
|
||||
token: string;
|
||||
ts: number;
|
||||
abortController?: AbortController;
|
||||
}>;
|
||||
protected closed: boolean;
|
||||
constructor(worker: LockManagerWorkerContext, opts: LockManagerOptions);
|
||||
/**
|
||||
* Starts the lock manager timers for lock renewal.
|
||||
*/
|
||||
start(): void;
|
||||
protected extendLocks(jobIds: string[]): Promise<void>;
|
||||
private startLockExtenderTimer;
|
||||
/**
|
||||
* Stops the lock manager and clears all timers.
|
||||
*/
|
||||
close(): Promise<void>;
|
||||
/**
|
||||
* Adds a job to be tracked for lock renewal.
|
||||
* Returns an AbortController if shouldCreateController is true, undefined otherwise.
|
||||
*/
|
||||
trackJob(jobId: string, token: string, ts: number, shouldCreateController?: boolean): AbortController | undefined;
|
||||
/**
|
||||
* Removes a job from lock renewal tracking.
|
||||
*/
|
||||
untrackJob(jobId: string): void;
|
||||
/**
|
||||
* Gets the number of jobs currently being tracked.
|
||||
*/
|
||||
getActiveJobCount(): number;
|
||||
/**
|
||||
* Checks if the lock manager is running.
|
||||
*/
|
||||
isRunning(): boolean;
|
||||
/**
|
||||
* Cancels a specific job by aborting its signal.
|
||||
* @param jobId - The ID of the job to cancel
|
||||
* @param reason - Optional reason for the cancellation
|
||||
* @returns true if the job was found and cancelled, false otherwise
|
||||
*/
|
||||
cancelJob(jobId: string, reason?: string): boolean;
|
||||
/**
|
||||
* Cancels all tracked jobs by aborting their signals.
|
||||
* @param reason - Optional reason for the cancellation
|
||||
*/
|
||||
cancelAllJobs(reason?: string): void;
|
||||
/**
|
||||
* Gets a list of all tracked job IDs.
|
||||
* @returns Array of job IDs currently being tracked
|
||||
*/
|
||||
getTrackedJobIds(): string[];
|
||||
}
|
||||
160
node_modules/bullmq/dist/esm/classes/lock-manager.js
generated
vendored
Normal file
160
node_modules/bullmq/dist/esm/classes/lock-manager.js
generated
vendored
Normal file
@@ -0,0 +1,160 @@
|
||||
import { AbortController } from './abort-controller';
|
||||
import { SpanKind, TelemetryAttributes } from '../enums';
|
||||
/**
|
||||
* Manages lock renewal for BullMQ workers.
|
||||
* It periodically extends locks for active jobs to prevent them from being
|
||||
* considered stalled by other workers.
|
||||
*/
|
||||
export class LockManager {
|
||||
constructor(worker, opts) {
|
||||
this.worker = worker;
|
||||
this.opts = opts;
|
||||
// Maps job ids with their tokens, timestamps, and abort controllers
|
||||
this.trackedJobs = new Map();
|
||||
this.closed = false;
|
||||
}
|
||||
/**
|
||||
* Starts the lock manager timers for lock renewal.
|
||||
*/
|
||||
start() {
|
||||
if (this.closed) {
|
||||
return;
|
||||
}
|
||||
// Start lock renewal timer if not disabled
|
||||
if (this.opts.lockRenewTime > 0) {
|
||||
this.startLockExtenderTimer();
|
||||
}
|
||||
}
|
||||
async extendLocks(jobIds) {
|
||||
await this.worker.trace(SpanKind.INTERNAL, 'extendLocks', this.worker.name, async (span) => {
|
||||
span === null || span === void 0 ? void 0 : span.setAttributes({
|
||||
[TelemetryAttributes.WorkerId]: this.opts.workerId,
|
||||
[TelemetryAttributes.WorkerName]: this.opts.workerName,
|
||||
[TelemetryAttributes.WorkerJobsToExtendLocks]: jobIds,
|
||||
});
|
||||
try {
|
||||
const jobTokens = jobIds.map(id => { var _a; return ((_a = this.trackedJobs.get(id)) === null || _a === void 0 ? void 0 : _a.token) || ''; });
|
||||
const erroredJobIds = await this.worker.extendJobLocks(jobIds, jobTokens, this.opts.lockDuration);
|
||||
if (erroredJobIds.length > 0) {
|
||||
this.worker.emit('lockRenewalFailed', erroredJobIds);
|
||||
for (const jobId of erroredJobIds) {
|
||||
this.worker.emit('error', new Error(`could not renew lock for job ${jobId}`));
|
||||
}
|
||||
}
|
||||
const succeededJobIds = jobIds.filter(id => !erroredJobIds.includes(id));
|
||||
if (succeededJobIds.length > 0) {
|
||||
this.worker.emit('locksRenewed', {
|
||||
count: succeededJobIds.length,
|
||||
jobIds: succeededJobIds,
|
||||
});
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
this.worker.emit('error', err);
|
||||
}
|
||||
});
|
||||
}
|
||||
startLockExtenderTimer() {
|
||||
clearTimeout(this.lockRenewalTimer);
|
||||
if (!this.closed) {
|
||||
this.lockRenewalTimer = setTimeout(async () => {
|
||||
// Get all the jobs whose locks expire in less than 1/2 of the lockRenewTime
|
||||
const now = Date.now();
|
||||
const jobsToExtend = [];
|
||||
for (const jobId of this.trackedJobs.keys()) {
|
||||
const tracked = this.trackedJobs.get(jobId);
|
||||
const { ts, token, abortController } = tracked;
|
||||
if (!ts) {
|
||||
this.trackedJobs.set(jobId, { token, ts: now, abortController });
|
||||
continue;
|
||||
}
|
||||
if (ts + this.opts.lockRenewTime / 2 < now) {
|
||||
this.trackedJobs.set(jobId, { token, ts: now, abortController });
|
||||
jobsToExtend.push(jobId);
|
||||
}
|
||||
}
|
||||
if (jobsToExtend.length) {
|
||||
await this.extendLocks(jobsToExtend);
|
||||
}
|
||||
this.startLockExtenderTimer();
|
||||
}, this.opts.lockRenewTime / 2);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Stops the lock manager and clears all timers.
|
||||
*/
|
||||
async close() {
|
||||
if (this.closed) {
|
||||
return;
|
||||
}
|
||||
this.closed = true;
|
||||
if (this.lockRenewalTimer) {
|
||||
clearTimeout(this.lockRenewalTimer);
|
||||
this.lockRenewalTimer = undefined;
|
||||
}
|
||||
this.trackedJobs.clear();
|
||||
}
|
||||
/**
|
||||
* Adds a job to be tracked for lock renewal.
|
||||
* Returns an AbortController if shouldCreateController is true, undefined otherwise.
|
||||
*/
|
||||
trackJob(jobId, token, ts, shouldCreateController = false) {
|
||||
const abortController = shouldCreateController
|
||||
? new AbortController()
|
||||
: undefined;
|
||||
if (!this.closed && jobId) {
|
||||
this.trackedJobs.set(jobId, { token, ts, abortController });
|
||||
}
|
||||
return abortController;
|
||||
}
|
||||
/**
|
||||
* Removes a job from lock renewal tracking.
|
||||
*/
|
||||
untrackJob(jobId) {
|
||||
this.trackedJobs.delete(jobId);
|
||||
}
|
||||
/**
|
||||
* Gets the number of jobs currently being tracked.
|
||||
*/
|
||||
getActiveJobCount() {
|
||||
return this.trackedJobs.size;
|
||||
}
|
||||
/**
|
||||
* Checks if the lock manager is running.
|
||||
*/
|
||||
isRunning() {
|
||||
return !this.closed && this.lockRenewalTimer !== undefined;
|
||||
}
|
||||
/**
|
||||
* Cancels a specific job by aborting its signal.
|
||||
* @param jobId - The ID of the job to cancel
|
||||
* @param reason - Optional reason for the cancellation
|
||||
* @returns true if the job was found and cancelled, false otherwise
|
||||
*/
|
||||
cancelJob(jobId, reason) {
|
||||
const tracked = this.trackedJobs.get(jobId);
|
||||
if (tracked === null || tracked === void 0 ? void 0 : tracked.abortController) {
|
||||
tracked.abortController.abort(reason);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
/**
|
||||
* Cancels all tracked jobs by aborting their signals.
|
||||
* @param reason - Optional reason for the cancellation
|
||||
*/
|
||||
cancelAllJobs(reason) {
|
||||
for (const tracked of this.trackedJobs.values()) {
|
||||
if (tracked.abortController) {
|
||||
tracked.abortController.abort(reason);
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Gets a list of all tracked job IDs.
|
||||
* @returns Array of job IDs currently being tracked
|
||||
*/
|
||||
getTrackedJobIds() {
|
||||
return Array.from(this.trackedJobs.keys());
|
||||
}
|
||||
}
|
||||
3
node_modules/bullmq/dist/esm/classes/main-base.d.ts
generated
vendored
Normal file
3
node_modules/bullmq/dist/esm/classes/main-base.d.ts
generated
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
import { Receiver } from '../interfaces';
|
||||
declare const _default: (send: (msg: any) => Promise<void>, receiver: Receiver) => void;
|
||||
export default _default;
|
||||
44
node_modules/bullmq/dist/esm/classes/main-base.js
generated
vendored
Normal file
44
node_modules/bullmq/dist/esm/classes/main-base.js
generated
vendored
Normal file
@@ -0,0 +1,44 @@
|
||||
/**
|
||||
* Wrapper for sandboxing.
|
||||
*
|
||||
*/
|
||||
import { ChildProcessor } from './child-processor';
|
||||
import { ParentCommand, ChildCommand } from '../enums';
|
||||
import { errorToJSON, toString } from '../utils';
|
||||
export default (send, receiver) => {
|
||||
const childProcessor = new ChildProcessor(send, receiver);
|
||||
receiver === null || receiver === void 0 ? void 0 : receiver.on('message', async (msg) => {
|
||||
try {
|
||||
switch (msg.cmd) {
|
||||
case ChildCommand.Init:
|
||||
await childProcessor.init(msg.value);
|
||||
break;
|
||||
case ChildCommand.Start:
|
||||
await childProcessor.start(msg.job, msg === null || msg === void 0 ? void 0 : msg.token);
|
||||
break;
|
||||
case ChildCommand.Stop:
|
||||
break;
|
||||
case ChildCommand.Cancel:
|
||||
childProcessor.cancel(msg.value);
|
||||
break;
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
console.error('Error handling child message');
|
||||
}
|
||||
});
|
||||
process.on('SIGTERM', () => childProcessor.waitForCurrentJobAndExit());
|
||||
process.on('SIGINT', () => childProcessor.waitForCurrentJobAndExit());
|
||||
process.on('uncaughtException', async (err) => {
|
||||
if (typeof err !== 'object') {
|
||||
err = new Error(toString(err));
|
||||
}
|
||||
await send({
|
||||
cmd: ParentCommand.Failed,
|
||||
value: errorToJSON(err),
|
||||
});
|
||||
// An uncaughException leaves this process in a potentially undetermined state so
|
||||
// we must exit
|
||||
process.exit();
|
||||
});
|
||||
};
|
||||
1
node_modules/bullmq/dist/esm/classes/main-worker.d.ts
generated
vendored
Normal file
1
node_modules/bullmq/dist/esm/classes/main-worker.d.ts
generated
vendored
Normal file
@@ -0,0 +1 @@
|
||||
export {};
|
||||
7
node_modules/bullmq/dist/esm/classes/main-worker.js
generated
vendored
Normal file
7
node_modules/bullmq/dist/esm/classes/main-worker.js
generated
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
/**
|
||||
* Worker Thread wrapper for sandboxing
|
||||
*
|
||||
*/
|
||||
import { parentPort } from 'worker_threads';
|
||||
import mainBase from './main-base';
|
||||
mainBase(async (msg) => parentPort.postMessage(msg), parentPort);
|
||||
1
node_modules/bullmq/dist/esm/classes/main.d.ts
generated
vendored
Normal file
1
node_modules/bullmq/dist/esm/classes/main.d.ts
generated
vendored
Normal file
@@ -0,0 +1 @@
|
||||
export {};
|
||||
7
node_modules/bullmq/dist/esm/classes/main.js
generated
vendored
Normal file
7
node_modules/bullmq/dist/esm/classes/main.js
generated
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
/**
|
||||
* Child process wrapper for sandboxing.
|
||||
*
|
||||
*/
|
||||
import { childSend } from '../utils';
|
||||
import mainBase from './main-base';
|
||||
mainBase((msg) => childSend(process, msg), process);
|
||||
96
node_modules/bullmq/dist/esm/classes/node-redis-client.d.ts
generated
vendored
Normal file
96
node_modules/bullmq/dist/esm/classes/node-redis-client.d.ts
generated
vendored
Normal file
@@ -0,0 +1,96 @@
|
||||
import { IRedisClient } from '../interfaces/redis-client';
|
||||
export type NodeRedisCommandArgument = string | Buffer;
|
||||
export interface NodeRedisRawTransaction {
|
||||
hGetAll(key: string): this;
|
||||
hSet(key: string, data: Record<string, string | number>): this;
|
||||
hScan(key: string, cursor: string, options?: Record<string, unknown>): this;
|
||||
sMembers(key: string): this;
|
||||
sScan(key: string, cursor: string, options?: Record<string, unknown>): this;
|
||||
zRange(key: string, start: number, end: number): this;
|
||||
lRange(key: string, start: number, end: number): this;
|
||||
lLen(key: string): this;
|
||||
del(keys: string[]): this;
|
||||
evalSha(sha: string, options: {
|
||||
keys: string[];
|
||||
arguments: NodeRedisCommandArgument[];
|
||||
}): this;
|
||||
exec(): Promise<unknown[] | null>;
|
||||
}
|
||||
export interface NodeRedisRawClient {
|
||||
isReady: boolean;
|
||||
isOpen: boolean;
|
||||
options?: Record<string, unknown>;
|
||||
on(event: string, listener: (...args: any[]) => void): this;
|
||||
connect(): Promise<unknown>;
|
||||
close?(): Promise<void>;
|
||||
destroy(): void | Promise<void>;
|
||||
quit(): Promise<unknown>;
|
||||
duplicate(): NodeRedisRawClient;
|
||||
scriptLoad(lua: string): Promise<unknown>;
|
||||
evalSha(sha: string, options: {
|
||||
keys: string[];
|
||||
arguments: NodeRedisCommandArgument[];
|
||||
}): Promise<unknown>;
|
||||
eval(lua: string, options: {
|
||||
keys: string[];
|
||||
arguments: NodeRedisCommandArgument[];
|
||||
}): Promise<unknown>;
|
||||
multi(): NodeRedisRawTransaction;
|
||||
hGetAll(key: string): Promise<Record<string, string>>;
|
||||
hGet(key: string, field: string): Promise<string | null | undefined>;
|
||||
hmGet(key: string, fields: string[]): Promise<(string | null | undefined)[]>;
|
||||
hSet(key: string, data: Record<string, string | number>): Promise<number>;
|
||||
hDel(key: string, fields: string[]): Promise<number>;
|
||||
hExists(key: string, field: string): Promise<boolean>;
|
||||
get(key: string): Promise<string | null | undefined>;
|
||||
set(key: string, value: string, options?: Record<string, unknown>): Promise<string | null>;
|
||||
del(keys: string[]): Promise<number>;
|
||||
zRange(key: string, start: number, end: number, options?: Record<string, unknown>): Promise<string[]>;
|
||||
zRangeWithScores?(key: string, start: number, end: number, options?: Record<string, unknown>): Promise<{
|
||||
value: string;
|
||||
score: number;
|
||||
}[]>;
|
||||
zCard(key: string): Promise<number>;
|
||||
zScore(key: string, member: string): Promise<number | string | null>;
|
||||
zAdd(key: string, members: {
|
||||
score: number;
|
||||
value: string;
|
||||
}[]): Promise<number>;
|
||||
zRem(key: string, members: string[]): Promise<number>;
|
||||
lRange(key: string, start: number, end: number): Promise<string[]>;
|
||||
lLen(key: string): Promise<number>;
|
||||
lTrim(key: string, start: number, end: number): Promise<unknown>;
|
||||
lPos(key: string, value: string): Promise<number | null | undefined>;
|
||||
lPush(key: string, values: string[]): Promise<number>;
|
||||
rPop(key: string): Promise<string | null>;
|
||||
sMembers(key: string): Promise<string[]>;
|
||||
sAdd(key: string, members: string[]): Promise<number>;
|
||||
sCard(key: string): Promise<number>;
|
||||
xAdd(key: string, id: string, fields: Record<string, string>, options?: Record<string, unknown>): Promise<string>;
|
||||
xRead(streams: {
|
||||
key: string;
|
||||
id: string;
|
||||
}[], options?: Record<string, unknown>): Promise<any>;
|
||||
xTrim(key: string, strategy: 'MAXLEN', threshold: number, options?: Record<string, unknown>): Promise<number>;
|
||||
xLen(key: string): Promise<number>;
|
||||
xRevRange(key: string, end: string, start: string, options?: Record<string, unknown>): Promise<any[]>;
|
||||
bzPopMin(key: string, timeout: number): Promise<{
|
||||
key: string;
|
||||
value: string;
|
||||
score: number | string;
|
||||
} | null>;
|
||||
info(): Promise<string>;
|
||||
clientSetName(name: string): Promise<unknown>;
|
||||
sendCommand(args: string[]): Promise<string>;
|
||||
scan(cursor: string, options?: Record<string, unknown>): Promise<{
|
||||
cursor: number | string;
|
||||
keys: string[];
|
||||
}>;
|
||||
scanIterator(options?: Record<string, unknown>): AsyncIterable<string | string[]>;
|
||||
keys(pattern: string): Promise<string[]>;
|
||||
exists(keys: string[]): Promise<number>;
|
||||
incr(key: string): Promise<number>;
|
||||
incrBy(key: string, increment: number): Promise<number>;
|
||||
flushAll(): Promise<string>;
|
||||
}
|
||||
export declare function createNodeRedisClient(client: unknown): IRedisClient;
|
||||
768
node_modules/bullmq/dist/esm/classes/node-redis-client.js
generated
vendored
Normal file
768
node_modules/bullmq/dist/esm/classes/node-redis-client.js
generated
vendored
Normal file
@@ -0,0 +1,768 @@
|
||||
import { __asyncValues } from "tslib";
|
||||
import { createHash } from 'crypto';
|
||||
import { EventEmitter } from 'events';
|
||||
import { Readable } from 'stream';
|
||||
import { ConnectionClosedError } from './errors/connection-closed-error';
|
||||
function normalizeScriptArgs(args) {
|
||||
return args.length === 1 && Array.isArray(args[0]) ? args[0] : args;
|
||||
}
|
||||
function isConnectionClosedError(err) {
|
||||
return ((err === null || err === void 0 ? void 0 : err.message) === 'Disconnects client' ||
|
||||
(err === null || err === void 0 ? void 0 : err.message) === 'The client is closed' ||
|
||||
(err === null || err === void 0 ? void 0 : err.message) === 'Connection is closed.');
|
||||
}
|
||||
export function createNodeRedisClient(client) {
|
||||
return new NodeRedisAdapter(client);
|
||||
}
|
||||
/**
|
||||
* Full wrapper (not augmentation) because node-redis's API is structurally
|
||||
* different from ioredis and cannot be patched in-place.
|
||||
*/
|
||||
class NodeRedisAdapter extends EventEmitter {
|
||||
/**
|
||||
* Expose connection status using the vocabulary that
|
||||
* {@link RedisConnection.waitUntilReady} expects:
|
||||
* 'wait' → not yet connected, call connect()
|
||||
* 'ready' → usable
|
||||
* 'end' → permanently closed
|
||||
*/
|
||||
get status() {
|
||||
if (this.statusOverride) {
|
||||
return this.statusOverride;
|
||||
}
|
||||
if (this.raw.isReady) {
|
||||
return 'ready';
|
||||
}
|
||||
if (this.raw.isOpen) {
|
||||
return 'connect';
|
||||
}
|
||||
// Distinguish "never connected" from "disconnected after use"
|
||||
return this.hasConnected ? 'end' : 'wait';
|
||||
}
|
||||
set status(val) {
|
||||
// Allow RedisConnection to forcibly set 'end'
|
||||
if (val === 'end') {
|
||||
this.destroying = true;
|
||||
if (this.raw.isOpen) {
|
||||
try {
|
||||
this.raw.quit().catch(() => { });
|
||||
}
|
||||
catch (_a) {
|
||||
// already closed
|
||||
}
|
||||
}
|
||||
}
|
||||
this.statusOverride = val;
|
||||
}
|
||||
get options() {
|
||||
var _a;
|
||||
return (_a = this.raw.options) !== null && _a !== void 0 ? _a : {};
|
||||
}
|
||||
set options(val) {
|
||||
// no-op – callers sometimes assign
|
||||
}
|
||||
constructor(raw) {
|
||||
super();
|
||||
this.raw = raw;
|
||||
this.scripts = new Map();
|
||||
this.hasConnected = false;
|
||||
this.destroying = false;
|
||||
this.isCluster = false; // TODO: cluster support
|
||||
// Track first connection so status can distinguish 'wait' vs 'end'.
|
||||
// When a connectionName is set (via duplicate()), delay the 'ready'
|
||||
// event until CLIENT SETNAME completes so that callers waiting for
|
||||
// 'ready' (e.g. RedisConnection.waitUntilReady) see the name already
|
||||
// applied.
|
||||
raw.on('ready', () => {
|
||||
this.hasConnected = true;
|
||||
if (this.connectionName) {
|
||||
this.raw.clientSetName(this.connectionName).then(() => this.emit('ready'), () => this.emit('ready'));
|
||||
}
|
||||
else {
|
||||
this.emit('ready');
|
||||
}
|
||||
});
|
||||
raw.on('error', (err) => {
|
||||
// Suppress the expected DisconnectsClientError that node-redis emits
|
||||
// when destroy() is called intentionally (e.g. during close/disconnect).
|
||||
if (this.destroying && isConnectionClosedError(err)) {
|
||||
return;
|
||||
}
|
||||
this.emit('error', err);
|
||||
});
|
||||
raw.on('end', () => this.emit('close'));
|
||||
raw.on('reconnecting', () => this.emit('reconnecting'));
|
||||
// Auto-connect eagerly, like ioredis does in its constructor.
|
||||
// This ensures commands can be issued immediately without an
|
||||
// explicit connect() call. The promise is stored so that
|
||||
// connect() is idempotent and callers can still await it.
|
||||
if (!raw.isOpen) {
|
||||
this.connectPromise = raw.connect().then(() => {
|
||||
this.connectPromise = undefined;
|
||||
}, (err) => {
|
||||
this.connectPromise = undefined;
|
||||
// Don't throw — errors surface via the 'error' event.
|
||||
});
|
||||
}
|
||||
}
|
||||
// ---------------------------------------------------------------
|
||||
// Connection lifecycle
|
||||
// ---------------------------------------------------------------
|
||||
async connect() {
|
||||
if (this.connectPromise) {
|
||||
return this.connectPromise;
|
||||
}
|
||||
if (!this.raw.isOpen) {
|
||||
this.connectPromise = this.raw.connect().then(() => {
|
||||
this.connectPromise = undefined;
|
||||
}, (err) => {
|
||||
this.connectPromise = undefined;
|
||||
throw err;
|
||||
});
|
||||
return this.connectPromise;
|
||||
}
|
||||
if (!this.raw.isReady) {
|
||||
await new Promise((resolve, reject) => {
|
||||
const onReady = () => {
|
||||
cleanup();
|
||||
resolve();
|
||||
};
|
||||
const onError = (err) => {
|
||||
cleanup();
|
||||
reject(err);
|
||||
};
|
||||
const onEnd = () => {
|
||||
cleanup();
|
||||
reject(new Error('Connection ended before ready event'));
|
||||
};
|
||||
const cleanup = () => {
|
||||
this.off('ready', onReady);
|
||||
this.off('error', onError);
|
||||
this.off('end', onEnd);
|
||||
};
|
||||
this.once('ready', onReady);
|
||||
this.once('error', onError);
|
||||
this.once('end', onEnd);
|
||||
});
|
||||
}
|
||||
}
|
||||
disconnect(reconnect = false) {
|
||||
this.destroying = true;
|
||||
if (!reconnect) {
|
||||
this.statusOverride = 'end';
|
||||
}
|
||||
try {
|
||||
if (this.raw.isOpen) {
|
||||
// Use destroy() for immediate teardown. This interrupts any pending
|
||||
// blocking commands (e.g. BZPOPMIN) without waiting for them to
|
||||
// complete. The resulting "Disconnects client" rejections are handled
|
||||
// by BullMQ's isNotConnectionError() checks.
|
||||
this.raw.destroy();
|
||||
}
|
||||
}
|
||||
catch (_a) {
|
||||
// Swallow errors from already-closed connections
|
||||
}
|
||||
this.emit('close');
|
||||
if (reconnect) {
|
||||
this.statusOverride = undefined;
|
||||
this.emit('reconnecting');
|
||||
this.connect()
|
||||
.catch(err => {
|
||||
if (!isConnectionClosedError(err)) {
|
||||
this.emit('error', err);
|
||||
}
|
||||
})
|
||||
.finally(() => {
|
||||
this.destroying = false;
|
||||
});
|
||||
}
|
||||
else {
|
||||
// Emit both 'close' and 'end' so that all listeners are unblocked.
|
||||
// RedisConnection.close() listens for 'close', RedisConnection.disconnect() listens for 'end'.
|
||||
this.emit('end');
|
||||
}
|
||||
}
|
||||
async quit() {
|
||||
if (this.destroying || this.statusOverride === 'end') {
|
||||
setImmediate(() => {
|
||||
this.emit('end');
|
||||
this.emit('close');
|
||||
});
|
||||
return 'OK';
|
||||
}
|
||||
this.destroying = true;
|
||||
try {
|
||||
if (this.raw.isOpen) {
|
||||
try {
|
||||
await this.raw.quit();
|
||||
}
|
||||
catch (_a) {
|
||||
// Swallow errors from already-closing connections
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (_b) {
|
||||
// Swallow errors from already-closing connections
|
||||
}
|
||||
this.statusOverride = 'end';
|
||||
// Emit on next tick so callers can register listeners after await quit()
|
||||
setImmediate(() => {
|
||||
this.emit('end');
|
||||
this.emit('close');
|
||||
});
|
||||
return 'OK';
|
||||
}
|
||||
duplicate(...args) {
|
||||
const dup = this.raw.duplicate();
|
||||
const adapter = new NodeRedisAdapter(dup);
|
||||
// Copy registered scripts to the duplicate
|
||||
for (const [name, script] of this.scripts) {
|
||||
adapter.scripts.set(name, script);
|
||||
adapter[name] = (...args) => adapter.runCommand(name, args);
|
||||
}
|
||||
// Handle connectionName option (ioredis calls CLIENT SETNAME automatically).
|
||||
// Setting connectionName BEFORE auto-connect resolves ensures the
|
||||
// constructor's 'ready' handler applies CLIENT SETNAME before emitting
|
||||
// 'ready', so callers (like RedisConnection.waitUntilReady) see the name
|
||||
// already set when the connection is reported as ready.
|
||||
const opts = args[0];
|
||||
if (opts && typeof opts === 'object' && opts.connectionName) {
|
||||
adapter.connectionName = opts.connectionName;
|
||||
}
|
||||
return adapter;
|
||||
}
|
||||
// ---------------------------------------------------------------
|
||||
// Lua script engine
|
||||
// ---------------------------------------------------------------
|
||||
defineCommand(name, definition) {
|
||||
const sha = createHash('sha1').update(definition.lua).digest('hex');
|
||||
this.scripts.set(name, {
|
||||
sha,
|
||||
lua: definition.lua,
|
||||
numberOfKeys: definition.numberOfKeys,
|
||||
});
|
||||
// Mimic ioredis behavior: add a callable property on the instance
|
||||
// so that `(client as any)[name]` is defined (used by ScriptLoader cache check)
|
||||
this[name] = (...args) => this.runCommand(name, args);
|
||||
// Pre-load the script into Redis so that EVALSHA in transactions works
|
||||
// immediately. This mirrors what ioredis does under the hood.
|
||||
this.raw.scriptLoad(definition.lua).catch(() => {
|
||||
// Ignore errors here – runCommand has NOSCRIPT fallback for non-tx path
|
||||
});
|
||||
}
|
||||
async runCommand(name, args) {
|
||||
var _a, _b;
|
||||
const script = this.scripts.get(name);
|
||||
if (!script) {
|
||||
throw new Error(`BullMQ: unknown command "${name}"`);
|
||||
}
|
||||
const commandArgs = normalizeScriptArgs(args);
|
||||
const { sha, lua, numberOfKeys } = script;
|
||||
const keys = commandArgs.slice(0, numberOfKeys).map(String);
|
||||
// Preserve Buffer arguments (e.g. msgpack data) – only stringify non-Buffers.
|
||||
// Convert undefined/null to '' to match ioredis behavior (keeps arg positions).
|
||||
const argv = commandArgs.slice(numberOfKeys).map((a) => {
|
||||
if (Buffer.isBuffer(a)) {
|
||||
return a;
|
||||
}
|
||||
if (a === undefined || a === null) {
|
||||
return '';
|
||||
}
|
||||
return String(a);
|
||||
});
|
||||
try {
|
||||
return await this.raw.evalSha(sha, { keys, arguments: argv });
|
||||
}
|
||||
catch (err) {
|
||||
if (this.destroying && isConnectionClosedError(err)) {
|
||||
return null;
|
||||
}
|
||||
if (isConnectionClosedError(err)) {
|
||||
throw new ConnectionClosedError(err.message, err);
|
||||
}
|
||||
// NOSCRIPT – fall back to EVAL which also caches the script
|
||||
if ((_b = (_a = err === null || err === void 0 ? void 0 : err.message) === null || _a === void 0 ? void 0 : _a.includes) === null || _b === void 0 ? void 0 : _b.call(_a, 'NOSCRIPT')) {
|
||||
try {
|
||||
return await this.raw.eval(lua, { keys, arguments: argv });
|
||||
}
|
||||
catch (evalErr) {
|
||||
if (this.destroying && isConnectionClosedError(evalErr)) {
|
||||
return null;
|
||||
}
|
||||
if (isConnectionClosedError(evalErr)) {
|
||||
throw new ConnectionClosedError(evalErr.message, evalErr);
|
||||
}
|
||||
throw evalErr;
|
||||
}
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
// ---------------------------------------------------------------
|
||||
// Pipeline / Transaction
|
||||
// ---------------------------------------------------------------
|
||||
multi() {
|
||||
return new NodeRedisTransaction(this.raw.multi(), this.scripts);
|
||||
}
|
||||
pipeline() {
|
||||
// node-redis doesn't have a separate pipeline concept;
|
||||
// use multi() which behaves similarly for batching.
|
||||
return this.multi();
|
||||
}
|
||||
// ---------------------------------------------------------------
|
||||
// Hash commands
|
||||
// ---------------------------------------------------------------
|
||||
async hgetall(key) {
|
||||
const result = await this.raw.hGetAll(key);
|
||||
return result !== null && result !== void 0 ? result : {};
|
||||
}
|
||||
async hget(key, field) {
|
||||
var _a;
|
||||
return (_a = (await this.raw.hGet(key, field))) !== null && _a !== void 0 ? _a : null;
|
||||
}
|
||||
async hmget(key, ...fields) {
|
||||
const result = await this.raw.hmGet(key, fields);
|
||||
return result.map((v) => v !== null && v !== void 0 ? v : null);
|
||||
}
|
||||
async hset(key, dataOrField, ...rest) {
|
||||
if (typeof dataOrField === 'object') {
|
||||
// Record-based call: hset(key, { field: value, ... })
|
||||
return await this.raw.hSet(key, dataOrField);
|
||||
}
|
||||
// Varargs call (ioredis compat): hset(key, field, value, field, value, ...)
|
||||
const record = {};
|
||||
record[dataOrField] = String(rest[0]);
|
||||
for (let i = 1; i < rest.length; i += 2) {
|
||||
record[String(rest[i])] = String(rest[i + 1]);
|
||||
}
|
||||
return await this.raw.hSet(key, record);
|
||||
}
|
||||
async hdel(key, ...fields) {
|
||||
return await this.raw.hDel(key, fields);
|
||||
}
|
||||
async hexists(key, field) {
|
||||
const exists = await this.raw.hExists(key, field);
|
||||
return exists ? 1 : 0;
|
||||
}
|
||||
// ---------------------------------------------------------------
|
||||
// String commands
|
||||
// ---------------------------------------------------------------
|
||||
async get(key) {
|
||||
var _a;
|
||||
return (_a = (await this.raw.get(key))) !== null && _a !== void 0 ? _a : null;
|
||||
}
|
||||
async set(key, value, options) {
|
||||
const opts = {};
|
||||
if ((options === null || options === void 0 ? void 0 : options.PX) != null) {
|
||||
opts.PX = options.PX;
|
||||
}
|
||||
else if ((options === null || options === void 0 ? void 0 : options.EX) != null) {
|
||||
opts.EX = options.EX;
|
||||
}
|
||||
return await this.raw.set(key, String(value), opts);
|
||||
}
|
||||
async del(...keys) {
|
||||
if (keys.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
return await this.raw.del(keys);
|
||||
}
|
||||
// ---------------------------------------------------------------
|
||||
// Sorted set commands
|
||||
// ---------------------------------------------------------------
|
||||
async zrange(key, start, end, options) {
|
||||
if (options === null || options === void 0 ? void 0 : options.WITHSCORES) {
|
||||
// node-redis v5 uses a separate method for WITHSCORES
|
||||
const items = await this.raw.zRangeWithScores(key, start, end);
|
||||
// Flatten to [member, score, member, score, …] like ioredis
|
||||
const flat = [];
|
||||
for (const item of items) {
|
||||
flat.push(item.value, String(item.score));
|
||||
}
|
||||
return flat;
|
||||
}
|
||||
return await this.raw.zRange(key, start, end);
|
||||
}
|
||||
async zrevrange(key, start, end, options) {
|
||||
if (options === null || options === void 0 ? void 0 : options.WITHSCORES) {
|
||||
const items = await this.raw.zRangeWithScores(key, start, end, {
|
||||
REV: true,
|
||||
});
|
||||
const flat = [];
|
||||
for (const item of items) {
|
||||
flat.push(item.value, String(item.score));
|
||||
}
|
||||
return flat;
|
||||
}
|
||||
return await this.raw.zRange(key, start, end, { REV: true });
|
||||
}
|
||||
async zcard(key) {
|
||||
return await this.raw.zCard(key);
|
||||
}
|
||||
async zscore(key, member) {
|
||||
const score = await this.raw.zScore(key, member);
|
||||
return score != null ? String(score) : null;
|
||||
}
|
||||
// ---------------------------------------------------------------
|
||||
// List commands
|
||||
// ---------------------------------------------------------------
|
||||
async lrange(key, start, end) {
|
||||
return await this.raw.lRange(key, start, end);
|
||||
}
|
||||
async llen(key) {
|
||||
return await this.raw.lLen(key);
|
||||
}
|
||||
async ltrim(key, start, end) {
|
||||
await this.raw.lTrim(key, start, end);
|
||||
return 'OK';
|
||||
}
|
||||
async lpos(key, value) {
|
||||
var _a;
|
||||
return (_a = (await this.raw.lPos(key, value))) !== null && _a !== void 0 ? _a : null;
|
||||
}
|
||||
// ---------------------------------------------------------------
|
||||
// Set commands
|
||||
// ---------------------------------------------------------------
|
||||
async smembers(key) {
|
||||
return await this.raw.sMembers(key);
|
||||
}
|
||||
// ---------------------------------------------------------------
|
||||
// Stream commands
|
||||
// ---------------------------------------------------------------
|
||||
async xadd(key, id, fields, options) {
|
||||
const opts = {};
|
||||
if ((options === null || options === void 0 ? void 0 : options.MAXLEN) != null) {
|
||||
opts.TRIM = {
|
||||
strategy: 'MAXLEN',
|
||||
threshold: options.MAXLEN,
|
||||
strategyModifier: options.approximate === false ? undefined : '~',
|
||||
};
|
||||
}
|
||||
// node-redis xAdd rejects numeric field values — stringify all values
|
||||
const strFields = {};
|
||||
for (const [k, v] of Object.entries(fields)) {
|
||||
strFields[k] = String(v);
|
||||
}
|
||||
return await this.raw.xAdd(key, id, strFields, opts);
|
||||
}
|
||||
async xread(streams, options) {
|
||||
const opts = {};
|
||||
if ((options === null || options === void 0 ? void 0 : options.BLOCK) != null) {
|
||||
opts.BLOCK = options.BLOCK;
|
||||
}
|
||||
if ((options === null || options === void 0 ? void 0 : options.COUNT) != null) {
|
||||
opts.COUNT = options.COUNT;
|
||||
}
|
||||
const streamArgs = streams.map(s => ({ key: s.key, id: s.id }));
|
||||
let result;
|
||||
try {
|
||||
result = await this.raw.xRead(streamArgs, opts);
|
||||
}
|
||||
catch (err) {
|
||||
if (this.destroying && isConnectionClosedError(err)) {
|
||||
return null;
|
||||
}
|
||||
if (isConnectionClosedError(err)) {
|
||||
throw new ConnectionClosedError(err.message, err);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
if (!result) {
|
||||
return null;
|
||||
}
|
||||
// Normalize to ioredis format: [[streamName, [[id, [field, value, …]], …]], …]
|
||||
return result.map((stream) => [
|
||||
stream.name,
|
||||
stream.messages.map((msg) => [
|
||||
msg.id,
|
||||
Object.entries(msg.message).flat(),
|
||||
]),
|
||||
]);
|
||||
}
|
||||
async xtrim(key, strategy, threshold, options) {
|
||||
const strategyModifier = (options === null || options === void 0 ? void 0 : options.approximate) === false ? undefined : '~';
|
||||
return await this.raw.xTrim(key, strategy, threshold, {
|
||||
strategyModifier,
|
||||
});
|
||||
}
|
||||
// ---------------------------------------------------------------
|
||||
// Blocking commands
|
||||
// ---------------------------------------------------------------
|
||||
async bzpopmin(key, timeout) {
|
||||
let result;
|
||||
try {
|
||||
result = await this.raw.bzPopMin(key, timeout);
|
||||
}
|
||||
catch (err) {
|
||||
if (this.destroying && isConnectionClosedError(err)) {
|
||||
return null;
|
||||
}
|
||||
if (isConnectionClosedError(err)) {
|
||||
throw new ConnectionClosedError(err.message, err);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
if (!result) {
|
||||
return null;
|
||||
}
|
||||
return [result.key, result.value, String(result.score)];
|
||||
}
|
||||
// ---------------------------------------------------------------
|
||||
// Server / admin commands
|
||||
// ---------------------------------------------------------------
|
||||
async info() {
|
||||
return await this.raw.info();
|
||||
}
|
||||
async clientSetName(name) {
|
||||
return await this.raw.clientSetName(name);
|
||||
}
|
||||
async clientList() {
|
||||
return await this.raw.sendCommand(['CLIENT', 'LIST']);
|
||||
}
|
||||
// ---------------------------------------------------------------
|
||||
// Key scanning
|
||||
// ---------------------------------------------------------------
|
||||
async scan(cursor, options) {
|
||||
const opts = {};
|
||||
if (options === null || options === void 0 ? void 0 : options.MATCH) {
|
||||
opts.MATCH = options.MATCH;
|
||||
}
|
||||
if (options === null || options === void 0 ? void 0 : options.COUNT) {
|
||||
opts.COUNT = options.COUNT;
|
||||
}
|
||||
const result = await this.raw.scan(String(cursor), opts);
|
||||
return [String(result.cursor), result.keys];
|
||||
}
|
||||
scanStream(options) {
|
||||
const raw = this.raw;
|
||||
const connectPromise = this.connectPromise;
|
||||
const scanOpts = {};
|
||||
if (options.match) {
|
||||
scanOpts.MATCH = options.match;
|
||||
}
|
||||
if (options.count) {
|
||||
scanOpts.COUNT = options.count;
|
||||
}
|
||||
const readable = new Readable({
|
||||
objectMode: true,
|
||||
async read() {
|
||||
var _a, e_1, _b, _c;
|
||||
try {
|
||||
if (connectPromise) {
|
||||
await connectPromise;
|
||||
}
|
||||
try {
|
||||
for (var _d = true, _e = __asyncValues(raw.scanIterator(scanOpts)), _f; _f = await _e.next(), _a = _f.done, !_a; _d = true) {
|
||||
_c = _f.value;
|
||||
_d = false;
|
||||
const keys = _c;
|
||||
if (!readable.push(Array.isArray(keys) ? keys : [keys])) {
|
||||
return; // backpressure
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
||||
finally {
|
||||
try {
|
||||
if (!_d && !_a && (_b = _e.return)) await _b.call(_e);
|
||||
}
|
||||
finally { if (e_1) throw e_1.error; }
|
||||
}
|
||||
readable.push(null); // EOF
|
||||
}
|
||||
catch (err) {
|
||||
readable.destroy(err);
|
||||
}
|
||||
},
|
||||
});
|
||||
return readable;
|
||||
}
|
||||
// ---------------------------------------------------------------
|
||||
// Extra Redis commands (not part of IRedisClient but used by tests
|
||||
// and occasionally by user code that accesses the raw client).
|
||||
// ---------------------------------------------------------------
|
||||
async keys(pattern) {
|
||||
return await this.raw.keys(pattern);
|
||||
}
|
||||
async exists(...keys) {
|
||||
if (keys.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
return await this.raw.exists(keys);
|
||||
}
|
||||
async zadd(key, ...args) {
|
||||
// ioredis: zadd(key, score, member, score, member, ...)
|
||||
const members = [];
|
||||
for (let i = 0; i < args.length; i += 2) {
|
||||
members.push({ score: Number(args[i]), value: String(args[i + 1]) });
|
||||
}
|
||||
return await this.raw.zAdd(key, members);
|
||||
}
|
||||
async zrem(key, ...members) {
|
||||
return await this.raw.zRem(key, members);
|
||||
}
|
||||
async xlen(key) {
|
||||
return await this.raw.xLen(key);
|
||||
}
|
||||
async xrevrange(key, end, start, ...rest) {
|
||||
const opts = {};
|
||||
// ioredis: xrevrange(key, end, start, 'COUNT', n)
|
||||
if (rest[0] === 'COUNT') {
|
||||
opts.COUNT = Number(rest[1]);
|
||||
}
|
||||
const result = await this.raw.xRevRange(key, end, start, opts);
|
||||
// Normalize to ioredis format: [[id, [field, value, …]], …]
|
||||
return result.map((msg) => [
|
||||
msg.id,
|
||||
Object.entries(msg.message).flat(),
|
||||
]);
|
||||
}
|
||||
async sadd(key, ...members) {
|
||||
return await this.raw.sAdd(key, members.map(String));
|
||||
}
|
||||
async scard(key) {
|
||||
return await this.raw.sCard(key);
|
||||
}
|
||||
async lpush(key, ...values) {
|
||||
return await this.raw.lPush(key, values);
|
||||
}
|
||||
async rpop(key) {
|
||||
return await this.raw.rPop(key);
|
||||
}
|
||||
async incr(key) {
|
||||
return await this.raw.incr(key);
|
||||
}
|
||||
async incrby(key, increment) {
|
||||
return await this.raw.incrBy(key, increment);
|
||||
}
|
||||
async flushall() {
|
||||
return await this.raw.flushAll();
|
||||
}
|
||||
}
|
||||
// ---------------------------------------------------------------------------
|
||||
// Transaction / Pipeline wrapper
|
||||
// ---------------------------------------------------------------------------
|
||||
class NodeRedisTransaction {
|
||||
constructor(raw, scripts) {
|
||||
this.raw = raw;
|
||||
this.scripts = scripts;
|
||||
this.transformers = [];
|
||||
}
|
||||
addIdentityTransformer() {
|
||||
this.transformers.push((v) => v);
|
||||
}
|
||||
hgetall(key) {
|
||||
this.raw.hGetAll(key);
|
||||
this.addIdentityTransformer();
|
||||
return this;
|
||||
}
|
||||
hset(key, data) {
|
||||
this.raw.hSet(key, data);
|
||||
this.addIdentityTransformer();
|
||||
return this;
|
||||
}
|
||||
hscan(key, cursor, options) {
|
||||
const opts = {};
|
||||
if ((options === null || options === void 0 ? void 0 : options.COUNT) != null) {
|
||||
opts.COUNT = options.COUNT;
|
||||
}
|
||||
this.raw.hScan(key, String(cursor), opts);
|
||||
// Transform node-redis { cursor, entries: [{field, value}] }
|
||||
// to ioredis [cursor, [field, value, field, value, ...]]
|
||||
this.transformers.push((val) => {
|
||||
if (!val) {
|
||||
return ['0', []];
|
||||
}
|
||||
const flat = [];
|
||||
for (const entry of val.entries || []) {
|
||||
flat.push(entry.field, entry.value);
|
||||
}
|
||||
return [String(val.cursor), flat];
|
||||
});
|
||||
return this;
|
||||
}
|
||||
smembers(key) {
|
||||
this.raw.sMembers(key);
|
||||
this.addIdentityTransformer();
|
||||
return this;
|
||||
}
|
||||
sscan(key, cursor, options) {
|
||||
const opts = {};
|
||||
if ((options === null || options === void 0 ? void 0 : options.COUNT) != null) {
|
||||
opts.COUNT = options.COUNT;
|
||||
}
|
||||
this.raw.sScan(key, String(cursor), opts);
|
||||
// Transform node-redis { cursor, members: [...] }
|
||||
// to ioredis [cursor, [member, member, ...]]
|
||||
this.transformers.push((val) => {
|
||||
if (!val) {
|
||||
return ['0', []];
|
||||
}
|
||||
return [String(val.cursor), val.members || []];
|
||||
});
|
||||
return this;
|
||||
}
|
||||
zrange(key, start, end) {
|
||||
this.raw.zRange(key, start, end);
|
||||
this.addIdentityTransformer();
|
||||
return this;
|
||||
}
|
||||
lrange(key, start, end) {
|
||||
this.raw.lRange(key, start, end);
|
||||
this.addIdentityTransformer();
|
||||
return this;
|
||||
}
|
||||
llen(key) {
|
||||
this.raw.lLen(key);
|
||||
this.addIdentityTransformer();
|
||||
return this;
|
||||
}
|
||||
del(...keys) {
|
||||
if (keys.length > 0) {
|
||||
this.raw.del(keys);
|
||||
this.addIdentityTransformer();
|
||||
}
|
||||
return this;
|
||||
}
|
||||
runCommand(name, args) {
|
||||
const script = this.scripts.get(name);
|
||||
if (!script) {
|
||||
throw new Error(`BullMQ: unknown command "${name}" in transaction`);
|
||||
}
|
||||
const commandArgs = normalizeScriptArgs(args);
|
||||
const { sha, lua, numberOfKeys } = script;
|
||||
const cmdKeys = commandArgs.slice(0, numberOfKeys).map(String);
|
||||
const argv = commandArgs.slice(numberOfKeys).map((a) => {
|
||||
if (Buffer.isBuffer(a)) {
|
||||
return a;
|
||||
}
|
||||
if (a === undefined || a === null) {
|
||||
return '';
|
||||
}
|
||||
return String(a);
|
||||
});
|
||||
// Use evalSha in pipeline; NOSCRIPT handling happens at exec()-time
|
||||
// node-redis multi supports evalSha
|
||||
this.raw.evalSha(sha, { keys: cmdKeys, arguments: argv });
|
||||
this.addIdentityTransformer();
|
||||
return this;
|
||||
}
|
||||
async exec() {
|
||||
const results = await this.raw.exec();
|
||||
if (!results) {
|
||||
return null;
|
||||
}
|
||||
// node-redis multi.exec() returns values directly (or throws on error).
|
||||
// Normalize to ioredis format: [Error | null, value][]
|
||||
// Apply per-command transformers for format differences (hscan, sscan).
|
||||
return results.map((result, i) => {
|
||||
if (result instanceof Error) {
|
||||
return [result, null];
|
||||
}
|
||||
const transformer = this.transformers[i];
|
||||
const value = transformer ? transformer(result) : result;
|
||||
return [null, value];
|
||||
});
|
||||
}
|
||||
}
|
||||
82
node_modules/bullmq/dist/esm/classes/queue-base.d.ts
generated
vendored
Normal file
82
node_modules/bullmq/dist/esm/classes/queue-base.d.ts
generated
vendored
Normal file
@@ -0,0 +1,82 @@
|
||||
import { EventEmitter } from 'events';
|
||||
import { BackendFactory, IQueueBackend, MinimalQueue, QueueBaseOptions, Span } from '../interfaces';
|
||||
import { Job } from './job';
|
||||
import { KeysMap } from './queue-keys';
|
||||
import { SpanKind } from '../enums';
|
||||
/**
|
||||
* Base class for all classes that need to interact with queues.
|
||||
* This class is normally not used directly, but extended by the other classes.
|
||||
*
|
||||
*/
|
||||
export declare class QueueBase<B extends IQueueBackend = IQueueBackend> extends EventEmitter implements MinimalQueue {
|
||||
readonly name: string;
|
||||
opts: QueueBaseOptions;
|
||||
toKey: (type: string) => string;
|
||||
keys: KeysMap;
|
||||
closing: Promise<void> | undefined;
|
||||
protected closed: boolean;
|
||||
protected hasBlockingConnection: boolean;
|
||||
backend: B;
|
||||
protected readonly backendFactory: BackendFactory<B>;
|
||||
readonly qualifiedName: string;
|
||||
/**
|
||||
*
|
||||
* @param name - The name of the queue.
|
||||
* @param opts - Options for the queue.
|
||||
* @param backendFactory - Factory used to build the {@link IQueueBackend}.
|
||||
* Defaults to the Redis backend; inject a different factory to use another
|
||||
* datastore or a test mock.
|
||||
*/
|
||||
constructor(name: string, opts?: QueueBaseOptions, backendFactory?: BackendFactory<B>, hasBlockingConnection?: boolean);
|
||||
/**
|
||||
* Resolves once the underlying backend (and its connection) is ready.
|
||||
*/
|
||||
waitUntilReady(): Promise<void>;
|
||||
/**
|
||||
* Returns the datastore backend that powers this instance.
|
||||
*
|
||||
* The backend owns its connection(s) and exposes every datastore-agnostic
|
||||
* operation through {@link IQueueBackend}. Datastore-specific escape hatches
|
||||
* (e.g. the raw Redis client) live on the concrete backend implementation,
|
||||
* and are exposed here when the class is parameterized on that concrete
|
||||
* backend type (the default for the built-in, Redis-backed classes).
|
||||
*/
|
||||
getBackend(): B;
|
||||
protected createBackend(): void;
|
||||
/**
|
||||
* Helper to easily extend Job class calls.
|
||||
*/
|
||||
protected get Job(): typeof Job;
|
||||
/**
|
||||
* Emits an event. Normally used by subclasses to emit events.
|
||||
*
|
||||
* @param event - The emitted event.
|
||||
* @param args - The arguments to pass to the event listeners.
|
||||
* @returns True if the event had listeners, false otherwise.
|
||||
*/
|
||||
emit(event: string | symbol, ...args: any[]): boolean;
|
||||
protected base64Name(): string;
|
||||
protected clientName(suffix?: string): string;
|
||||
/**
|
||||
*
|
||||
* Closes the connection and returns a promise that resolves when the connection is closed.
|
||||
*/
|
||||
close(): Promise<void>;
|
||||
/**
|
||||
*
|
||||
* Force disconnects a connection.
|
||||
*/
|
||||
disconnect(): Promise<void>;
|
||||
protected checkConnectionError<T>(fn: () => Promise<T>, delayInMs?: number): Promise<T | undefined>;
|
||||
/**
|
||||
* Wraps the code with telemetry and provides a span for configuration.
|
||||
*
|
||||
* @param spanKind - kind of the span: Producer, Consumer, Internal
|
||||
* @param operation - operation name (such as add, process, etc)
|
||||
* @param destination - destination name (normally the queue name)
|
||||
* @param callback - code to wrap with telemetry
|
||||
* @param srcPropagationMetadata - The source propagation metadata for telemetry context.
|
||||
* @returns The result of the callback function.
|
||||
*/
|
||||
trace<T>(spanKind: SpanKind, operation: string, destination: string, callback: (span?: Span, dstPropagationMetadata?: string) => Promise<T> | T, srcPropagationMetadata?: string): Promise<T | Promise<T>>;
|
||||
}
|
||||
152
node_modules/bullmq/dist/esm/classes/queue-base.js
generated
vendored
Normal file
152
node_modules/bullmq/dist/esm/classes/queue-base.js
generated
vendored
Normal file
@@ -0,0 +1,152 @@
|
||||
import { EventEmitter } from 'events';
|
||||
import { delay, DELAY_TIME_5, isNotConnectionError, trace } from '../utils';
|
||||
import { getDefaultBackendFactory } from '../utils/create-backend';
|
||||
import { Job } from './job';
|
||||
/**
|
||||
* Base class for all classes that need to interact with queues.
|
||||
* This class is normally not used directly, but extended by the other classes.
|
||||
*
|
||||
*/
|
||||
export class QueueBase extends EventEmitter {
|
||||
/**
|
||||
*
|
||||
* @param name - The name of the queue.
|
||||
* @param opts - Options for the queue.
|
||||
* @param backendFactory - Factory used to build the {@link IQueueBackend}.
|
||||
* Defaults to the Redis backend; inject a different factory to use another
|
||||
* datastore or a test mock.
|
||||
*/
|
||||
constructor(name, opts = { connection: {} }, backendFactory = getDefaultBackendFactory(), hasBlockingConnection = false) {
|
||||
super();
|
||||
this.name = name;
|
||||
this.opts = opts;
|
||||
this.closed = false;
|
||||
this.hasBlockingConnection = false;
|
||||
this.backendFactory = backendFactory;
|
||||
this.hasBlockingConnection = hasBlockingConnection;
|
||||
this.opts = Object.assign({}, opts);
|
||||
if (!name) {
|
||||
throw new Error('Queue name must be provided');
|
||||
}
|
||||
if (name.includes(':')) {
|
||||
throw new Error('Queue name cannot contain :');
|
||||
}
|
||||
this.createBackend();
|
||||
// Queue identity and key building are owned by the backend (a datastore
|
||||
// concern). The Redis backend encodes the key `prefix`
|
||||
// (`"<prefix>:<queue>"`); other backends format their own identity.
|
||||
this.qualifiedName = this.backend.qualifiedName;
|
||||
this.keys = this.backend.keys;
|
||||
this.toKey = (type) => this.backend.toKey(type);
|
||||
this.backend.on('error', (error) => this.emit('error', error));
|
||||
this.backend.on('close', () => {
|
||||
if (!this.closing) {
|
||||
this.emit('ioredis:close');
|
||||
}
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Resolves once the underlying backend (and its connection) is ready.
|
||||
*/
|
||||
waitUntilReady() {
|
||||
return this.backend.waitUntilReady();
|
||||
}
|
||||
/**
|
||||
* Returns the datastore backend that powers this instance.
|
||||
*
|
||||
* The backend owns its connection(s) and exposes every datastore-agnostic
|
||||
* operation through {@link IQueueBackend}. Datastore-specific escape hatches
|
||||
* (e.g. the raw Redis client) live on the concrete backend implementation,
|
||||
* and are exposed here when the class is parameterized on that concrete
|
||||
* backend type (the default for the built-in, Redis-backed classes).
|
||||
*/
|
||||
getBackend() {
|
||||
return this.backend;
|
||||
}
|
||||
createBackend() {
|
||||
this.backend = this.backendFactory(this.name, this.opts, {
|
||||
blocking: this.hasBlockingConnection,
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Helper to easily extend Job class calls.
|
||||
*/
|
||||
get Job() {
|
||||
return Job;
|
||||
}
|
||||
/**
|
||||
* Emits an event. Normally used by subclasses to emit events.
|
||||
*
|
||||
* @param event - The emitted event.
|
||||
* @param args - The arguments to pass to the event listeners.
|
||||
* @returns True if the event had listeners, false otherwise.
|
||||
*/
|
||||
emit(event, ...args) {
|
||||
try {
|
||||
return super.emit(event, ...args);
|
||||
}
|
||||
catch (err) {
|
||||
try {
|
||||
return super.emit('error', err);
|
||||
}
|
||||
catch (err) {
|
||||
// We give up if the error event also throws an exception.
|
||||
console.error(err);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
base64Name() {
|
||||
return Buffer.from(this.name).toString('base64');
|
||||
}
|
||||
clientName(suffix = '') {
|
||||
return this.backend.clientName(suffix);
|
||||
}
|
||||
/**
|
||||
*
|
||||
* Closes the connection and returns a promise that resolves when the connection is closed.
|
||||
*/
|
||||
async close() {
|
||||
if (!this.closing) {
|
||||
this.closing = this.backend.close();
|
||||
}
|
||||
await this.closing;
|
||||
this.closed = true;
|
||||
}
|
||||
/**
|
||||
*
|
||||
* Force disconnects a connection.
|
||||
*/
|
||||
disconnect() {
|
||||
return this.backend.disconnect();
|
||||
}
|
||||
async checkConnectionError(fn, delayInMs = DELAY_TIME_5) {
|
||||
try {
|
||||
return await fn();
|
||||
}
|
||||
catch (error) {
|
||||
if (isNotConnectionError(error)) {
|
||||
this.emit('error', error);
|
||||
}
|
||||
if (!this.closing && delayInMs) {
|
||||
await delay(delayInMs);
|
||||
}
|
||||
else {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Wraps the code with telemetry and provides a span for configuration.
|
||||
*
|
||||
* @param spanKind - kind of the span: Producer, Consumer, Internal
|
||||
* @param operation - operation name (such as add, process, etc)
|
||||
* @param destination - destination name (normally the queue name)
|
||||
* @param callback - code to wrap with telemetry
|
||||
* @param srcPropagationMetadata - The source propagation metadata for telemetry context.
|
||||
* @returns The result of the callback function.
|
||||
*/
|
||||
trace(spanKind, operation, destination, callback, srcPropagationMetadata) {
|
||||
return trace(this.opts.telemetry, spanKind, this.name, operation, destination, callback, srcPropagationMetadata);
|
||||
}
|
||||
}
|
||||
20
node_modules/bullmq/dist/esm/classes/queue-events-producer.d.ts
generated
vendored
Normal file
20
node_modules/bullmq/dist/esm/classes/queue-events-producer.d.ts
generated
vendored
Normal file
@@ -0,0 +1,20 @@
|
||||
import { BackendFactory, QueueEventsProducerOptions } from '../interfaces';
|
||||
import { QueueBase } from './queue-base';
|
||||
/**
|
||||
* The QueueEventsProducer class is used for publishing custom events.
|
||||
*/
|
||||
export declare class QueueEventsProducer extends QueueBase {
|
||||
constructor(name: string, opts?: QueueEventsProducerOptions, backendFactory?: BackendFactory);
|
||||
/**
|
||||
* Publish custom event to be processed in QueueEvents.
|
||||
* @param argsObj - Event payload
|
||||
* @param maxEvents - Max quantity of events to be saved
|
||||
*/
|
||||
publishEvent<T extends {
|
||||
eventName: string;
|
||||
}>(argsObj: T, maxEvents?: number): Promise<void>;
|
||||
/**
|
||||
* Closes the connection and returns a promise that resolves when the connection is closed.
|
||||
*/
|
||||
close(): Promise<void>;
|
||||
}
|
||||
32
node_modules/bullmq/dist/esm/classes/queue-events-producer.js
generated
vendored
Normal file
32
node_modules/bullmq/dist/esm/classes/queue-events-producer.js
generated
vendored
Normal file
@@ -0,0 +1,32 @@
|
||||
import { __rest } from "tslib";
|
||||
import { QueueBase } from './queue-base';
|
||||
/**
|
||||
* The QueueEventsProducer class is used for publishing custom events.
|
||||
*/
|
||||
export class QueueEventsProducer extends QueueBase {
|
||||
constructor(name, opts = {
|
||||
connection: {},
|
||||
}, backendFactory) {
|
||||
super(name, Object.assign({ blockingConnection: false }, opts), backendFactory);
|
||||
this.opts = opts;
|
||||
}
|
||||
/**
|
||||
* Publish custom event to be processed in QueueEvents.
|
||||
* @param argsObj - Event payload
|
||||
* @param maxEvents - Max quantity of events to be saved
|
||||
*/
|
||||
async publishEvent(argsObj, maxEvents = 1000) {
|
||||
const { eventName } = argsObj, restArgs = __rest(argsObj, ["eventName"]);
|
||||
const fields = Object.assign({ event: eventName }, restArgs);
|
||||
await this.backend.publishEvent(fields, maxEvents);
|
||||
}
|
||||
/**
|
||||
* Closes the connection and returns a promise that resolves when the connection is closed.
|
||||
*/
|
||||
async close() {
|
||||
if (!this.closing) {
|
||||
this.closing = this.backend.close();
|
||||
}
|
||||
await this.closing;
|
||||
}
|
||||
}
|
||||
277
node_modules/bullmq/dist/esm/classes/queue-events.d.ts
generated
vendored
Normal file
277
node_modules/bullmq/dist/esm/classes/queue-events.d.ts
generated
vendored
Normal file
@@ -0,0 +1,277 @@
|
||||
import { JobProgress } from '../types';
|
||||
import { BackendFactory, IoredisListener, QueueEventsOptions } from '../interfaces';
|
||||
import { QueueBase } from './queue-base';
|
||||
export interface QueueEventsListener<ReturnType = any> extends IoredisListener {
|
||||
/**
|
||||
* Listen to 'active' event.
|
||||
*
|
||||
* This event is triggered when a job enters the 'active' state, meaning it is being processed.
|
||||
*
|
||||
* @param args - An object containing details about the job that became active.
|
||||
* - `jobId`: The unique identifier of the job that entered the active state.
|
||||
* - `prev`: The previous state of the job before it became active (e.g., 'waiting'), if applicable.
|
||||
*
|
||||
* @param id - The identifier of the event.
|
||||
*/
|
||||
active: (args: {
|
||||
jobId: string;
|
||||
prev?: string;
|
||||
}, id: string) => void;
|
||||
/**
|
||||
* Listen to 'added' event.
|
||||
*
|
||||
* This event is triggered when a job is created and added to the queue.
|
||||
*
|
||||
* @param args - An object containing details about the newly added job.
|
||||
* - `jobId` - The unique identifier of the job that was added.
|
||||
* - `name` - The name of the job, typically indicating its type or purpose.
|
||||
* @param id - The identifier of the event.
|
||||
*/
|
||||
added: (args: {
|
||||
jobId: string;
|
||||
name: string;
|
||||
}, id: string) => void;
|
||||
/**
|
||||
* Listen to 'cleaned' event.
|
||||
*
|
||||
* This event is triggered when jobs are cleaned (e.g., removed) from the queue, typically via a cleanup method.
|
||||
*
|
||||
* @param args - An object containing the count of cleaned jobs.
|
||||
* - `count` - The number of jobs that were cleaned, represented as a string due to Redis serialization.
|
||||
* @param id - The identifier of the event.
|
||||
*/
|
||||
cleaned: (args: {
|
||||
count: string;
|
||||
}, id: string) => void;
|
||||
/**
|
||||
* Listen to 'completed' event.
|
||||
*
|
||||
* This event is triggered when a job has successfully completed its execution.
|
||||
*
|
||||
* @param args - An object containing details about the completed job.
|
||||
* - `jobId` - The unique identifier of the job that completed.
|
||||
* - `returnvalue` - The value returned by the job's processor, deserialized from JSON.
|
||||
* - `prev` - The previous state of the job before completion (e.g., 'active'), if applicable.
|
||||
* @param id - The identifier of the event.
|
||||
*/
|
||||
completed: (args: {
|
||||
jobId: string;
|
||||
returnvalue: ReturnType;
|
||||
prev?: string;
|
||||
}, id: string) => void;
|
||||
/**
|
||||
* Listen to 'deduplicated' event.
|
||||
*
|
||||
* This event is triggered when a job is not added to the queue because a job with the same deduplicationId
|
||||
* already exists.
|
||||
*
|
||||
* @param args - An object containing details about the deduplicated job.
|
||||
* - `jobId` - The unique identifier of the job that was attempted to be added.
|
||||
* - `deduplicationId` - The deduplication identifier that caused the job to be deduplicated.
|
||||
* - `deduplicatedJobId` - The unique identifier of the existing job that caused the deduplication.
|
||||
* @param id - The identifier of the event.
|
||||
*/
|
||||
deduplicated: (args: {
|
||||
jobId: string;
|
||||
deduplicationId: string;
|
||||
deduplicatedJobId: string;
|
||||
}, id: string) => void;
|
||||
/**
|
||||
* Listen to 'delayed' event.
|
||||
*
|
||||
* This event is triggered when a job is scheduled with a delay before it becomes active.
|
||||
*
|
||||
* @param args - An object containing details about the delayed job.
|
||||
* - `jobId` - The unique identifier of the job that was delayed.
|
||||
* - `delay` - The timestamp at which the job will become active, in milliseconds
|
||||
* elapsed since January 1, 1970 UTC.
|
||||
* @param id - The identifier of the event.
|
||||
*/
|
||||
delayed: (args: {
|
||||
jobId: string;
|
||||
delay: number;
|
||||
}, id: string) => void;
|
||||
/**
|
||||
* Listen to 'drained' event.
|
||||
*
|
||||
* This event is triggered when the queue has drained its waiting list, meaning there are no jobs
|
||||
* in the 'waiting' state.
|
||||
* Note that there could still be delayed jobs waiting their timers to expire
|
||||
* and this event will still be triggered as long as the waiting list has emptied.
|
||||
*
|
||||
* @param id - The identifier of the event.
|
||||
*/
|
||||
drained: (id: string) => void;
|
||||
/**
|
||||
* Listen to 'duplicated' event.
|
||||
*
|
||||
* This event is triggered when a job is not created because a job with the same identifier already exists.
|
||||
*
|
||||
* @param args - An object containing the job identifier.
|
||||
* - `jobId` - The unique identifier of the job that was attempted to be added.
|
||||
* @param id - The identifier of the event.
|
||||
*/
|
||||
duplicated: (args: {
|
||||
jobId: string;
|
||||
}, id: string) => void;
|
||||
/**
|
||||
* Listen to 'error' event.
|
||||
*
|
||||
* This event is triggered when an error in the Redis backend is thrown.
|
||||
*
|
||||
* @param args - The error that was thrown.
|
||||
*/
|
||||
error: (args: Error) => void;
|
||||
/**
|
||||
* Listen to 'failed' event.
|
||||
*
|
||||
* This event is triggered when a job fails by throwing an exception during execution.
|
||||
*
|
||||
* @param args - An object containing details about the failed job.
|
||||
* - `jobId` - The unique identifier of the job that failed.
|
||||
* - `failedReason` - The reason or message describing why the job failed.
|
||||
* - `prev` - The previous state of the job before failure (e.g., 'active'), if applicable.
|
||||
* @param id - The identifier of the event.
|
||||
*/
|
||||
failed: (args: {
|
||||
jobId: string;
|
||||
failedReason: string;
|
||||
prev?: string;
|
||||
}, id: string) => void;
|
||||
/**
|
||||
* Listen to 'paused' event.
|
||||
*
|
||||
* This event is triggered when the queue is paused, halting the processing of new jobs.
|
||||
*
|
||||
* @param args - An empty object (no additional data provided).
|
||||
* @param id - The identifier of the event.
|
||||
*/
|
||||
paused: (args: object, id: string) => void;
|
||||
/**
|
||||
* Listen to 'progress' event.
|
||||
*
|
||||
* This event is triggered when a job updates its progress via the `Job#updateProgress()` method, allowing
|
||||
* progress or custom data to be communicated externally.
|
||||
*
|
||||
* @param args - An object containing the job identifier and progress data.
|
||||
* - `jobId` - The unique identifier of the job reporting progress.
|
||||
* - `data` - The progress data, which can be a number (e.g., percentage) or an object with custom data.
|
||||
* @param id - The identifier of the event.
|
||||
*/
|
||||
progress: (args: {
|
||||
jobId: string;
|
||||
data: JobProgress;
|
||||
}, id: string) => void;
|
||||
/**
|
||||
* Listen to 'removed' event.
|
||||
*
|
||||
* This event is triggered when a job is manually removed from the queue.
|
||||
*
|
||||
* @param args - An object containing details about the removed job.
|
||||
* - `jobId` - The unique identifier of the job that was removed.
|
||||
* - `prev` - The previous state of the job before removal (e.g., 'active' or 'waiting').
|
||||
* @param id - The identifier of the event.
|
||||
*/
|
||||
removed: (args: {
|
||||
jobId: string;
|
||||
prev: string;
|
||||
}, id: string) => void;
|
||||
/**
|
||||
* Listen to 'resumed' event.
|
||||
*
|
||||
* This event is triggered when the queue is resumed, allowing job processing to continue.
|
||||
*
|
||||
* @param args - An empty object (no additional data provided).
|
||||
* @param id - The identifier of the event.
|
||||
*/
|
||||
resumed: (args: object, id: string) => void;
|
||||
/**
|
||||
* Listen to 'retries-exhausted' event.
|
||||
*
|
||||
* This event is triggered when a job has exhausted its maximum retry attempts after repeated failures.
|
||||
*
|
||||
* @param args - An object containing details about the job that exhausted retries.
|
||||
* - `jobId` - The unique identifier of the job that exhausted its retries.
|
||||
* - `attemptsMade` - The number of retry attempts made, represented as a string
|
||||
* (due to Redis serialization).
|
||||
* @param id - The identifier of the event.
|
||||
*/
|
||||
'retries-exhausted': (args: {
|
||||
jobId: string;
|
||||
attemptsMade: string;
|
||||
}, id: string) => void;
|
||||
/**
|
||||
* Listen to 'stalled' event.
|
||||
*
|
||||
* This event is triggered when a job moves from 'active' back to 'waiting' or
|
||||
* 'failed' because the processor could not renew its lock, indicating a
|
||||
* potential processing issue.
|
||||
*
|
||||
* @param args - An object containing the job identifier.
|
||||
* - `jobId` - The unique identifier of the job that stalled.
|
||||
* @param id - The identifier of the event.
|
||||
*/
|
||||
stalled: (args: {
|
||||
jobId: string;
|
||||
}, id: string) => void;
|
||||
/**
|
||||
* Listen to 'waiting' event.
|
||||
*
|
||||
* This event is triggered when a job enters the 'waiting' state, indicating it is queued and
|
||||
* awaiting processing.
|
||||
*
|
||||
* @param args - An object containing details about the job in the waiting state.
|
||||
* - `jobId` - The unique identifier of the job that is waiting.
|
||||
* - `prev` - The previous state of the job before entering 'waiting' (e.g., 'stalled'),
|
||||
* if applicable.
|
||||
* @param id - The identifier of the event.
|
||||
*/
|
||||
waiting: (args: {
|
||||
jobId: string;
|
||||
prev?: string;
|
||||
}, id: string) => void;
|
||||
/**
|
||||
* Listen to 'waiting-children' event.
|
||||
*
|
||||
* This event is triggered when a job enters the 'waiting-children' state, indicating it is
|
||||
* waiting for its child jobs to complete.
|
||||
*
|
||||
* @param args - An object containing the job identifier.
|
||||
* - `jobId` - The unique identifier of the job waiting for its children.
|
||||
* @param id - The identifier of the event.
|
||||
*/
|
||||
'waiting-children': (args: {
|
||||
jobId: string;
|
||||
}, id: string) => void;
|
||||
}
|
||||
type CustomParameters<T> = T extends (...args: infer Args) => void ? Args : never;
|
||||
type KeyOf<T extends object> = Extract<keyof T, string>;
|
||||
/**
|
||||
* The QueueEvents class is used for listening to the global events
|
||||
* emitted by a given queue.
|
||||
*
|
||||
* This class requires a dedicated redis connection.
|
||||
*
|
||||
*/
|
||||
export declare class QueueEvents<ReturnType = any> extends QueueBase {
|
||||
private running;
|
||||
private blocking;
|
||||
constructor(name: string, { connection, autorun, ...opts }?: QueueEventsOptions, backendFactory?: BackendFactory);
|
||||
emit<QEL extends QueueEventsListener<ReturnType> = QueueEventsListener<ReturnType>, U extends KeyOf<QEL> = KeyOf<QEL>>(event: U, ...args: CustomParameters<QEL[U]>): boolean;
|
||||
off<QEL extends QueueEventsListener<ReturnType> = QueueEventsListener<ReturnType>, U extends KeyOf<QEL> = KeyOf<QEL>>(eventName: U, listener: QEL[U]): this;
|
||||
on<QEL extends QueueEventsListener<ReturnType> = QueueEventsListener<ReturnType>, U extends KeyOf<QEL> = KeyOf<QEL>>(event: U, listener: QEL[U]): this;
|
||||
once<QEL extends QueueEventsListener<ReturnType> = QueueEventsListener<ReturnType>, U extends KeyOf<QEL> = KeyOf<QEL>>(event: U, listener: QEL[U]): this;
|
||||
/**
|
||||
* Manually starts running the event consumming loop. This shall be used if you do not
|
||||
* use the default "autorun" option on the constructor.
|
||||
*/
|
||||
run(): Promise<void>;
|
||||
private consumeEvents;
|
||||
/**
|
||||
* Stops consuming events and close the underlying Redis connection if necessary.
|
||||
*
|
||||
* @returns
|
||||
*/
|
||||
close(): Promise<void>;
|
||||
}
|
||||
export {};
|
||||
130
node_modules/bullmq/dist/esm/classes/queue-events.js
generated
vendored
Normal file
130
node_modules/bullmq/dist/esm/classes/queue-events.js
generated
vendored
Normal file
@@ -0,0 +1,130 @@
|
||||
import { __rest } from "tslib";
|
||||
import { array2obj, isRedisInstance, QUEUE_EVENT_SUFFIX } from '../utils';
|
||||
import { QueueBase } from './queue-base';
|
||||
import { createIORedisClient, isIRedisClient } from './ioredis-client';
|
||||
/**
|
||||
* The QueueEvents class is used for listening to the global events
|
||||
* emitted by a given queue.
|
||||
*
|
||||
* This class requires a dedicated redis connection.
|
||||
*
|
||||
*/
|
||||
export class QueueEvents extends QueueBase {
|
||||
constructor(name, _a = {
|
||||
connection: {},
|
||||
}, backendFactory) {
|
||||
var { connection, autorun = true } = _a, opts = __rest(_a, ["connection", "autorun"]);
|
||||
super(name, Object.assign(Object.assign({}, opts), { connection: isRedisInstance(connection)
|
||||
? (isIRedisClient(connection)
|
||||
? connection
|
||||
: createIORedisClient(connection)).duplicate()
|
||||
: connection }), backendFactory, true);
|
||||
this.running = false;
|
||||
this.blocking = false;
|
||||
this.opts = Object.assign({
|
||||
blockingTimeout: 10000,
|
||||
}, this.opts);
|
||||
if (autorun) {
|
||||
this.run().catch(error => this.emit('error', error));
|
||||
}
|
||||
}
|
||||
emit(event, ...args) {
|
||||
return super.emit(event, ...args);
|
||||
}
|
||||
off(eventName, listener) {
|
||||
super.off(eventName, listener);
|
||||
return this;
|
||||
}
|
||||
on(event, listener) {
|
||||
super.on(event, listener);
|
||||
return this;
|
||||
}
|
||||
once(event, listener) {
|
||||
super.once(event, listener);
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* Manually starts running the event consumming loop. This shall be used if you do not
|
||||
* use the default "autorun" option on the constructor.
|
||||
*/
|
||||
async run() {
|
||||
if (!this.running) {
|
||||
try {
|
||||
this.running = true;
|
||||
// TODO: Planned for deprecation as it really has no use case
|
||||
await this.backend.setName(this.clientName(QUEUE_EVENT_SUFFIX));
|
||||
await this.consumeEvents();
|
||||
}
|
||||
catch (error) {
|
||||
this.running = false;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
else {
|
||||
throw new Error('Queue Events is already running.');
|
||||
}
|
||||
}
|
||||
async consumeEvents() {
|
||||
const opts = this.opts;
|
||||
let id = opts.lastEventId || '$';
|
||||
while (!this.closing) {
|
||||
this.blocking = true;
|
||||
// Cast to actual return type, see: https://github.com/DefinitelyTyped/DefinitelyTyped/issues/44301
|
||||
const data = await this.checkConnectionError(() => this.backend.readEvents(id, opts.blockingTimeout));
|
||||
this.blocking = false;
|
||||
if (data) {
|
||||
const stream = data[0];
|
||||
const events = stream[1];
|
||||
for (let i = 0; i < events.length; i++) {
|
||||
id = events[i][0];
|
||||
const args = array2obj(events[i][1]);
|
||||
//
|
||||
// TODO: we may need to have a separate stream for progress data
|
||||
// to avoid this hack.
|
||||
switch (args.event) {
|
||||
case 'progress':
|
||||
args.data = JSON.parse(args.data);
|
||||
break;
|
||||
case 'completed':
|
||||
args.returnvalue = JSON.parse(args.returnvalue);
|
||||
break;
|
||||
case 'delayed':
|
||||
args.delay = Number(args.delay);
|
||||
break;
|
||||
}
|
||||
const { event } = args, restArgs = __rest(args, ["event"]);
|
||||
if (event === 'drained') {
|
||||
this.emit(event, id);
|
||||
}
|
||||
else {
|
||||
this.emit(event, restArgs, id);
|
||||
if (restArgs.jobId) {
|
||||
this.emit(`${event}:${restArgs.jobId}`, restArgs, id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Stops consuming events and close the underlying Redis connection if necessary.
|
||||
*
|
||||
* @returns
|
||||
*/
|
||||
async close() {
|
||||
if (!this.closing) {
|
||||
this.closing = (async () => {
|
||||
try {
|
||||
// Force a disconnect first to interrupt the blocking XREAD, then
|
||||
// close the underlying connection.
|
||||
await this.backend.disconnect();
|
||||
await this.backend.close();
|
||||
}
|
||||
finally {
|
||||
this.closed = true;
|
||||
}
|
||||
})();
|
||||
}
|
||||
return this.closing;
|
||||
}
|
||||
}
|
||||
268
node_modules/bullmq/dist/esm/classes/queue-getters.d.ts
generated
vendored
Normal file
268
node_modules/bullmq/dist/esm/classes/queue-getters.d.ts
generated
vendored
Normal file
@@ -0,0 +1,268 @@
|
||||
import { QueueBase } from './queue-base';
|
||||
import { Job } from './job';
|
||||
import { JobState, JobType } from '../types';
|
||||
import { IQueueBackend, JobJson, Metrics, QueueMeta } from '../interfaces';
|
||||
/**
|
||||
* Provides different getters for different aspects of a queue.
|
||||
*/
|
||||
export declare class QueueGetters<JobBase extends Job = Job, B extends IQueueBackend = IQueueBackend> extends QueueBase<B> {
|
||||
getJob(jobId: string): Promise<JobBase | undefined>;
|
||||
private commandByType;
|
||||
private sanitizeJobTypes;
|
||||
/**
|
||||
Returns the number of jobs waiting to be processed. This includes jobs that are
|
||||
"waiting" or "delayed" or "prioritized" or "waiting-children".
|
||||
*/
|
||||
count(): Promise<number>;
|
||||
/**
|
||||
* Returns the time to live for a rate limited key in milliseconds.
|
||||
* @param maxJobs - max jobs to be considered in rate limit state. If not passed
|
||||
* it will return the remaining ttl without considering if max jobs is exceeded.
|
||||
* @returns -2 if the key does not exist.
|
||||
* -1 if the key exists but has no associated expire.
|
||||
* @see {@link https://redis.io/commands/pttl/}
|
||||
*/
|
||||
getRateLimitTtl(maxJobs?: number): Promise<number>;
|
||||
/**
|
||||
* Get jobId that starts debounced state.
|
||||
* @deprecated use getDeduplicationJobId method
|
||||
*
|
||||
* @param id - debounce identifier
|
||||
*/
|
||||
getDebounceJobId(id: string): Promise<string | null>;
|
||||
/**
|
||||
* Get jobId from deduplicated state.
|
||||
*
|
||||
* @param id - deduplication identifier
|
||||
*/
|
||||
getDeduplicationJobId(id: string): Promise<string | null>;
|
||||
/**
|
||||
* Get global concurrency value.
|
||||
* Returns null in case no value is set.
|
||||
*/
|
||||
getGlobalConcurrency(): Promise<number | null>;
|
||||
/**
|
||||
* Get global rate limit values.
|
||||
* Returns null in case no value is set.
|
||||
*/
|
||||
getGlobalRateLimit(): Promise<{
|
||||
max: number;
|
||||
duration: number;
|
||||
} | null>;
|
||||
/**
|
||||
* Job counts by type
|
||||
*
|
||||
* Queue#getJobCountByTypes('completed') =\> completed count
|
||||
* Queue#getJobCountByTypes('completed', 'failed') =\> completed + failed count
|
||||
* Queue#getJobCountByTypes('completed', 'waiting', 'failed') =\> completed + waiting + failed count
|
||||
*/
|
||||
getJobCountByTypes(...types: JobType[]): Promise<number>;
|
||||
/**
|
||||
* Returns the job counts for each type specified or every list/set in the queue by default.
|
||||
* @param types - the types of jobs to count. If not specified, it will return the counts for all types.
|
||||
* @returns An object, key (type) and value (count)
|
||||
*/
|
||||
getJobCounts(...types: JobType[]): Promise<{
|
||||
[index: string]: number;
|
||||
}>;
|
||||
/**
|
||||
* Records job counts as gauge metrics for telemetry purposes.
|
||||
* Each job state count is recorded with the queue name and state as attributes.
|
||||
* @param types - the types of jobs to count. If not specified, it will return the counts for all types.
|
||||
* @returns An object, key (type) and value (count)
|
||||
*/
|
||||
recordJobCountsMetric(...types: JobType[]): Promise<{
|
||||
[index: string]: number;
|
||||
}>;
|
||||
/**
|
||||
* Get current job state.
|
||||
*
|
||||
* @param jobId - job identifier.
|
||||
* @returns Returns one of these values:
|
||||
* 'completed', 'failed', 'delayed', 'active', 'waiting', 'waiting-children', 'unknown'.
|
||||
*/
|
||||
getJobState(jobId: string): Promise<JobState | 'unknown'>;
|
||||
/**
|
||||
* Get global queue configuration.
|
||||
*
|
||||
* @returns Returns the global queue configuration.
|
||||
*/
|
||||
getMeta(): Promise<QueueMeta>;
|
||||
/**
|
||||
* @returns Returns the number of jobs in completed status.
|
||||
*/
|
||||
getCompletedCount(): Promise<number>;
|
||||
/**
|
||||
* Returns the number of jobs in failed status.
|
||||
*/
|
||||
getFailedCount(): Promise<number>;
|
||||
/**
|
||||
* Returns the number of jobs in delayed status.
|
||||
*/
|
||||
getDelayedCount(): Promise<number>;
|
||||
/**
|
||||
* Returns the number of jobs in active status.
|
||||
*/
|
||||
getActiveCount(): Promise<number>;
|
||||
/**
|
||||
* Returns the number of jobs in prioritized status.
|
||||
*/
|
||||
getPrioritizedCount(): Promise<number>;
|
||||
/**
|
||||
* Returns the number of jobs per priority.
|
||||
*/
|
||||
getCountsPerPriority(priorities: number[]): Promise<{
|
||||
[index: string]: number;
|
||||
}>;
|
||||
/**
|
||||
* Returns the number of jobs in waiting or paused statuses.
|
||||
*/
|
||||
getWaitingCount(): Promise<number>;
|
||||
/**
|
||||
* Returns the number of jobs in waiting-children status.
|
||||
*/
|
||||
getWaitingChildrenCount(): Promise<number>;
|
||||
/**
|
||||
* Returns the jobs that are in the "waiting" status.
|
||||
* @param start - zero based index from where to start returning jobs.
|
||||
* @param end - zero based index where to stop returning jobs.
|
||||
*/
|
||||
getWaiting(start?: number, end?: number): Promise<JobBase[]>;
|
||||
/**
|
||||
* Returns the jobs that are in the "waiting-children" status.
|
||||
* I.E. parent jobs that have at least one child that has not completed yet.
|
||||
* @param start - zero based index from where to start returning jobs.
|
||||
* @param end - zero based index where to stop returning jobs.
|
||||
*/
|
||||
getWaitingChildren(start?: number, end?: number): Promise<JobBase[]>;
|
||||
/**
|
||||
* Returns the jobs that are in the "active" status.
|
||||
* @param start - zero based index from where to start returning jobs.
|
||||
* @param end - zero based index where to stop returning jobs.
|
||||
*/
|
||||
getActive(start?: number, end?: number): Promise<JobBase[]>;
|
||||
/**
|
||||
* Returns the jobs that are in the "delayed" status.
|
||||
* @param start - zero based index from where to start returning jobs.
|
||||
* @param end - zero based index where to stop returning jobs.
|
||||
*/
|
||||
getDelayed(start?: number, end?: number): Promise<JobBase[]>;
|
||||
/**
|
||||
* Returns the jobs that are in the "prioritized" status.
|
||||
* @param start - zero based index from where to start returning jobs.
|
||||
* @param end - zero based index where to stop returning jobs.
|
||||
*/
|
||||
getPrioritized(start?: number, end?: number): Promise<JobBase[]>;
|
||||
/**
|
||||
* Returns the jobs that are in the "completed" status.
|
||||
* @param start - zero based index from where to start returning jobs.
|
||||
* @param end - zero based index where to stop returning jobs.
|
||||
*/
|
||||
getCompleted(start?: number, end?: number): Promise<JobBase[]>;
|
||||
/**
|
||||
* Returns the jobs that are in the "failed" status.
|
||||
* @param start - zero based index from where to start returning jobs.
|
||||
* @param end - zero based index where to stop returning jobs.
|
||||
*/
|
||||
getFailed(start?: number, end?: number): Promise<JobBase[]>;
|
||||
/**
|
||||
* Returns the qualified job ids and the raw job data (if available) of the
|
||||
* children jobs of the given parent job.
|
||||
* It is possible to get either the already processed children, in this case
|
||||
* an array of qualified job ids and their result values will be returned,
|
||||
* or the pending children, in this case an array of qualified job ids will
|
||||
* be returned.
|
||||
* A qualified job id is a string representing the job id in a given queue,
|
||||
* for example: "bull:myqueue:jobid".
|
||||
*
|
||||
* @param parentId - The id of the parent job
|
||||
* @param type - "processed" | "pending"
|
||||
* @param opts - Options for the query.
|
||||
*
|
||||
* @returns an object with the following shape:
|
||||
* `{ items: { id: string, v?: any, err?: string } [], jobs: JobJson[], total: number}`
|
||||
*/
|
||||
getDependencies(parentId: string, type: 'processed' | 'pending', start: number, end: number): Promise<{
|
||||
items: {
|
||||
id: string;
|
||||
v?: any;
|
||||
err?: string;
|
||||
}[];
|
||||
jobs: JobJson[];
|
||||
total: number;
|
||||
}>;
|
||||
getRanges(types: JobType[], start?: number, end?: number, asc?: boolean): Promise<string[]>;
|
||||
/**
|
||||
* Returns the jobs that are on the given statuses (note that JobType is synonym for job status)
|
||||
* @param types - the statuses of the jobs to return.
|
||||
* @param start - zero based index from where to start returning jobs.
|
||||
* @param end - zero based index where to stop returning jobs.
|
||||
* @param asc - if true, the jobs will be returned in ascending order.
|
||||
*/
|
||||
getJobs(types?: JobType[] | JobType, start?: number, end?: number, asc?: boolean): Promise<JobBase[]>;
|
||||
/**
|
||||
* Returns the logs for a given Job.
|
||||
* @param jobId - the id of the job to get the logs for.
|
||||
* @param start - zero based index from where to start returning jobs.
|
||||
* @param end - zero based index where to stop returning jobs.
|
||||
* @param asc - if true, the jobs will be returned in ascending order.
|
||||
*/
|
||||
getJobLogs(jobId: string, start?: number, end?: number, asc?: boolean): Promise<{
|
||||
logs: string[];
|
||||
count: number;
|
||||
}>;
|
||||
private baseGetClients;
|
||||
/**
|
||||
* Get the worker list related to the queue. i.e. all the known
|
||||
* workers that are available to process jobs for this queue.
|
||||
* Note: GCP does not support SETNAME, so this call will not work
|
||||
*
|
||||
* @returns - Returns an array with workers info.
|
||||
*/
|
||||
getWorkers(): Promise<{
|
||||
[index: string]: string;
|
||||
}[]>;
|
||||
/**
|
||||
* Returns the current count of workers for the queue.
|
||||
*
|
||||
* getWorkersCount(): Promise<number>
|
||||
*
|
||||
*/
|
||||
getWorkersCount(): Promise<number>;
|
||||
/**
|
||||
* Get queue events list related to the queue.
|
||||
* Note: GCP does not support SETNAME, so this call will not work
|
||||
*
|
||||
* @deprecated do not use this method, it will be removed in the future.
|
||||
*
|
||||
* @returns - Returns an array with queue events info.
|
||||
*/
|
||||
getQueueEvents(): Promise<{
|
||||
[index: string]: string;
|
||||
}[]>;
|
||||
/**
|
||||
* Get queue metrics related to the queue.
|
||||
*
|
||||
* This method returns the gathered metrics for the queue.
|
||||
* The metrics are represented as an array of job counts
|
||||
* per unit of time (1 minute).
|
||||
*
|
||||
* @param start - Start point of the metrics, where 0
|
||||
* is the newest point to be returned.
|
||||
* @param end - End point of the metrics, where -1 is the
|
||||
* oldest point to be returned.
|
||||
*
|
||||
* @returns - Returns an object with queue metrics.
|
||||
*/
|
||||
getMetrics(type: 'completed' | 'failed', start?: number, end?: number): Promise<Metrics>;
|
||||
private parseClientList;
|
||||
/**
|
||||
* Export the metrics for the queue in the Prometheus format.
|
||||
* Automatically exports all the counts returned by getJobCounts().
|
||||
*
|
||||
* @returns - Returns a string with the metrics in the Prometheus format.
|
||||
*
|
||||
* @see {@link https://prometheus.io/docs/instrumenting/exposition_formats/}
|
||||
**/
|
||||
exportPrometheusMetrics(globalVariables?: Record<string, string>): Promise<string>;
|
||||
}
|
||||
550
node_modules/bullmq/dist/esm/classes/queue-getters.js
generated
vendored
Normal file
550
node_modules/bullmq/dist/esm/classes/queue-getters.js
generated
vendored
Normal file
@@ -0,0 +1,550 @@
|
||||
'use strict';
|
||||
import { __rest } from "tslib";
|
||||
import { QueueBase } from './queue-base';
|
||||
import { clientCommandMessageReg, QUEUE_EVENT_SUFFIX } from '../utils';
|
||||
import { MetricNames, TelemetryAttributes } from '../enums';
|
||||
import { RedisQueueBackend } from './redis-queue-backend';
|
||||
/**
|
||||
* Escape a Prometheus label value per the text exposition format.
|
||||
* https://prometheus.io/docs/instrumenting/exposition_formats/
|
||||
*
|
||||
* Backslashes, double quotes, and newlines must be escaped.
|
||||
*/
|
||||
function escapePrometheusLabelValue(value) {
|
||||
return String(value)
|
||||
.replace(/\\/g, '\\\\')
|
||||
.replace(/"/g, '\\"')
|
||||
.replace(/\n/g, '\\n');
|
||||
}
|
||||
/**
|
||||
* Provides different getters for different aspects of a queue.
|
||||
*/
|
||||
export class QueueGetters extends QueueBase {
|
||||
getJob(jobId) {
|
||||
return this.Job.fromId(this, jobId);
|
||||
}
|
||||
commandByType(types, count, callback) {
|
||||
return types.map((type) => {
|
||||
type = type === 'waiting' ? 'wait' : type; // alias
|
||||
const key = this.toKey(type);
|
||||
switch (type) {
|
||||
case 'completed':
|
||||
case 'failed':
|
||||
case 'delayed':
|
||||
case 'prioritized':
|
||||
case 'repeat':
|
||||
case 'waiting-children':
|
||||
return callback(key, count ? 'zcard' : 'zrange');
|
||||
case 'active':
|
||||
case 'wait':
|
||||
return callback(key, count ? 'llen' : 'lrange');
|
||||
}
|
||||
});
|
||||
}
|
||||
sanitizeJobTypes(types) {
|
||||
const currentTypes = typeof types === 'string' ? [types] : types;
|
||||
if (Array.isArray(currentTypes) && currentTypes.length > 0) {
|
||||
const sanitizedTypes = [...currentTypes];
|
||||
return [...new Set(sanitizedTypes)];
|
||||
}
|
||||
return [
|
||||
'active',
|
||||
'completed',
|
||||
'delayed',
|
||||
'failed',
|
||||
'prioritized',
|
||||
'waiting',
|
||||
'waiting-children',
|
||||
];
|
||||
}
|
||||
/**
|
||||
Returns the number of jobs waiting to be processed. This includes jobs that are
|
||||
"waiting" or "delayed" or "prioritized" or "waiting-children".
|
||||
*/
|
||||
async count() {
|
||||
const count = await this.getJobCountByTypes('waiting', 'delayed', 'prioritized', 'waiting-children');
|
||||
return count;
|
||||
}
|
||||
/**
|
||||
* Returns the time to live for a rate limited key in milliseconds.
|
||||
* @param maxJobs - max jobs to be considered in rate limit state. If not passed
|
||||
* it will return the remaining ttl without considering if max jobs is exceeded.
|
||||
* @returns -2 if the key does not exist.
|
||||
* -1 if the key exists but has no associated expire.
|
||||
* @see {@link https://redis.io/commands/pttl/}
|
||||
*/
|
||||
async getRateLimitTtl(maxJobs) {
|
||||
return this.backend.getRateLimitTtl(maxJobs);
|
||||
}
|
||||
/**
|
||||
* Get jobId that starts debounced state.
|
||||
* @deprecated use getDeduplicationJobId method
|
||||
*
|
||||
* @param id - debounce identifier
|
||||
*/
|
||||
async getDebounceJobId(id) {
|
||||
return this.backend.getDeduplicationJobId(id);
|
||||
}
|
||||
/**
|
||||
* Get jobId from deduplicated state.
|
||||
*
|
||||
* @param id - deduplication identifier
|
||||
*/
|
||||
async getDeduplicationJobId(id) {
|
||||
return this.backend.getDeduplicationJobId(id);
|
||||
}
|
||||
/**
|
||||
* Get global concurrency value.
|
||||
* Returns null in case no value is set.
|
||||
*/
|
||||
async getGlobalConcurrency() {
|
||||
const concurrency = await this.backend.getQueueMetaField('concurrency');
|
||||
if (concurrency) {
|
||||
return Number(concurrency);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
/**
|
||||
* Get global rate limit values.
|
||||
* Returns null in case no value is set.
|
||||
*/
|
||||
async getGlobalRateLimit() {
|
||||
const [max, duration] = await this.backend.getQueueMetaFields([
|
||||
'max',
|
||||
'duration',
|
||||
]);
|
||||
if (max && duration) {
|
||||
return {
|
||||
max: Number(max),
|
||||
duration: Number(duration),
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
/**
|
||||
* Job counts by type
|
||||
*
|
||||
* Queue#getJobCountByTypes('completed') =\> completed count
|
||||
* Queue#getJobCountByTypes('completed', 'failed') =\> completed + failed count
|
||||
* Queue#getJobCountByTypes('completed', 'waiting', 'failed') =\> completed + waiting + failed count
|
||||
*/
|
||||
async getJobCountByTypes(...types) {
|
||||
const result = await this.getJobCounts(...types);
|
||||
return Object.values(result).reduce((sum, count) => sum + count, 0);
|
||||
}
|
||||
/**
|
||||
* Returns the job counts for each type specified or every list/set in the queue by default.
|
||||
* @param types - the types of jobs to count. If not specified, it will return the counts for all types.
|
||||
* @returns An object, key (type) and value (count)
|
||||
*/
|
||||
async getJobCounts(...types) {
|
||||
const currentTypes = this.sanitizeJobTypes(types);
|
||||
const responses = await this.backend.getCounts(currentTypes);
|
||||
const counts = {};
|
||||
responses.forEach((res, index) => {
|
||||
counts[currentTypes[index]] = res || 0;
|
||||
});
|
||||
return counts;
|
||||
}
|
||||
/**
|
||||
* Records job counts as gauge metrics for telemetry purposes.
|
||||
* Each job state count is recorded with the queue name and state as attributes.
|
||||
* @param types - the types of jobs to count. If not specified, it will return the counts for all types.
|
||||
* @returns An object, key (type) and value (count)
|
||||
*/
|
||||
async recordJobCountsMetric(...types) {
|
||||
var _a;
|
||||
const counts = await this.getJobCounts(...types);
|
||||
const meter = (_a = this.opts.telemetry) === null || _a === void 0 ? void 0 : _a.meter;
|
||||
if (meter) {
|
||||
const gauge = meter.createGauge(MetricNames.QueueJobsCount, {
|
||||
description: 'Number of jobs in the queue by state',
|
||||
unit: '{jobs}',
|
||||
});
|
||||
for (const [state, jobCount] of Object.entries(counts)) {
|
||||
gauge.record(jobCount, {
|
||||
[TelemetryAttributes.QueueName]: this.name,
|
||||
[TelemetryAttributes.QueueJobsState]: state,
|
||||
});
|
||||
}
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
/**
|
||||
* Get current job state.
|
||||
*
|
||||
* @param jobId - job identifier.
|
||||
* @returns Returns one of these values:
|
||||
* 'completed', 'failed', 'delayed', 'active', 'waiting', 'waiting-children', 'unknown'.
|
||||
*/
|
||||
getJobState(jobId) {
|
||||
return this.backend.getState(jobId);
|
||||
}
|
||||
/**
|
||||
* Get global queue configuration.
|
||||
*
|
||||
* @returns Returns the global queue configuration.
|
||||
*/
|
||||
async getMeta() {
|
||||
const config = await this.backend.getQueueMeta();
|
||||
const { concurrency, max, duration, paused, 'opts.maxLenEvents': maxLenEvents } = config, rest = __rest(config, ["concurrency", "max", "duration", "paused", 'opts.maxLenEvents']);
|
||||
const parsedConfig = rest;
|
||||
if (concurrency) {
|
||||
parsedConfig['concurrency'] = Number(concurrency);
|
||||
}
|
||||
if (maxLenEvents) {
|
||||
parsedConfig['maxLenEvents'] = Number(maxLenEvents);
|
||||
}
|
||||
if (max) {
|
||||
parsedConfig['max'] = Number(max);
|
||||
}
|
||||
if (duration) {
|
||||
parsedConfig['duration'] = Number(duration);
|
||||
}
|
||||
parsedConfig['paused'] = paused === '1';
|
||||
return parsedConfig;
|
||||
}
|
||||
/**
|
||||
* @returns Returns the number of jobs in completed status.
|
||||
*/
|
||||
getCompletedCount() {
|
||||
return this.getJobCountByTypes('completed');
|
||||
}
|
||||
/**
|
||||
* Returns the number of jobs in failed status.
|
||||
*/
|
||||
getFailedCount() {
|
||||
return this.getJobCountByTypes('failed');
|
||||
}
|
||||
/**
|
||||
* Returns the number of jobs in delayed status.
|
||||
*/
|
||||
getDelayedCount() {
|
||||
return this.getJobCountByTypes('delayed');
|
||||
}
|
||||
/**
|
||||
* Returns the number of jobs in active status.
|
||||
*/
|
||||
getActiveCount() {
|
||||
return this.getJobCountByTypes('active');
|
||||
}
|
||||
/**
|
||||
* Returns the number of jobs in prioritized status.
|
||||
*/
|
||||
getPrioritizedCount() {
|
||||
return this.getJobCountByTypes('prioritized');
|
||||
}
|
||||
/**
|
||||
* Returns the number of jobs per priority.
|
||||
*/
|
||||
async getCountsPerPriority(priorities) {
|
||||
const uniquePriorities = [...new Set(priorities)];
|
||||
const responses = await this.backend.getCountsPerPriority(uniquePriorities);
|
||||
const counts = {};
|
||||
responses.forEach((res, index) => {
|
||||
counts[`${uniquePriorities[index]}`] = res || 0;
|
||||
});
|
||||
return counts;
|
||||
}
|
||||
/**
|
||||
* Returns the number of jobs in waiting or paused statuses.
|
||||
*/
|
||||
getWaitingCount() {
|
||||
return this.getJobCountByTypes('waiting');
|
||||
}
|
||||
/**
|
||||
* Returns the number of jobs in waiting-children status.
|
||||
*/
|
||||
getWaitingChildrenCount() {
|
||||
return this.getJobCountByTypes('waiting-children');
|
||||
}
|
||||
/**
|
||||
* Returns the jobs that are in the "waiting" status.
|
||||
* @param start - zero based index from where to start returning jobs.
|
||||
* @param end - zero based index where to stop returning jobs.
|
||||
*/
|
||||
getWaiting(start = 0, end = -1) {
|
||||
return this.getJobs(['waiting'], start, end, true);
|
||||
}
|
||||
/**
|
||||
* Returns the jobs that are in the "waiting-children" status.
|
||||
* I.E. parent jobs that have at least one child that has not completed yet.
|
||||
* @param start - zero based index from where to start returning jobs.
|
||||
* @param end - zero based index where to stop returning jobs.
|
||||
*/
|
||||
getWaitingChildren(start = 0, end = -1) {
|
||||
return this.getJobs(['waiting-children'], start, end, true);
|
||||
}
|
||||
/**
|
||||
* Returns the jobs that are in the "active" status.
|
||||
* @param start - zero based index from where to start returning jobs.
|
||||
* @param end - zero based index where to stop returning jobs.
|
||||
*/
|
||||
getActive(start = 0, end = -1) {
|
||||
return this.getJobs(['active'], start, end, true);
|
||||
}
|
||||
/**
|
||||
* Returns the jobs that are in the "delayed" status.
|
||||
* @param start - zero based index from where to start returning jobs.
|
||||
* @param end - zero based index where to stop returning jobs.
|
||||
*/
|
||||
getDelayed(start = 0, end = -1) {
|
||||
return this.getJobs(['delayed'], start, end, true);
|
||||
}
|
||||
/**
|
||||
* Returns the jobs that are in the "prioritized" status.
|
||||
* @param start - zero based index from where to start returning jobs.
|
||||
* @param end - zero based index where to stop returning jobs.
|
||||
*/
|
||||
getPrioritized(start = 0, end = -1) {
|
||||
return this.getJobs(['prioritized'], start, end, true);
|
||||
}
|
||||
/**
|
||||
* Returns the jobs that are in the "completed" status.
|
||||
* @param start - zero based index from where to start returning jobs.
|
||||
* @param end - zero based index where to stop returning jobs.
|
||||
*/
|
||||
getCompleted(start = 0, end = -1) {
|
||||
return this.getJobs(['completed'], start, end, false);
|
||||
}
|
||||
/**
|
||||
* Returns the jobs that are in the "failed" status.
|
||||
* @param start - zero based index from where to start returning jobs.
|
||||
* @param end - zero based index where to stop returning jobs.
|
||||
*/
|
||||
getFailed(start = 0, end = -1) {
|
||||
return this.getJobs(['failed'], start, end, false);
|
||||
}
|
||||
/**
|
||||
* Returns the qualified job ids and the raw job data (if available) of the
|
||||
* children jobs of the given parent job.
|
||||
* It is possible to get either the already processed children, in this case
|
||||
* an array of qualified job ids and their result values will be returned,
|
||||
* or the pending children, in this case an array of qualified job ids will
|
||||
* be returned.
|
||||
* A qualified job id is a string representing the job id in a given queue,
|
||||
* for example: "bull:myqueue:jobid".
|
||||
*
|
||||
* @param parentId - The id of the parent job
|
||||
* @param type - "processed" | "pending"
|
||||
* @param opts - Options for the query.
|
||||
*
|
||||
* @returns an object with the following shape:
|
||||
* `{ items: { id: string, v?: any, err?: string } [], jobs: JobJson[], total: number}`
|
||||
*/
|
||||
async getDependencies(parentId, type, start, end) {
|
||||
const key = this.toKey(type == 'processed'
|
||||
? `${parentId}:processed`
|
||||
: `${parentId}:dependencies`);
|
||||
const { items, total, jobs } = await this.backend.paginate(key, {
|
||||
start,
|
||||
end,
|
||||
fetchJobs: true,
|
||||
});
|
||||
return {
|
||||
items,
|
||||
jobs,
|
||||
total,
|
||||
};
|
||||
}
|
||||
async getRanges(types, start = 0, end = 1, asc = false) {
|
||||
const multiCommands = [];
|
||||
this.commandByType(types, false, (key, command) => {
|
||||
switch (command) {
|
||||
case 'lrange':
|
||||
multiCommands.push('lrange');
|
||||
break;
|
||||
case 'zrange':
|
||||
multiCommands.push('zrange');
|
||||
break;
|
||||
}
|
||||
});
|
||||
const responses = await this.backend.getRanges(types, start, end, asc);
|
||||
let results = [];
|
||||
responses.forEach((response, index) => {
|
||||
const result = response || [];
|
||||
if (asc && multiCommands[index] === 'lrange') {
|
||||
results = results.concat(result.reverse());
|
||||
}
|
||||
else {
|
||||
results = results.concat(result);
|
||||
}
|
||||
});
|
||||
return [...new Set(results)];
|
||||
}
|
||||
/**
|
||||
* Returns the jobs that are on the given statuses (note that JobType is synonym for job status)
|
||||
* @param types - the statuses of the jobs to return.
|
||||
* @param start - zero based index from where to start returning jobs.
|
||||
* @param end - zero based index where to stop returning jobs.
|
||||
* @param asc - if true, the jobs will be returned in ascending order.
|
||||
*/
|
||||
async getJobs(types, start = 0, end = -1, asc = false) {
|
||||
const currentTypes = this.sanitizeJobTypes(types);
|
||||
let jobIds;
|
||||
const backend = this.getBackend();
|
||||
if (backend instanceof RedisQueueBackend) {
|
||||
const jobDataByType = await backend.getJobs(currentTypes, start, end, asc);
|
||||
const seen = new Set();
|
||||
jobIds = jobDataByType.reduce((ids, jobData) => {
|
||||
for (const [jobId] of jobData || []) {
|
||||
if (!seen.has(jobId)) {
|
||||
seen.add(jobId);
|
||||
ids.push(jobId);
|
||||
}
|
||||
}
|
||||
return ids;
|
||||
}, []);
|
||||
}
|
||||
else {
|
||||
jobIds = await this.getRanges(currentTypes, start, end, asc);
|
||||
}
|
||||
const jobs = await Promise.all(jobIds.map(jobId => this.getJob(jobId)));
|
||||
return jobs.filter(Boolean);
|
||||
}
|
||||
/**
|
||||
* Returns the logs for a given Job.
|
||||
* @param jobId - the id of the job to get the logs for.
|
||||
* @param start - zero based index from where to start returning jobs.
|
||||
* @param end - zero based index where to stop returning jobs.
|
||||
* @param asc - if true, the jobs will be returned in ascending order.
|
||||
*/
|
||||
async getJobLogs(jobId, start = 0, end = -1, asc = true) {
|
||||
return this.backend.getJobLogs(jobId, start, end, asc);
|
||||
}
|
||||
async baseGetClients(matcher) {
|
||||
var _a;
|
||||
try {
|
||||
const lists = await this.backend.getClientList();
|
||||
if (lists.length > 1) {
|
||||
// Cluster: pick the node with the most matching clients.
|
||||
const clientsPerNode = lists.map(list => this.parseClientList(list, matcher));
|
||||
return clientsPerNode.reduce((prev, current) => (prev.length > current.length ? prev : current), []);
|
||||
}
|
||||
return this.parseClientList((_a = lists[0]) !== null && _a !== void 0 ? _a : '', matcher);
|
||||
}
|
||||
catch (err) {
|
||||
if (!clientCommandMessageReg.test(err.message)) {
|
||||
throw err;
|
||||
}
|
||||
return [{ name: 'GCP does not support client list' }];
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Get the worker list related to the queue. i.e. all the known
|
||||
* workers that are available to process jobs for this queue.
|
||||
* Note: GCP does not support SETNAME, so this call will not work
|
||||
*
|
||||
* @returns - Returns an array with workers info.
|
||||
*/
|
||||
getWorkers() {
|
||||
const unnamedWorkerClientName = `${this.clientName()}`;
|
||||
const namedWorkerClientName = `${this.clientName()}:w:`;
|
||||
const matcher = (name) => name &&
|
||||
(name === unnamedWorkerClientName ||
|
||||
name.startsWith(namedWorkerClientName));
|
||||
return this.baseGetClients(matcher);
|
||||
}
|
||||
/**
|
||||
* Returns the current count of workers for the queue.
|
||||
*
|
||||
* getWorkersCount(): Promise<number>
|
||||
*
|
||||
*/
|
||||
async getWorkersCount() {
|
||||
const workers = await this.getWorkers();
|
||||
return workers.length;
|
||||
}
|
||||
/**
|
||||
* Get queue events list related to the queue.
|
||||
* Note: GCP does not support SETNAME, so this call will not work
|
||||
*
|
||||
* @deprecated do not use this method, it will be removed in the future.
|
||||
*
|
||||
* @returns - Returns an array with queue events info.
|
||||
*/
|
||||
async getQueueEvents() {
|
||||
const clientName = `${this.clientName()}${QUEUE_EVENT_SUFFIX}`;
|
||||
return this.baseGetClients((name) => name === clientName);
|
||||
}
|
||||
/**
|
||||
* Get queue metrics related to the queue.
|
||||
*
|
||||
* This method returns the gathered metrics for the queue.
|
||||
* The metrics are represented as an array of job counts
|
||||
* per unit of time (1 minute).
|
||||
*
|
||||
* @param start - Start point of the metrics, where 0
|
||||
* is the newest point to be returned.
|
||||
* @param end - End point of the metrics, where -1 is the
|
||||
* oldest point to be returned.
|
||||
*
|
||||
* @returns - Returns an object with queue metrics.
|
||||
*/
|
||||
async getMetrics(type, start = 0, end = -1) {
|
||||
const [meta, data, count] = await this.backend.getMetrics(type, start, end);
|
||||
return {
|
||||
meta: {
|
||||
count: parseInt(meta[0] || '0', 10),
|
||||
prevTS: parseInt(meta[1] || '0', 10),
|
||||
prevCount: parseInt(meta[2] || '0', 10),
|
||||
},
|
||||
data: data.map(point => +point || 0),
|
||||
count,
|
||||
};
|
||||
}
|
||||
parseClientList(list, matcher) {
|
||||
const lines = list.split(/\r?\n/);
|
||||
const clients = [];
|
||||
lines.forEach((line) => {
|
||||
const client = {};
|
||||
const keyValues = line.split(' ');
|
||||
keyValues.forEach(function (keyValue) {
|
||||
const index = keyValue.indexOf('=');
|
||||
const key = keyValue.substring(0, index);
|
||||
const value = keyValue.substring(index + 1);
|
||||
client[key] = value;
|
||||
});
|
||||
const name = client['name'];
|
||||
if (matcher(name)) {
|
||||
client['name'] = this.name;
|
||||
client['rawname'] = name;
|
||||
clients.push(client);
|
||||
}
|
||||
});
|
||||
return clients;
|
||||
}
|
||||
/**
|
||||
* Export the metrics for the queue in the Prometheus format.
|
||||
* Automatically exports all the counts returned by getJobCounts().
|
||||
*
|
||||
* @returns - Returns a string with the metrics in the Prometheus format.
|
||||
*
|
||||
* @see {@link https://prometheus.io/docs/instrumenting/exposition_formats/}
|
||||
**/
|
||||
async exportPrometheusMetrics(globalVariables) {
|
||||
const counts = await this.getJobCounts();
|
||||
const metrics = [];
|
||||
// Match the test's expected HELP text
|
||||
metrics.push('# HELP bullmq_job_count Number of jobs in the queue by state');
|
||||
metrics.push('# TYPE bullmq_job_count gauge');
|
||||
const escapedQueueName = escapePrometheusLabelValue(this.name);
|
||||
const variables = !globalVariables
|
||||
? ''
|
||||
: Object.keys(globalVariables).reduce((acc, curr) => `${acc}, ${curr}="${escapePrometheusLabelValue(globalVariables[curr])}"`, '');
|
||||
for (const [state, count] of Object.entries(counts)) {
|
||||
metrics.push(`bullmq_job_count{queue="${escapedQueueName}", state="${state}"${variables}} ${count}`);
|
||||
}
|
||||
const [completedMetrics, failedMetrics] = await Promise.all([
|
||||
this.getMetrics('completed'),
|
||||
this.getMetrics('failed'),
|
||||
]);
|
||||
metrics.push('# HELP bullmq_job_completed_total Total number of completed jobs');
|
||||
metrics.push('# TYPE bullmq_job_completed_total counter');
|
||||
metrics.push(`bullmq_job_completed_total{queue="${escapedQueueName}"${variables}} ${completedMetrics.meta.count}`);
|
||||
metrics.push('# HELP bullmq_job_failed_total Total number of failed jobs');
|
||||
metrics.push('# TYPE bullmq_job_failed_total counter');
|
||||
metrics.push(`bullmq_job_failed_total{queue="${escapedQueueName}"${variables}} ${failedMetrics.meta.count}`);
|
||||
return metrics.join('\n');
|
||||
}
|
||||
}
|
||||
10
node_modules/bullmq/dist/esm/classes/queue-keys.d.ts
generated
vendored
Normal file
10
node_modules/bullmq/dist/esm/classes/queue-keys.d.ts
generated
vendored
Normal file
@@ -0,0 +1,10 @@
|
||||
export type KeysMap = {
|
||||
[index in string]: string;
|
||||
};
|
||||
export declare class QueueKeys {
|
||||
readonly prefix: string;
|
||||
constructor(prefix?: string);
|
||||
getKeys(name: string): KeysMap;
|
||||
toKey(name: string, type: string): string;
|
||||
getQueueQualifiedName(name: string): string;
|
||||
}
|
||||
38
node_modules/bullmq/dist/esm/classes/queue-keys.js
generated
vendored
Normal file
38
node_modules/bullmq/dist/esm/classes/queue-keys.js
generated
vendored
Normal file
@@ -0,0 +1,38 @@
|
||||
export class QueueKeys {
|
||||
constructor(prefix = 'bull') {
|
||||
this.prefix = prefix;
|
||||
}
|
||||
getKeys(name) {
|
||||
const keys = {};
|
||||
[
|
||||
'',
|
||||
'active',
|
||||
'wait',
|
||||
'waiting-children',
|
||||
'paused',
|
||||
'id',
|
||||
'delayed',
|
||||
'prioritized',
|
||||
'stalled-check',
|
||||
'completed',
|
||||
'failed',
|
||||
'stalled',
|
||||
'repeat',
|
||||
'limiter',
|
||||
'meta',
|
||||
'events',
|
||||
'pc', // priority counter key
|
||||
'marker', // marker key
|
||||
'de', // deduplication key
|
||||
].forEach(key => {
|
||||
keys[key] = this.toKey(name, key);
|
||||
});
|
||||
return keys;
|
||||
}
|
||||
toKey(name, type) {
|
||||
return `${this.getQueueQualifiedName(name)}:${type}`;
|
||||
}
|
||||
getQueueQualifiedName(name) {
|
||||
return `${this.prefix}:${name}`;
|
||||
}
|
||||
}
|
||||
414
node_modules/bullmq/dist/esm/classes/queue.d.ts
generated
vendored
Normal file
414
node_modules/bullmq/dist/esm/classes/queue.d.ts
generated
vendored
Normal file
@@ -0,0 +1,414 @@
|
||||
import { BackendFactory, BaseJobOptions, BulkJobOptions, IoredisListener, IQueueBackend, JobSchedulerJson, QueueOptions, RepeatOptions } from '../interfaces';
|
||||
import { FinishedStatus, JobsOptions, JobSchedulerTemplateOptions, JobProgress } from '../types';
|
||||
import { Job } from './job';
|
||||
import { QueueGetters } from './queue-getters';
|
||||
import { RedisQueueBackend } from './redis-queue-backend';
|
||||
import { JobScheduler } from './job-scheduler';
|
||||
export interface ObliterateOpts {
|
||||
/**
|
||||
* Use force = true to force obliteration even with active jobs in the queue
|
||||
* @defaultValue false
|
||||
*/
|
||||
force?: boolean;
|
||||
/**
|
||||
* Use count with the maximum number of deleted keys per iteration
|
||||
* @defaultValue 1000
|
||||
*/
|
||||
count?: number;
|
||||
}
|
||||
export interface QueueListener<JobBase extends Job = Job> extends IoredisListener {
|
||||
/**
|
||||
* Listen to 'cleaned' event.
|
||||
*
|
||||
* This event is triggered when the queue calls clean method.
|
||||
*/
|
||||
cleaned: (jobs: string[], type: string) => void;
|
||||
/**
|
||||
* Listen to 'error' event.
|
||||
*
|
||||
* This event is triggered when an error is thrown.
|
||||
*/
|
||||
error: (err: Error) => void;
|
||||
/**
|
||||
* Listen to 'paused' event.
|
||||
*
|
||||
* This event is triggered when the queue is paused.
|
||||
*/
|
||||
paused: () => void;
|
||||
/**
|
||||
* Listen to 'progress' event.
|
||||
*
|
||||
* This event is triggered when the job updates its progress.
|
||||
*/
|
||||
progress: (jobId: string, progress: JobProgress) => void;
|
||||
/**
|
||||
* Listen to 'removed' event.
|
||||
*
|
||||
* This event is triggered when a job is removed.
|
||||
*/
|
||||
removed: (jobId: string) => void;
|
||||
/**
|
||||
* Listen to 'resumed' event.
|
||||
*
|
||||
* This event is triggered when the queue is resumed.
|
||||
*/
|
||||
resumed: () => void;
|
||||
/**
|
||||
* Listen to 'waiting' event.
|
||||
*
|
||||
* This event is triggered when the queue creates a new job.
|
||||
*/
|
||||
waiting: (job: JobBase) => void;
|
||||
}
|
||||
/**
|
||||
* IsAny<T> A type helper to determine if a given type `T` is `any`.
|
||||
* This works by using `any` type with the intersection
|
||||
* operator (`&`). If `T` is `any`, then `1 & T` resolves to `any`, and since `0`
|
||||
* is assignable to `any`, the conditional type returns `true`.
|
||||
*/
|
||||
type IsAny<T> = 0 extends 1 & T ? true : false;
|
||||
type JobBase<T, ResultType, NameType extends string> = IsAny<T> extends true ? Job<T, ResultType, NameType> : T extends Job<any, any, any> ? T : Job<T, ResultType, NameType>;
|
||||
type ExtractDataType<DataTypeOrJob, Default> = DataTypeOrJob extends Job<infer D, any, any> ? D : Default;
|
||||
type ExtractResultType<DataTypeOrJob, Default> = DataTypeOrJob extends Job<any, infer R, any> ? R : Default;
|
||||
type ExtractNameType<DataTypeOrJob, Default extends string> = DataTypeOrJob extends Job<any, any, infer N> ? N : Default;
|
||||
/**
|
||||
* Queue
|
||||
*
|
||||
* This class provides methods to add jobs to a queue and some other high-level
|
||||
* administration such as pausing or deleting queues.
|
||||
*
|
||||
* @typeParam DataType - The type of the data that the job will process.
|
||||
* @typeParam ResultType - The type of the result of the job.
|
||||
* @typeParam NameType - The type of the name of the job.
|
||||
*
|
||||
* @example
|
||||
*
|
||||
* ```typescript
|
||||
* import { Queue } from 'bullmq';
|
||||
*
|
||||
* interface MyDataType {
|
||||
* foo: string;
|
||||
* }
|
||||
*
|
||||
* interface MyResultType {
|
||||
* bar: string;
|
||||
* }
|
||||
*
|
||||
* const queue = new Queue<MyDataType, MyResultType, "blue" | "brown">('myQueue');
|
||||
* ```
|
||||
*/
|
||||
export declare class Queue<DataTypeOrJob = any, DefaultResultType = any, DefaultNameType extends string = string, DataType = ExtractDataType<DataTypeOrJob, DataTypeOrJob>, ResultType = ExtractResultType<DataTypeOrJob, DefaultResultType>, NameType extends string = ExtractNameType<DataTypeOrJob, DefaultNameType>, B extends IQueueBackend = RedisQueueBackend> extends QueueGetters<JobBase<DataTypeOrJob, ResultType, NameType>, B> {
|
||||
token: `${string}-${string}-${string}-${string}-${string}`;
|
||||
jobsOpts: BaseJobOptions;
|
||||
opts: QueueOptions;
|
||||
protected libName: string;
|
||||
protected _jobScheduler?: JobScheduler;
|
||||
private readonly queueMetaInitialized;
|
||||
constructor(name: string, opts?: QueueOptions, backendFactory?: BackendFactory<B>);
|
||||
emit<U extends keyof QueueListener<JobBase<DataType, ResultType, NameType>>>(event: U, ...args: Parameters<QueueListener<JobBase<DataType, ResultType, NameType>>[U]>): boolean;
|
||||
off<U extends keyof QueueListener<JobBase<DataType, ResultType, NameType>>>(eventName: U, listener: QueueListener<JobBase<DataType, ResultType, NameType>>[U]): this;
|
||||
on<U extends keyof QueueListener<JobBase<DataType, ResultType, NameType>>>(event: U, listener: QueueListener<JobBase<DataType, ResultType, NameType>>[U]): this;
|
||||
once<U extends keyof QueueListener<JobBase<DataType, ResultType, NameType>>>(event: U, listener: QueueListener<JobBase<DataType, ResultType, NameType>>[U]): this;
|
||||
/**
|
||||
* Returns this instance current default job options.
|
||||
*/
|
||||
get defaultJobOptions(): JobsOptions;
|
||||
get metaValues(): Record<string, string | number>;
|
||||
/**
|
||||
* Get library version.
|
||||
*
|
||||
* @returns the content of the `meta.version` field, or `null` when the meta
|
||||
* hash has not been populated yet (e.g. when `skipMetasUpdate` is enabled and
|
||||
* no other instance has written it).
|
||||
*/
|
||||
getVersion(): Promise<string | null>;
|
||||
get jobScheduler(): Promise<JobScheduler>;
|
||||
/**
|
||||
* Enable and set global concurrency value.
|
||||
* @param concurrency - Maximum number of simultaneous jobs that the workers can handle.
|
||||
* For instance, setting this value to 1 ensures that no more than one job
|
||||
* is processed at any given time. If this limit is not defined, there will be no
|
||||
* restriction on the number of concurrent jobs.
|
||||
*/
|
||||
setGlobalConcurrency(concurrency: number): Promise<number>;
|
||||
/**
|
||||
* Enable and set rate limit.
|
||||
* @param max - Max number of jobs to process in the time period specified in `duration`
|
||||
* @param duration - Time in milliseconds. During this time, a maximum of `max` jobs will be processed.
|
||||
*/
|
||||
setGlobalRateLimit(max: number, duration: number): Promise<number>;
|
||||
/**
|
||||
* Remove global concurrency value.
|
||||
*/
|
||||
removeGlobalConcurrency(): Promise<number>;
|
||||
/**
|
||||
* Remove global rate limit values.
|
||||
*/
|
||||
removeGlobalRateLimit(): Promise<number>;
|
||||
/**
|
||||
* Adds a new job to the queue.
|
||||
*
|
||||
* @param name - Name of the job to be added to the queue.
|
||||
* @param data - Arbitrary data to append to the job. The value is
|
||||
* serialized with `JSON.stringify` before being stored in Redis, so it
|
||||
* must be JSON-serializable. The `DataType` type parameter describes the
|
||||
* shape of this payload, not a runtime class: passing a class instance
|
||||
* will store its enumerable own properties, but its prototype methods
|
||||
* and getters/setters will not be available when a worker reads
|
||||
* `job.data` back. Prefer plain objects, or implement `toJSON()` on
|
||||
* the class to control how it is serialized.
|
||||
* @param opts - Job options that affects how the job is going to be processed.
|
||||
*/
|
||||
add(name: NameType, data: DataType, opts?: JobsOptions): Promise<Job<DataType, ResultType, NameType>>;
|
||||
/**
|
||||
* addJob is a telemetry free version of the add method, useful in order to wrap it
|
||||
* with custom telemetry on subclasses.
|
||||
*
|
||||
* @param name - Name of the job to be added to the queue.
|
||||
* @param data - Arbitrary data to append to the job.
|
||||
* @param opts - Job options that affects how the job is going to be processed.
|
||||
*
|
||||
* @returns Job
|
||||
*/
|
||||
protected addJob(name: NameType, data: DataType, opts?: JobsOptions): Promise<Job<DataType, ResultType, NameType>>;
|
||||
/**
|
||||
* Adds an array of jobs to the queue. This method may be faster than adding
|
||||
* one job at a time in a sequence.
|
||||
*
|
||||
* @param jobs - The array of jobs to add to the queue. Each job is defined by 3
|
||||
* properties, 'name', 'data' and 'opts'. They follow the same signature as 'Queue.add',
|
||||
* including the JSON-serialization caveat for `data` (class instances lose their
|
||||
* prototype methods on the worker side).
|
||||
*/
|
||||
addBulk(jobs: {
|
||||
name: NameType;
|
||||
data: DataType;
|
||||
opts?: BulkJobOptions;
|
||||
}[]): Promise<Job<DataType, ResultType, NameType>[]>;
|
||||
/**
|
||||
* Upserts a scheduler.
|
||||
*
|
||||
* A scheduler is a job factory that creates jobs at a given interval.
|
||||
* Upserting a scheduler will create a new job scheduler or update an existing one.
|
||||
* It will also create the first job based on the repeat options and delayed accordingly.
|
||||
*
|
||||
* @param key - Unique key for the repeatable job meta.
|
||||
* @param repeatOpts - Repeat options
|
||||
* @param jobTemplate - Job template. If provided it will be used for all the jobs
|
||||
* created by the scheduler.
|
||||
*
|
||||
* @returns The next job to be scheduled (would normally be in delayed state).
|
||||
*/
|
||||
upsertJobScheduler(jobSchedulerId: NameType, repeatOpts: Omit<RepeatOptions, 'key'>, jobTemplate?: {
|
||||
name?: NameType;
|
||||
data?: DataType;
|
||||
opts?: JobSchedulerTemplateOptions;
|
||||
}): Promise<Job<DataType, ResultType, NameType>>;
|
||||
/**
|
||||
* Pauses the processing of this queue globally.
|
||||
*
|
||||
* We use an atomic RENAME operation on the wait queue. Since
|
||||
* we have blocking calls with BRPOPLPUSH on the wait queue, as long as the queue
|
||||
* is renamed to 'paused', no new jobs will be processed (the current ones
|
||||
* will run until finalized).
|
||||
*
|
||||
* Adding jobs requires a LUA script to check first if the paused list exist
|
||||
* and in that case it will add it there instead of the wait list.
|
||||
*/
|
||||
pause(): Promise<void>;
|
||||
/**
|
||||
* Close the queue instance.
|
||||
*
|
||||
*/
|
||||
close(): Promise<void>;
|
||||
/**
|
||||
* Overrides the rate limit to be active for the next jobs.
|
||||
*
|
||||
* @param expireTimeMs - expire time in ms of this rate limit.
|
||||
*/
|
||||
rateLimit(expireTimeMs: number): Promise<void>;
|
||||
/**
|
||||
* Resumes the processing of this queue globally.
|
||||
*
|
||||
* The method reverses the pause operation by resuming the processing of the
|
||||
* queue.
|
||||
*/
|
||||
resume(): Promise<void>;
|
||||
/**
|
||||
* Returns true if the queue is currently paused.
|
||||
*/
|
||||
isPaused(): Promise<boolean>;
|
||||
/**
|
||||
* Returns true if the queue is currently maxed.
|
||||
*/
|
||||
isMaxed(): Promise<boolean>;
|
||||
/**
|
||||
* Get Job Scheduler by id
|
||||
*
|
||||
* @param id - identifier of scheduler.
|
||||
*/
|
||||
getJobScheduler(id: string): Promise<JobSchedulerJson<DataType> | undefined>;
|
||||
/**
|
||||
* Get all Job Schedulers
|
||||
*
|
||||
* @param start - Offset of first scheduler to return.
|
||||
* @param end - Offset of last scheduler to return.
|
||||
* @param asc - Determine the order in which schedulers are returned based on their
|
||||
* next execution time.
|
||||
*/
|
||||
getJobSchedulers(start?: number, end?: number, asc?: boolean): Promise<JobSchedulerJson<DataType>[]>;
|
||||
/**
|
||||
*
|
||||
* Get the number of job schedulers.
|
||||
*
|
||||
* @returns The number of job schedulers.
|
||||
*/
|
||||
getJobSchedulersCount(): Promise<number>;
|
||||
/**
|
||||
*
|
||||
* Removes a job scheduler.
|
||||
*
|
||||
* @param jobSchedulerId - identifier of the job scheduler.
|
||||
*
|
||||
* @returns
|
||||
*/
|
||||
removeJobScheduler(jobSchedulerId: string): Promise<boolean>;
|
||||
/**
|
||||
* Removes a debounce key.
|
||||
* @deprecated use removeDeduplicationKey
|
||||
*
|
||||
* @param id - debounce identifier
|
||||
*/
|
||||
removeDebounceKey(id: string): Promise<number>;
|
||||
/**
|
||||
* Removes a deduplication key.
|
||||
*
|
||||
* @param id - identifier
|
||||
*/
|
||||
removeDeduplicationKey(id: string): Promise<number>;
|
||||
/**
|
||||
* Removes rate limit key.
|
||||
*/
|
||||
removeRateLimitKey(): Promise<number>;
|
||||
/**
|
||||
* Removes the given job from the queue as well as all its
|
||||
* dependencies.
|
||||
*
|
||||
* @param jobId - The id of the job to remove
|
||||
* @param opts - Options to remove a job
|
||||
* @returns 1 if it managed to remove the job or 0 if the job or
|
||||
* any of its dependencies were locked.
|
||||
*/
|
||||
remove(jobId: string, { removeChildren }?: {
|
||||
removeChildren?: boolean;
|
||||
}): Promise<number>;
|
||||
/**
|
||||
* Updates the given job's progress.
|
||||
*
|
||||
* @param jobId - The id of the job to update
|
||||
* @param progress - Number or object to be saved as progress.
|
||||
*/
|
||||
updateJobProgress(jobId: string, progress: JobProgress): Promise<void>;
|
||||
/**
|
||||
* Logs one row of job's log data.
|
||||
*
|
||||
* @param jobId - The job id to log against.
|
||||
* @param logRow - String with log data to be logged.
|
||||
* @param keepLogs - Max number of log entries to keep (0 for unlimited).
|
||||
*
|
||||
* @returns The total number of log entries for this job so far.
|
||||
*/
|
||||
addJobLog(jobId: string, logRow: string, keepLogs?: number): Promise<number>;
|
||||
/**
|
||||
* Drains the queue, i.e., removes all jobs that are waiting
|
||||
* or delayed, but not active, completed or failed.
|
||||
*
|
||||
* @param delayed - Pass true if it should also clean the
|
||||
* delayed jobs.
|
||||
*/
|
||||
drain(delayed?: boolean): Promise<void>;
|
||||
/**
|
||||
* Cleans jobs from a queue. Similar to drain but keeps jobs within a certain
|
||||
* grace period.
|
||||
*
|
||||
* @param grace - The grace period in milliseconds
|
||||
* @param limit - Max number of jobs to clean
|
||||
* @param type - The type of job to clean
|
||||
* Possible values are completed, wait, active, paused, delayed, failed. Defaults to completed.
|
||||
* @returns Id jobs from the deleted records
|
||||
*/
|
||||
clean(grace: number, limit: number, type?: 'completed' | 'wait' | 'waiting' | 'active' | 'paused' | 'prioritized' | 'delayed' | 'failed'): Promise<string[]>;
|
||||
/**
|
||||
* Completely destroys the queue and all of its contents irreversibly.
|
||||
* This method will *pause* the queue and requires that there are no
|
||||
* active jobs. It is possible to bypass this requirement, i.e. not
|
||||
* having active jobs using the "force" option.
|
||||
*
|
||||
* Note: This operation requires to iterate on all the jobs stored in the queue
|
||||
* and can be slow for very large queues.
|
||||
*
|
||||
* @param opts - Obliterate options.
|
||||
*/
|
||||
obliterate(opts?: ObliterateOpts): Promise<void>;
|
||||
/**
|
||||
* Retry all the failed or completed jobs.
|
||||
*
|
||||
* @param opts - An object with the following properties:
|
||||
* - count number to limit how many jobs will be moved to wait status per iteration,
|
||||
* - state failed by default or completed.
|
||||
* - timestamp from which timestamp to start moving jobs to wait status, default Date.now().
|
||||
*
|
||||
* @returns
|
||||
*/
|
||||
retryJobs(opts?: {
|
||||
count?: number;
|
||||
state?: FinishedStatus;
|
||||
timestamp?: number;
|
||||
}): Promise<void>;
|
||||
/**
|
||||
* Promote all the delayed jobs.
|
||||
*
|
||||
* @param opts - An object with the following properties:
|
||||
* - count number to limit how many jobs will be moved to wait status per iteration
|
||||
*
|
||||
* @returns
|
||||
*/
|
||||
promoteJobs(opts?: {
|
||||
count?: number;
|
||||
}): Promise<void>;
|
||||
/**
|
||||
* Trim the event stream to an approximately maxLength.
|
||||
*
|
||||
* @param maxLength - The approximate maximum length, or target length, of the event stream.
|
||||
*/
|
||||
trimEvents(maxLength: number): Promise<number>;
|
||||
/**
|
||||
* Delete old priority helper key.
|
||||
*/
|
||||
removeDeprecatedPriorityKey(): Promise<number>;
|
||||
/**
|
||||
* Removes orphaned job keys that are stored in the backend but are not
|
||||
* referenced in any queue state set.
|
||||
*
|
||||
* Orphaned keys can occur in rare cases when the removal-by-max-age logic
|
||||
* removes state entries without fully cleaning up the corresponding job
|
||||
* data (a regression introduced in v5.66.6 via #3694).
|
||||
* Under normal operation this method is
|
||||
* **not needed** — it is provided only as a one-time migration helper for
|
||||
* users who were affected by that specific bug and want to reclaim the
|
||||
* leaked storage.
|
||||
*
|
||||
* How the scan is performed (its atomicity, batching and how the queue's
|
||||
* state keys are discovered) is an implementation detail of the underlying
|
||||
* backend.
|
||||
*
|
||||
* @param count - Approximate number of keys to scan per iteration (default 1000).
|
||||
* @param limit - Maximum number of orphaned jobs to remove (0 = unlimited).
|
||||
* When set, the method returns as soon as the limit is reached.
|
||||
* Users with a very large number of orphans can call this method
|
||||
* in a loop: `while (await queue.removeOrphanedJobs(1000, 10000)) {}`
|
||||
* @returns The total number of orphaned jobs that were removed.
|
||||
*/
|
||||
removeOrphanedJobs(count?: number, limit?: number): Promise<number>;
|
||||
}
|
||||
export {};
|
||||
595
node_modules/bullmq/dist/esm/classes/queue.js
generated
vendored
Normal file
595
node_modules/bullmq/dist/esm/classes/queue.js
generated
vendored
Normal file
@@ -0,0 +1,595 @@
|
||||
import { Job } from './job';
|
||||
import { QueueGetters } from './queue-getters';
|
||||
import { SpanKind, TelemetryAttributes } from '../enums';
|
||||
import { JobScheduler } from './job-scheduler';
|
||||
import { version } from '../version';
|
||||
import { randomUUID } from '../utils';
|
||||
/**
|
||||
* Queue
|
||||
*
|
||||
* This class provides methods to add jobs to a queue and some other high-level
|
||||
* administration such as pausing or deleting queues.
|
||||
*
|
||||
* @typeParam DataType - The type of the data that the job will process.
|
||||
* @typeParam ResultType - The type of the result of the job.
|
||||
* @typeParam NameType - The type of the name of the job.
|
||||
*
|
||||
* @example
|
||||
*
|
||||
* ```typescript
|
||||
* import { Queue } from 'bullmq';
|
||||
*
|
||||
* interface MyDataType {
|
||||
* foo: string;
|
||||
* }
|
||||
*
|
||||
* interface MyResultType {
|
||||
* bar: string;
|
||||
* }
|
||||
*
|
||||
* const queue = new Queue<MyDataType, MyResultType, "blue" | "brown">('myQueue');
|
||||
* ```
|
||||
*/
|
||||
export class Queue extends QueueGetters {
|
||||
constructor(name, opts, backendFactory) {
|
||||
var _a;
|
||||
super(name, Object.assign({}, opts), backendFactory);
|
||||
this.token = randomUUID();
|
||||
this.libName = 'bullmq';
|
||||
this.jobsOpts = (_a = opts === null || opts === void 0 ? void 0 : opts.defaultJobOptions) !== null && _a !== void 0 ? _a : {};
|
||||
this.queueMetaInitialized = this.waitUntilReady()
|
||||
.then(() => {
|
||||
if (!this.closing && !(opts === null || opts === void 0 ? void 0 : opts.skipMetasUpdate)) {
|
||||
return this.backend
|
||||
.setQueueMeta(this.metaValues)
|
||||
.then(() => undefined);
|
||||
}
|
||||
})
|
||||
.catch((_err) => {
|
||||
// We ignore this error to avoid warnings. The error can still
|
||||
// be received by listening to event 'error'
|
||||
});
|
||||
}
|
||||
emit(event, ...args) {
|
||||
return super.emit(event, ...args);
|
||||
}
|
||||
off(eventName, listener) {
|
||||
super.off(eventName, listener);
|
||||
return this;
|
||||
}
|
||||
on(event, listener) {
|
||||
super.on(event, listener);
|
||||
return this;
|
||||
}
|
||||
once(event, listener) {
|
||||
super.once(event, listener);
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* Returns this instance current default job options.
|
||||
*/
|
||||
get defaultJobOptions() {
|
||||
return Object.assign({}, this.jobsOpts);
|
||||
}
|
||||
get metaValues() {
|
||||
var _a, _b, _c, _d;
|
||||
return {
|
||||
'opts.maxLenEvents': (_d = (_c = (_b = (_a = this.opts) === null || _a === void 0 ? void 0 : _a.streams) === null || _b === void 0 ? void 0 : _b.events) === null || _c === void 0 ? void 0 : _c.maxLen) !== null && _d !== void 0 ? _d : 10000,
|
||||
version: `${this.libName}:${version}`,
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Get library version.
|
||||
*
|
||||
* @returns the content of the `meta.version` field, or `null` when the meta
|
||||
* hash has not been populated yet (e.g. when `skipMetasUpdate` is enabled and
|
||||
* no other instance has written it).
|
||||
*/
|
||||
async getVersion() {
|
||||
var _a;
|
||||
if (!((_a = this.opts) === null || _a === void 0 ? void 0 : _a.skipMetasUpdate)) {
|
||||
await this.queueMetaInitialized;
|
||||
}
|
||||
return await this.backend.getQueueMetaField('version');
|
||||
}
|
||||
get jobScheduler() {
|
||||
return new Promise(async (resolve) => {
|
||||
if (!this._jobScheduler) {
|
||||
// Share this queue's backend (same queue name/keys) with the scheduler.
|
||||
this._jobScheduler = new JobScheduler(this.name, this.opts, () => this.backend);
|
||||
this._jobScheduler.on('error', this.emit.bind(this, 'error'));
|
||||
}
|
||||
resolve(this._jobScheduler);
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Enable and set global concurrency value.
|
||||
* @param concurrency - Maximum number of simultaneous jobs that the workers can handle.
|
||||
* For instance, setting this value to 1 ensures that no more than one job
|
||||
* is processed at any given time. If this limit is not defined, there will be no
|
||||
* restriction on the number of concurrent jobs.
|
||||
*/
|
||||
async setGlobalConcurrency(concurrency) {
|
||||
return this.backend.setQueueMeta({ concurrency });
|
||||
}
|
||||
/**
|
||||
* Enable and set rate limit.
|
||||
* @param max - Max number of jobs to process in the time period specified in `duration`
|
||||
* @param duration - Time in milliseconds. During this time, a maximum of `max` jobs will be processed.
|
||||
*/
|
||||
async setGlobalRateLimit(max, duration) {
|
||||
return this.backend.setQueueMeta({ max, duration });
|
||||
}
|
||||
/**
|
||||
* Remove global concurrency value.
|
||||
*/
|
||||
async removeGlobalConcurrency() {
|
||||
return this.backend.removeQueueMetaFields(['concurrency']);
|
||||
}
|
||||
/**
|
||||
* Remove global rate limit values.
|
||||
*/
|
||||
async removeGlobalRateLimit() {
|
||||
return this.backend.removeQueueMetaFields(['max', 'duration']);
|
||||
}
|
||||
/**
|
||||
* Adds a new job to the queue.
|
||||
*
|
||||
* @param name - Name of the job to be added to the queue.
|
||||
* @param data - Arbitrary data to append to the job. The value is
|
||||
* serialized with `JSON.stringify` before being stored in Redis, so it
|
||||
* must be JSON-serializable. The `DataType` type parameter describes the
|
||||
* shape of this payload, not a runtime class: passing a class instance
|
||||
* will store its enumerable own properties, but its prototype methods
|
||||
* and getters/setters will not be available when a worker reads
|
||||
* `job.data` back. Prefer plain objects, or implement `toJSON()` on
|
||||
* the class to control how it is serialized.
|
||||
* @param opts - Job options that affects how the job is going to be processed.
|
||||
*/
|
||||
async add(name, data, opts) {
|
||||
return this.trace(SpanKind.PRODUCER, 'add', `${this.name}.${name}`, async (span, srcPropagationMetadata) => {
|
||||
var _a;
|
||||
if (srcPropagationMetadata && !((_a = opts === null || opts === void 0 ? void 0 : opts.telemetry) === null || _a === void 0 ? void 0 : _a.omitContext)) {
|
||||
const telemetry = {
|
||||
metadata: srcPropagationMetadata,
|
||||
};
|
||||
opts = Object.assign(Object.assign({}, opts), { telemetry });
|
||||
}
|
||||
const job = await this.addJob(name, data, opts);
|
||||
span === null || span === void 0 ? void 0 : span.setAttributes({
|
||||
[TelemetryAttributes.JobName]: name,
|
||||
[TelemetryAttributes.JobId]: job.id,
|
||||
});
|
||||
return job;
|
||||
});
|
||||
}
|
||||
/**
|
||||
* addJob is a telemetry free version of the add method, useful in order to wrap it
|
||||
* with custom telemetry on subclasses.
|
||||
*
|
||||
* @param name - Name of the job to be added to the queue.
|
||||
* @param data - Arbitrary data to append to the job.
|
||||
* @param opts - Job options that affects how the job is going to be processed.
|
||||
*
|
||||
* @returns Job
|
||||
*/
|
||||
async addJob(name, data, opts) {
|
||||
const jobId = opts === null || opts === void 0 ? void 0 : opts.jobId;
|
||||
if (jobId == '0' || (jobId === null || jobId === void 0 ? void 0 : jobId.startsWith('0:'))) {
|
||||
throw new Error("JobId cannot be '0' or start with '0:'");
|
||||
}
|
||||
const mergedOpts = Object.assign(Object.assign(Object.assign({}, this.jobsOpts), opts), { jobId });
|
||||
const job = await this.Job.create(this, name, data, mergedOpts);
|
||||
this.emit('waiting', job);
|
||||
return job;
|
||||
}
|
||||
/**
|
||||
* Adds an array of jobs to the queue. This method may be faster than adding
|
||||
* one job at a time in a sequence.
|
||||
*
|
||||
* @param jobs - The array of jobs to add to the queue. Each job is defined by 3
|
||||
* properties, 'name', 'data' and 'opts'. They follow the same signature as 'Queue.add',
|
||||
* including the JSON-serialization caveat for `data` (class instances lose their
|
||||
* prototype methods on the worker side).
|
||||
*/
|
||||
async addBulk(jobs) {
|
||||
return this.trace(SpanKind.PRODUCER, 'addBulk', this.name, async (span, srcPropagationMetadata) => {
|
||||
if (span) {
|
||||
span.setAttributes({
|
||||
[TelemetryAttributes.BulkNames]: jobs.map(job => job.name),
|
||||
[TelemetryAttributes.BulkCount]: jobs.length,
|
||||
});
|
||||
}
|
||||
return await this.Job.createBulk(this, jobs.map(job => {
|
||||
var _a, _b, _c, _d, _e, _f;
|
||||
let telemetry = (_a = job.opts) === null || _a === void 0 ? void 0 : _a.telemetry;
|
||||
if (srcPropagationMetadata) {
|
||||
const omitContext = (_c = (_b = job.opts) === null || _b === void 0 ? void 0 : _b.telemetry) === null || _c === void 0 ? void 0 : _c.omitContext;
|
||||
const telemetryMetadata = ((_e = (_d = job.opts) === null || _d === void 0 ? void 0 : _d.telemetry) === null || _e === void 0 ? void 0 : _e.metadata) ||
|
||||
(!omitContext && srcPropagationMetadata);
|
||||
if (telemetryMetadata || omitContext) {
|
||||
telemetry = {
|
||||
metadata: telemetryMetadata,
|
||||
omitContext,
|
||||
};
|
||||
}
|
||||
}
|
||||
const mergedOpts = Object.assign(Object.assign(Object.assign({}, this.jobsOpts), job.opts), { jobId: (_f = job.opts) === null || _f === void 0 ? void 0 : _f.jobId, telemetry });
|
||||
return {
|
||||
name: job.name,
|
||||
data: job.data,
|
||||
opts: mergedOpts,
|
||||
};
|
||||
}));
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Upserts a scheduler.
|
||||
*
|
||||
* A scheduler is a job factory that creates jobs at a given interval.
|
||||
* Upserting a scheduler will create a new job scheduler or update an existing one.
|
||||
* It will also create the first job based on the repeat options and delayed accordingly.
|
||||
*
|
||||
* @param key - Unique key for the repeatable job meta.
|
||||
* @param repeatOpts - Repeat options
|
||||
* @param jobTemplate - Job template. If provided it will be used for all the jobs
|
||||
* created by the scheduler.
|
||||
*
|
||||
* @returns The next job to be scheduled (would normally be in delayed state).
|
||||
*/
|
||||
async upsertJobScheduler(jobSchedulerId, repeatOpts, jobTemplate) {
|
||||
var _a, _b;
|
||||
if (repeatOpts.endDate) {
|
||||
if (+new Date(repeatOpts.endDate) < Date.now()) {
|
||||
throw new Error('End date must be greater than current timestamp');
|
||||
}
|
||||
}
|
||||
return (await this.jobScheduler).upsertJobScheduler(jobSchedulerId, repeatOpts, (_a = jobTemplate === null || jobTemplate === void 0 ? void 0 : jobTemplate.name) !== null && _a !== void 0 ? _a : jobSchedulerId, (_b = jobTemplate === null || jobTemplate === void 0 ? void 0 : jobTemplate.data) !== null && _b !== void 0 ? _b : {}, Object.assign(Object.assign({}, this.jobsOpts), jobTemplate === null || jobTemplate === void 0 ? void 0 : jobTemplate.opts), { override: true });
|
||||
}
|
||||
/**
|
||||
* Pauses the processing of this queue globally.
|
||||
*
|
||||
* We use an atomic RENAME operation on the wait queue. Since
|
||||
* we have blocking calls with BRPOPLPUSH on the wait queue, as long as the queue
|
||||
* is renamed to 'paused', no new jobs will be processed (the current ones
|
||||
* will run until finalized).
|
||||
*
|
||||
* Adding jobs requires a LUA script to check first if the paused list exist
|
||||
* and in that case it will add it there instead of the wait list.
|
||||
*/
|
||||
async pause() {
|
||||
await this.trace(SpanKind.INTERNAL, 'pause', this.name, async () => {
|
||||
await this.backend.pause(true);
|
||||
this.emit('paused');
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Close the queue instance.
|
||||
*
|
||||
*/
|
||||
async close() {
|
||||
await this.trace(SpanKind.INTERNAL, 'close', this.name, async () => {
|
||||
await super.close();
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Overrides the rate limit to be active for the next jobs.
|
||||
*
|
||||
* @param expireTimeMs - expire time in ms of this rate limit.
|
||||
*/
|
||||
async rateLimit(expireTimeMs) {
|
||||
await this.trace(SpanKind.INTERNAL, 'rateLimit', this.name, async (span) => {
|
||||
span === null || span === void 0 ? void 0 : span.setAttributes({
|
||||
[TelemetryAttributes.QueueRateLimit]: expireTimeMs,
|
||||
});
|
||||
await this.backend.setRateLimit(expireTimeMs);
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Resumes the processing of this queue globally.
|
||||
*
|
||||
* The method reverses the pause operation by resuming the processing of the
|
||||
* queue.
|
||||
*/
|
||||
async resume() {
|
||||
await this.trace(SpanKind.INTERNAL, 'resume', this.name, async () => {
|
||||
await this.backend.pause(false);
|
||||
this.emit('resumed');
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Returns true if the queue is currently paused.
|
||||
*/
|
||||
async isPaused() {
|
||||
return this.backend.hasQueueMetaField('paused');
|
||||
}
|
||||
/**
|
||||
* Returns true if the queue is currently maxed.
|
||||
*/
|
||||
isMaxed() {
|
||||
return this.backend.isMaxed();
|
||||
}
|
||||
/**
|
||||
* Get Job Scheduler by id
|
||||
*
|
||||
* @param id - identifier of scheduler.
|
||||
*/
|
||||
async getJobScheduler(id) {
|
||||
return (await this.jobScheduler).getScheduler(id);
|
||||
}
|
||||
/**
|
||||
* Get all Job Schedulers
|
||||
*
|
||||
* @param start - Offset of first scheduler to return.
|
||||
* @param end - Offset of last scheduler to return.
|
||||
* @param asc - Determine the order in which schedulers are returned based on their
|
||||
* next execution time.
|
||||
*/
|
||||
async getJobSchedulers(start, end, asc) {
|
||||
return (await this.jobScheduler).getJobSchedulers(start, end, asc);
|
||||
}
|
||||
/**
|
||||
*
|
||||
* Get the number of job schedulers.
|
||||
*
|
||||
* @returns The number of job schedulers.
|
||||
*/
|
||||
async getJobSchedulersCount() {
|
||||
return (await this.jobScheduler).getSchedulersCount();
|
||||
}
|
||||
/**
|
||||
*
|
||||
* Removes a job scheduler.
|
||||
*
|
||||
* @param jobSchedulerId - identifier of the job scheduler.
|
||||
*
|
||||
* @returns
|
||||
*/
|
||||
async removeJobScheduler(jobSchedulerId) {
|
||||
const jobScheduler = await this.jobScheduler;
|
||||
const removed = await jobScheduler.removeJobScheduler(jobSchedulerId);
|
||||
return !removed;
|
||||
}
|
||||
/**
|
||||
* Removes a debounce key.
|
||||
* @deprecated use removeDeduplicationKey
|
||||
*
|
||||
* @param id - debounce identifier
|
||||
*/
|
||||
async removeDebounceKey(id) {
|
||||
return this.trace(SpanKind.INTERNAL, 'removeDebounceKey', `${this.name}`, async (span) => {
|
||||
span === null || span === void 0 ? void 0 : span.setAttributes({
|
||||
[TelemetryAttributes.JobKey]: id,
|
||||
});
|
||||
return await this.backend.deleteDeduplicationKey(id);
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Removes a deduplication key.
|
||||
*
|
||||
* @param id - identifier
|
||||
*/
|
||||
async removeDeduplicationKey(id) {
|
||||
return this.trace(SpanKind.INTERNAL, 'removeDeduplicationKey', `${this.name}`, async (span) => {
|
||||
span === null || span === void 0 ? void 0 : span.setAttributes({
|
||||
[TelemetryAttributes.DeduplicationKey]: id,
|
||||
});
|
||||
return this.backend.deleteDeduplicationKey(id);
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Removes rate limit key.
|
||||
*/
|
||||
async removeRateLimitKey() {
|
||||
return this.backend.removeRateLimitKey();
|
||||
}
|
||||
/**
|
||||
* Removes the given job from the queue as well as all its
|
||||
* dependencies.
|
||||
*
|
||||
* @param jobId - The id of the job to remove
|
||||
* @param opts - Options to remove a job
|
||||
* @returns 1 if it managed to remove the job or 0 if the job or
|
||||
* any of its dependencies were locked.
|
||||
*/
|
||||
async remove(jobId, { removeChildren = true } = {}) {
|
||||
return this.trace(SpanKind.INTERNAL, 'remove', this.name, async (span) => {
|
||||
span === null || span === void 0 ? void 0 : span.setAttributes({
|
||||
[TelemetryAttributes.JobId]: jobId,
|
||||
[TelemetryAttributes.JobOptions]: JSON.stringify({
|
||||
removeChildren,
|
||||
}),
|
||||
});
|
||||
const code = await this.backend.remove(jobId, removeChildren);
|
||||
if (code === 1) {
|
||||
this.emit('removed', jobId);
|
||||
}
|
||||
return code;
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Updates the given job's progress.
|
||||
*
|
||||
* @param jobId - The id of the job to update
|
||||
* @param progress - Number or object to be saved as progress.
|
||||
*/
|
||||
async updateJobProgress(jobId, progress) {
|
||||
await this.trace(SpanKind.INTERNAL, 'updateJobProgress', this.name, async (span) => {
|
||||
span === null || span === void 0 ? void 0 : span.setAttributes({
|
||||
[TelemetryAttributes.JobId]: jobId,
|
||||
[TelemetryAttributes.JobProgress]: JSON.stringify(progress),
|
||||
});
|
||||
await this.backend.updateProgress(jobId, progress);
|
||||
this.emit('progress', jobId, progress);
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Logs one row of job's log data.
|
||||
*
|
||||
* @param jobId - The job id to log against.
|
||||
* @param logRow - String with log data to be logged.
|
||||
* @param keepLogs - Max number of log entries to keep (0 for unlimited).
|
||||
*
|
||||
* @returns The total number of log entries for this job so far.
|
||||
*/
|
||||
async addJobLog(jobId, logRow, keepLogs) {
|
||||
return Job.addJobLog(this, jobId, logRow, keepLogs);
|
||||
}
|
||||
/**
|
||||
* Drains the queue, i.e., removes all jobs that are waiting
|
||||
* or delayed, but not active, completed or failed.
|
||||
*
|
||||
* @param delayed - Pass true if it should also clean the
|
||||
* delayed jobs.
|
||||
*/
|
||||
async drain(delayed = false) {
|
||||
await this.trace(SpanKind.INTERNAL, 'drain', this.name, async (span) => {
|
||||
span === null || span === void 0 ? void 0 : span.setAttributes({
|
||||
[TelemetryAttributes.QueueDrainDelay]: delayed,
|
||||
});
|
||||
await this.backend.drain(delayed);
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Cleans jobs from a queue. Similar to drain but keeps jobs within a certain
|
||||
* grace period.
|
||||
*
|
||||
* @param grace - The grace period in milliseconds
|
||||
* @param limit - Max number of jobs to clean
|
||||
* @param type - The type of job to clean
|
||||
* Possible values are completed, wait, active, paused, delayed, failed. Defaults to completed.
|
||||
* @returns Id jobs from the deleted records
|
||||
*/
|
||||
async clean(grace, limit, type = 'completed') {
|
||||
return this.trace(SpanKind.INTERNAL, 'clean', this.name, async (span) => {
|
||||
const maxCount = limit || Infinity;
|
||||
const maxCountPerCall = Math.min(10000, maxCount);
|
||||
const timestamp = Date.now() - grace;
|
||||
let deletedCount = 0;
|
||||
const deletedJobsIds = [];
|
||||
// Normalize 'waiting' to 'wait' for consistency with internal Redis keys
|
||||
const normalizedType = type === 'waiting' ? 'wait' : type;
|
||||
while (deletedCount < maxCount) {
|
||||
const jobsIds = await this.backend.cleanJobsByState(normalizedType, timestamp, maxCountPerCall);
|
||||
this.emit('cleaned', jobsIds, normalizedType);
|
||||
deletedCount += jobsIds.length;
|
||||
deletedJobsIds.push(...jobsIds);
|
||||
if (jobsIds.length < maxCountPerCall) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
span === null || span === void 0 ? void 0 : span.setAttributes({
|
||||
[TelemetryAttributes.QueueGrace]: grace,
|
||||
[TelemetryAttributes.JobType]: type,
|
||||
[TelemetryAttributes.QueueCleanLimit]: maxCount,
|
||||
[TelemetryAttributes.QueueCleanCount]: deletedCount,
|
||||
});
|
||||
return deletedJobsIds;
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Completely destroys the queue and all of its contents irreversibly.
|
||||
* This method will *pause* the queue and requires that there are no
|
||||
* active jobs. It is possible to bypass this requirement, i.e. not
|
||||
* having active jobs using the "force" option.
|
||||
*
|
||||
* Note: This operation requires to iterate on all the jobs stored in the queue
|
||||
* and can be slow for very large queues.
|
||||
*
|
||||
* @param opts - Obliterate options.
|
||||
*/
|
||||
async obliterate(opts) {
|
||||
await this.trace(SpanKind.INTERNAL, 'obliterate', this.name, async () => {
|
||||
await this.pause();
|
||||
let cursor = 0;
|
||||
do {
|
||||
cursor = await this.backend.obliterate(Object.assign({ force: false, count: 1000 }, opts));
|
||||
} while (cursor);
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Retry all the failed or completed jobs.
|
||||
*
|
||||
* @param opts - An object with the following properties:
|
||||
* - count number to limit how many jobs will be moved to wait status per iteration,
|
||||
* - state failed by default or completed.
|
||||
* - timestamp from which timestamp to start moving jobs to wait status, default Date.now().
|
||||
*
|
||||
* @returns
|
||||
*/
|
||||
async retryJobs(opts = {}) {
|
||||
await this.trace(SpanKind.PRODUCER, 'retryJobs', this.name, async (span) => {
|
||||
span === null || span === void 0 ? void 0 : span.setAttributes({
|
||||
[TelemetryAttributes.QueueOptions]: JSON.stringify(opts),
|
||||
});
|
||||
let cursor = 0;
|
||||
do {
|
||||
cursor = await this.backend.retryFinishedJobs(opts.state, opts.count, opts.timestamp);
|
||||
} while (cursor);
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Promote all the delayed jobs.
|
||||
*
|
||||
* @param opts - An object with the following properties:
|
||||
* - count number to limit how many jobs will be moved to wait status per iteration
|
||||
*
|
||||
* @returns
|
||||
*/
|
||||
async promoteJobs(opts = {}) {
|
||||
await this.trace(SpanKind.INTERNAL, 'promoteJobs', this.name, async (span) => {
|
||||
span === null || span === void 0 ? void 0 : span.setAttributes({
|
||||
[TelemetryAttributes.QueueOptions]: JSON.stringify(opts),
|
||||
});
|
||||
let cursor = 0;
|
||||
do {
|
||||
cursor = await this.backend.promoteJobs(opts.count);
|
||||
} while (cursor);
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Trim the event stream to an approximately maxLength.
|
||||
*
|
||||
* @param maxLength - The approximate maximum length, or target length, of the event stream.
|
||||
*/
|
||||
async trimEvents(maxLength) {
|
||||
return this.trace(SpanKind.INTERNAL, 'trimEvents', this.name, async (span) => {
|
||||
span === null || span === void 0 ? void 0 : span.setAttributes({
|
||||
[TelemetryAttributes.QueueEventMaxLength]: maxLength,
|
||||
});
|
||||
return await this.backend.trimEvents(maxLength);
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Delete old priority helper key.
|
||||
*/
|
||||
async removeDeprecatedPriorityKey() {
|
||||
return this.backend.removeDeprecatedPriorityKey();
|
||||
}
|
||||
/**
|
||||
* Removes orphaned job keys that are stored in the backend but are not
|
||||
* referenced in any queue state set.
|
||||
*
|
||||
* Orphaned keys can occur in rare cases when the removal-by-max-age logic
|
||||
* removes state entries without fully cleaning up the corresponding job
|
||||
* data (a regression introduced in v5.66.6 via #3694).
|
||||
* Under normal operation this method is
|
||||
* **not needed** — it is provided only as a one-time migration helper for
|
||||
* users who were affected by that specific bug and want to reclaim the
|
||||
* leaked storage.
|
||||
*
|
||||
* How the scan is performed (its atomicity, batching and how the queue's
|
||||
* state keys are discovered) is an implementation detail of the underlying
|
||||
* backend.
|
||||
*
|
||||
* @param count - Approximate number of keys to scan per iteration (default 1000).
|
||||
* @param limit - Maximum number of orphaned jobs to remove (0 = unlimited).
|
||||
* When set, the method returns as soon as the limit is reached.
|
||||
* Users with a very large number of orphans can call this method
|
||||
* in a loop: `while (await queue.removeOrphanedJobs(1000, 10000)) {}`
|
||||
* @returns The total number of orphaned jobs that were removed.
|
||||
*/
|
||||
async removeOrphanedJobs(count = 1000, limit = 0) {
|
||||
return this.backend.removeOrphanedJobs(count, limit);
|
||||
}
|
||||
}
|
||||
88
node_modules/bullmq/dist/esm/classes/redis-connection.d.ts
generated
vendored
Normal file
88
node_modules/bullmq/dist/esm/classes/redis-connection.d.ts
generated
vendored
Normal file
@@ -0,0 +1,88 @@
|
||||
import { EventEmitter } from 'events';
|
||||
import { ConnectionOptions, RedisOptions, RedisClient } from '../interfaces';
|
||||
import { IRedisClient } from '../interfaces/redis-client';
|
||||
import { DatabaseType } from '../types';
|
||||
interface RedisCapabilities {
|
||||
canDoubleTimeout: boolean;
|
||||
canBlockFor1Ms: boolean;
|
||||
}
|
||||
export interface RawCommand {
|
||||
content: string;
|
||||
name: string;
|
||||
keys: number;
|
||||
}
|
||||
export declare class RedisConnection extends EventEmitter {
|
||||
private readonly extraOptions?;
|
||||
static minimumVersion: string;
|
||||
static recommendedMinimumVersion: string;
|
||||
/**
|
||||
* Optional factory that creates an {@link IRedisClient} from raw options.
|
||||
*
|
||||
* When set, {@link RedisConnection} will call this factory instead of
|
||||
* creating an ioredis client internally. This allows swapping the Redis
|
||||
* driver (e.g. node-redis, Bun built-in) without changing consumer code.
|
||||
*
|
||||
* The factory receives the merged {@link RedisOptions} and must return
|
||||
* an **already-augmented** {@link IRedisClient} (e.g. via
|
||||
* `createNodeRedisClient`).
|
||||
*
|
||||
* @example
|
||||
* ```ts
|
||||
* import { createClient } from 'redis';
|
||||
* import { RedisConnection, createNodeRedisClient } from 'bullmq';
|
||||
*
|
||||
* RedisConnection.clientFactory = (opts) => {
|
||||
* const raw = createClient({ url: `redis://${opts.host ?? '127.0.0.1'}:${opts.port ?? 6379}` });
|
||||
* return createNodeRedisClient(raw);
|
||||
* };
|
||||
* ```
|
||||
*/
|
||||
static clientFactory?: (opts: RedisOptions) => IRedisClient;
|
||||
closing: boolean;
|
||||
capabilities: RedisCapabilities;
|
||||
status: 'initializing' | 'ready' | 'closing' | 'closed';
|
||||
private dbType;
|
||||
protected _client: RedisClient;
|
||||
private readonly opts;
|
||||
private readonly initializing;
|
||||
private version;
|
||||
protected packageVersion: string;
|
||||
private skipVersionCheck;
|
||||
private handleClientError;
|
||||
private handleClientClose;
|
||||
private handleClientReady;
|
||||
private patchedBlockingClusterClient?;
|
||||
private disabledBlockingClusterReconnect;
|
||||
constructor(opts: ConnectionOptions, extraOptions?: {
|
||||
shared?: boolean;
|
||||
blocking?: boolean;
|
||||
skipVersionCheck?: boolean;
|
||||
skipWaitingForReady?: boolean;
|
||||
clusterReconnectTimeoutMs?: number;
|
||||
});
|
||||
private checkBlockingOptions;
|
||||
/**
|
||||
* Waits for a redis client to be ready.
|
||||
* @param redis - client
|
||||
*/
|
||||
static waitUntilReady(client: RedisClient): Promise<void>;
|
||||
get client(): Promise<RedisClient>;
|
||||
protected loadCommands(packageVersion: string, providedScripts?: Record<string, RawCommand>): void;
|
||||
private init;
|
||||
private patchBlockingClusterClient;
|
||||
private disableBlockingClusterReconnect;
|
||||
private releaseBlockingClusterClientPatch;
|
||||
private static isClusterWithEmptyNodes;
|
||||
private static isReconnectingDisabled;
|
||||
private static reconnectClusterIfNeeded;
|
||||
private static shouldReconnectClusterAfterError;
|
||||
private static reconnectCluster;
|
||||
private static connectClusterWithTimeout;
|
||||
disconnect(wait?: boolean): Promise<void>;
|
||||
reconnect(): Promise<void>;
|
||||
close(force?: boolean): Promise<void>;
|
||||
private getRedisVersionAndType;
|
||||
get redisVersion(): string;
|
||||
get databaseType(): DatabaseType;
|
||||
}
|
||||
export {};
|
||||
536
node_modules/bullmq/dist/esm/classes/redis-connection.js
generated
vendored
Normal file
536
node_modules/bullmq/dist/esm/classes/redis-connection.js
generated
vendored
Normal file
@@ -0,0 +1,536 @@
|
||||
import { __rest } from "tslib";
|
||||
import { EventEmitter } from 'events';
|
||||
import { default as IORedis } from 'ioredis';
|
||||
import { decreaseMaxListeners, increaseMaxListeners, isNotConnectionError, isRedisCluster, isRedisInstance, isRedisVersionLowerThan, } from '../utils';
|
||||
import { version as packageVersion } from '../version';
|
||||
import * as scripts from '../scripts';
|
||||
import { createIORedisClient, isIRedisClient } from './ioredis-client';
|
||||
import { ConnectionClosedError, CONNECTION_CLOSED_ERROR_MSG, } from './errors/connection-closed-error';
|
||||
const overrideMessage = [
|
||||
'BullMQ: WARNING! Your redis options maxRetriesPerRequest must be null',
|
||||
'and will be overridden by BullMQ.',
|
||||
].join(' ');
|
||||
const deprecationMessage = 'BullMQ: Your redis options maxRetriesPerRequest must be null.';
|
||||
const clusterReconnectPromise = Symbol('bullmqClusterReconnectPromise');
|
||||
const clusterPatchedForBlocking = Symbol('bullmqClusterPatchedForBlocking');
|
||||
const clusterOriginalBzpopmin = Symbol('bullmqClusterOriginalBzpopmin');
|
||||
const clusterWrappedBzpopmin = Symbol('bullmqClusterWrappedBzpopmin');
|
||||
const clusterPatchRefCount = Symbol('bullmqClusterPatchRefCount');
|
||||
const clusterClosingRefCount = Symbol('bullmqClusterClosingRefCount');
|
||||
// Hard cap on a single cluster reconnect attempt. Without this, a hung
|
||||
// `client.connect()` (e.g. ioredis Cluster cannot recover and slot refresh
|
||||
// keeps retrying internally) leaves `clusterReconnectPromise` pinned forever
|
||||
// and every subsequent bzpopmin call awaits the same dead promise, deadlocking
|
||||
// the worker. 30s is comfortably above the default ioredis slotsRefreshTimeout
|
||||
// while bounded enough that wedged workers recover within one or two ticks.
|
||||
const DEFAULT_CLUSTER_RECONNECT_TIMEOUT_MS = 30000;
|
||||
export class RedisConnection extends EventEmitter {
|
||||
constructor(opts, extraOptions) {
|
||||
super();
|
||||
this.extraOptions = extraOptions;
|
||||
this.capabilities = {
|
||||
canDoubleTimeout: false,
|
||||
canBlockFor1Ms: true,
|
||||
};
|
||||
this.status = 'initializing';
|
||||
this.dbType = 'redis';
|
||||
this.packageVersion = packageVersion;
|
||||
this.disabledBlockingClusterReconnect = false;
|
||||
// Set extra options defaults
|
||||
this.extraOptions = Object.assign({ shared: false, blocking: true, skipVersionCheck: false, skipWaitingForReady: false, clusterReconnectTimeoutMs: DEFAULT_CLUSTER_RECONNECT_TIMEOUT_MS }, extraOptions);
|
||||
if (!isRedisInstance(opts)) {
|
||||
this.checkBlockingOptions(overrideMessage, opts);
|
||||
this.opts = Object.assign({ port: 6379, host: '127.0.0.1', retryStrategy: function (times) {
|
||||
return Math.max(Math.min(Math.exp(times), 20000), 1000);
|
||||
} }, opts);
|
||||
if (this.extraOptions.blocking) {
|
||||
this.opts.maxRetriesPerRequest = null;
|
||||
}
|
||||
}
|
||||
else {
|
||||
// Wrap raw ioredis instances in the IRedisClient adapter if not already wrapped
|
||||
this._client = isIRedisClient(opts)
|
||||
? opts
|
||||
: createIORedisClient(opts);
|
||||
// Test if the redis instance is using keyPrefix
|
||||
// and if so, throw an error.
|
||||
if (this._client.options.keyPrefix) {
|
||||
throw new Error('BullMQ: ioredis does not support ioredis prefixes, use the prefix option instead.');
|
||||
}
|
||||
if (this._client.isCluster) {
|
||||
this.opts = this._client.options.redisOptions;
|
||||
}
|
||||
else {
|
||||
this.opts = this._client.options;
|
||||
}
|
||||
this.checkBlockingOptions(deprecationMessage, this.opts, true);
|
||||
}
|
||||
this.skipVersionCheck =
|
||||
(extraOptions === null || extraOptions === void 0 ? void 0 : extraOptions.skipVersionCheck) ||
|
||||
!!(this.opts && this.opts.skipVersionCheck);
|
||||
this.handleClientError = (err) => {
|
||||
this.emit('error', err);
|
||||
};
|
||||
this.handleClientClose = () => {
|
||||
this.emit('close');
|
||||
};
|
||||
this.handleClientReady = () => {
|
||||
this.emit('ready');
|
||||
};
|
||||
this.initializing = this.init();
|
||||
this.initializing.catch(err => {
|
||||
// Only emit if there is an `error` listener attached. `EventEmitter.emit`
|
||||
// throws when emitting `error` with no listeners, which would surface as
|
||||
// an unhandled rejection — e.g. when the connection is force-closed during
|
||||
// shutdown while its version-check `INFO` command is still in flight and
|
||||
// rejects with "Connection is closed". The init error is still propagated
|
||||
// to anything awaiting `initializing` (the `client` getter and `close`).
|
||||
if (this.listenerCount('error') > 0) {
|
||||
this.emit('error', err);
|
||||
}
|
||||
});
|
||||
}
|
||||
checkBlockingOptions(msg, options, throwError = false) {
|
||||
if (this.extraOptions.blocking && options && options.maxRetriesPerRequest) {
|
||||
if (throwError) {
|
||||
throw new Error(msg);
|
||||
}
|
||||
else {
|
||||
console.error(msg);
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Waits for a redis client to be ready.
|
||||
* @param redis - client
|
||||
*/
|
||||
static async waitUntilReady(client) {
|
||||
if (client.status === 'ready') {
|
||||
return;
|
||||
}
|
||||
// ioredis Cluster reports 'connect' as its connected status instead of
|
||||
// 'ready' that standalone Redis uses. Treat it as already connected so we
|
||||
// don't hang waiting for a 'ready' event that will never fire and end up
|
||||
// throwing a spurious "Connection is closed" error during disconnect.
|
||||
if (client.status === 'connect' && isRedisCluster(client)) {
|
||||
return;
|
||||
}
|
||||
if (client.status === 'wait') {
|
||||
return client.connect();
|
||||
}
|
||||
if (client.status === 'end') {
|
||||
throw new ConnectionClosedError(CONNECTION_CLOSED_ERROR_MSG);
|
||||
}
|
||||
let handleReady;
|
||||
let handleEnd;
|
||||
let handleError;
|
||||
try {
|
||||
await new Promise((resolve, reject) => {
|
||||
let lastError;
|
||||
handleError = (err) => {
|
||||
lastError = err;
|
||||
};
|
||||
handleReady = () => {
|
||||
resolve();
|
||||
};
|
||||
handleEnd = () => {
|
||||
if (client.status !== 'end') {
|
||||
reject(lastError ||
|
||||
new ConnectionClosedError(CONNECTION_CLOSED_ERROR_MSG));
|
||||
}
|
||||
else {
|
||||
if (lastError) {
|
||||
reject(lastError);
|
||||
}
|
||||
else {
|
||||
// when custom 'end' status is set we already closed
|
||||
resolve();
|
||||
}
|
||||
}
|
||||
};
|
||||
increaseMaxListeners(client, 3);
|
||||
client.once('ready', handleReady);
|
||||
client.on('end', handleEnd);
|
||||
client.once('error', handleError);
|
||||
});
|
||||
}
|
||||
finally {
|
||||
client.removeListener('end', handleEnd);
|
||||
client.removeListener('error', handleError);
|
||||
client.removeListener('ready', handleReady);
|
||||
decreaseMaxListeners(client, 3);
|
||||
}
|
||||
}
|
||||
get client() {
|
||||
return this.initializing;
|
||||
}
|
||||
loadCommands(packageVersion, providedScripts) {
|
||||
const finalScripts = providedScripts || scripts;
|
||||
for (const property in finalScripts) {
|
||||
// Only define the command if not already defined
|
||||
const commandName = `${finalScripts[property].name}:${packageVersion}`;
|
||||
if (!this._client[commandName]) {
|
||||
this._client.defineCommand(commandName, {
|
||||
numberOfKeys: finalScripts[property].keys,
|
||||
lua: finalScripts[property].content,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
async init() {
|
||||
if (!this._client) {
|
||||
if (RedisConnection.clientFactory) {
|
||||
this._client = RedisConnection.clientFactory(this.opts);
|
||||
}
|
||||
else {
|
||||
const _a = this.opts, { url } = _a, rest = __rest(_a, ["url"]);
|
||||
const ioredisClient = url ? new IORedis(url, rest) : new IORedis(rest);
|
||||
this._client = createIORedisClient(ioredisClient);
|
||||
}
|
||||
}
|
||||
increaseMaxListeners(this._client, 3);
|
||||
this._client.on('error', this.handleClientError);
|
||||
// ioredis treats connection errors as a different event ('close')
|
||||
this._client.on('close', this.handleClientClose);
|
||||
this._client.on('ready', this.handleClientReady);
|
||||
this.patchBlockingClusterClient();
|
||||
if (!this.extraOptions.skipWaitingForReady) {
|
||||
await RedisConnection.waitUntilReady(this._client);
|
||||
}
|
||||
this.loadCommands(this.packageVersion);
|
||||
if (this._client['status'] !== 'end') {
|
||||
const versionResult = await this.getRedisVersionAndType();
|
||||
this.version = versionResult.version;
|
||||
this.dbType = versionResult.databaseType;
|
||||
if (this.skipVersionCheck !== true && !this.closing) {
|
||||
if (isRedisVersionLowerThan(this.version, RedisConnection.minimumVersion, this.dbType)) {
|
||||
throw new Error(`Redis version needs to be greater or equal than ${RedisConnection.minimumVersion} ` +
|
||||
`Current: ${this.version}`);
|
||||
}
|
||||
if (isRedisVersionLowerThan(this.version, RedisConnection.recommendedMinimumVersion, this.dbType)) {
|
||||
console.warn(`It is highly recommended to use a minimum Redis version of ${RedisConnection.recommendedMinimumVersion}
|
||||
Current: ${this.version}`);
|
||||
}
|
||||
}
|
||||
this.capabilities = {
|
||||
canDoubleTimeout: !isRedisVersionLowerThan(this.version, '6.0.0', this.dbType),
|
||||
canBlockFor1Ms: !isRedisVersionLowerThan(this.version, '7.0.8', this.dbType),
|
||||
};
|
||||
this.status = 'ready';
|
||||
}
|
||||
return this._client;
|
||||
}
|
||||
patchBlockingClusterClient() {
|
||||
var _a;
|
||||
const client = this._client;
|
||||
const blockingClient = client;
|
||||
if (!this.extraOptions.blocking ||
|
||||
!isRedisCluster(client) ||
|
||||
typeof blockingClient.bzpopmin !== 'function') {
|
||||
return;
|
||||
}
|
||||
const reconnectTimeoutMs = (_a = this.extraOptions.clusterReconnectTimeoutMs) !== null && _a !== void 0 ? _a : DEFAULT_CLUSTER_RECONNECT_TIMEOUT_MS;
|
||||
blockingClient[clusterPatchRefCount] =
|
||||
(blockingClient[clusterPatchRefCount] || 0) + 1;
|
||||
this.patchedBlockingClusterClient = blockingClient;
|
||||
if (blockingClient[clusterPatchedForBlocking]) {
|
||||
return;
|
||||
}
|
||||
const bzpopmin = blockingClient.bzpopmin;
|
||||
const wrappedBzpopmin = async (...args) => {
|
||||
await RedisConnection.reconnectClusterIfNeeded(blockingClient, reconnectTimeoutMs);
|
||||
try {
|
||||
return await bzpopmin.apply(blockingClient, args);
|
||||
}
|
||||
catch (error) {
|
||||
const commandError = error;
|
||||
if (RedisConnection.shouldReconnectClusterAfterError(blockingClient, commandError)) {
|
||||
try {
|
||||
await RedisConnection.reconnectCluster(blockingClient, reconnectTimeoutMs);
|
||||
}
|
||||
catch (_a) {
|
||||
// Preserve the original command failure if best-effort recovery fails.
|
||||
}
|
||||
}
|
||||
throw commandError;
|
||||
}
|
||||
};
|
||||
blockingClient[clusterOriginalBzpopmin] = bzpopmin;
|
||||
blockingClient[clusterWrappedBzpopmin] = wrappedBzpopmin;
|
||||
blockingClient[clusterPatchedForBlocking] = true;
|
||||
blockingClient.bzpopmin = wrappedBzpopmin;
|
||||
}
|
||||
disableBlockingClusterReconnect() {
|
||||
const client = this.patchedBlockingClusterClient;
|
||||
if (!client || this.disabledBlockingClusterReconnect) {
|
||||
return;
|
||||
}
|
||||
client[clusterClosingRefCount] = (client[clusterClosingRefCount] || 0) + 1;
|
||||
this.disabledBlockingClusterReconnect = true;
|
||||
}
|
||||
releaseBlockingClusterClientPatch() {
|
||||
const client = this.patchedBlockingClusterClient;
|
||||
if (!client) {
|
||||
return;
|
||||
}
|
||||
if (this.disabledBlockingClusterReconnect) {
|
||||
const closingRefCount = (client[clusterClosingRefCount] || 1) - 1;
|
||||
if (closingRefCount > 0) {
|
||||
client[clusterClosingRefCount] = closingRefCount;
|
||||
}
|
||||
else {
|
||||
delete client[clusterClosingRefCount];
|
||||
}
|
||||
this.disabledBlockingClusterReconnect = false;
|
||||
}
|
||||
const patchRefCount = (client[clusterPatchRefCount] || 1) - 1;
|
||||
if (patchRefCount > 0) {
|
||||
client[clusterPatchRefCount] = patchRefCount;
|
||||
this.patchedBlockingClusterClient = undefined;
|
||||
return;
|
||||
}
|
||||
if (client[clusterOriginalBzpopmin] &&
|
||||
client.bzpopmin === client[clusterWrappedBzpopmin]) {
|
||||
client.bzpopmin = client[clusterOriginalBzpopmin];
|
||||
}
|
||||
delete client[clusterPatchRefCount];
|
||||
delete client[clusterClosingRefCount];
|
||||
delete client[clusterOriginalBzpopmin];
|
||||
delete client[clusterWrappedBzpopmin];
|
||||
delete client[clusterPatchedForBlocking];
|
||||
this.patchedBlockingClusterClient = undefined;
|
||||
}
|
||||
static isClusterWithEmptyNodes(client) {
|
||||
return typeof client.nodes === 'function' && client.nodes().length === 0;
|
||||
}
|
||||
static isReconnectingDisabled(client) {
|
||||
const patchRefCount = client[clusterPatchRefCount] || 0;
|
||||
const closingRefCount = client[clusterClosingRefCount] || 0;
|
||||
return (patchRefCount === 0 ||
|
||||
closingRefCount >= patchRefCount ||
|
||||
client.status === 'end' ||
|
||||
client.status === 'closing');
|
||||
}
|
||||
static async reconnectClusterIfNeeded(client, timeoutMs) {
|
||||
if (!RedisConnection.isReconnectingDisabled(client) &&
|
||||
RedisConnection.isClusterWithEmptyNodes(client)) {
|
||||
await RedisConnection.reconnectCluster(client, timeoutMs);
|
||||
}
|
||||
}
|
||||
static shouldReconnectClusterAfterError(client, error) {
|
||||
var _a, _b;
|
||||
if (RedisConnection.isReconnectingDisabled(client)) {
|
||||
return false;
|
||||
}
|
||||
const message = [
|
||||
error.message,
|
||||
(_a = error.cause) === null || _a === void 0 ? void 0 : _a.message,
|
||||
(_b = error.lastNodeError) === null || _b === void 0 ? void 0 : _b.message,
|
||||
].join(' ');
|
||||
return (RedisConnection.isClusterWithEmptyNodes(client) ||
|
||||
/Command timed out|Failed to refresh slots cache/i.test(message));
|
||||
}
|
||||
static async reconnectCluster(client, timeoutMs) {
|
||||
if (RedisConnection.isReconnectingDisabled(client)) {
|
||||
return;
|
||||
}
|
||||
if (!client[clusterReconnectPromise]) {
|
||||
client[clusterReconnectPromise] =
|
||||
RedisConnection.connectClusterWithTimeout(client, timeoutMs).finally(() => {
|
||||
client[clusterReconnectPromise] = null;
|
||||
});
|
||||
}
|
||||
await client[clusterReconnectPromise];
|
||||
}
|
||||
// Disconnects and reconnects a cluster client, racing connect() against a
|
||||
// hard cap so a hung reconnect cannot pin `clusterReconnectPromise`
|
||||
// indefinitely. On timeout, the underlying connect() is left running
|
||||
// (ioredis Cluster handles its own retry/state); we simply stop awaiting it.
|
||||
// The next bzpopmin call's `reconnectClusterIfNeeded` check will trigger a
|
||||
// fresh reconnect if the pool is still empty.
|
||||
static async connectClusterWithTimeout(client, timeoutMs) {
|
||||
client.disconnect(false);
|
||||
let timeoutHandle;
|
||||
try {
|
||||
await Promise.race([
|
||||
client.connect(),
|
||||
new Promise((_, reject) => {
|
||||
var _a;
|
||||
timeoutHandle = setTimeout(() => {
|
||||
reject(new ConnectionClosedError(`BullMQ: cluster reconnect timed out after ${timeoutMs}ms`));
|
||||
}, timeoutMs);
|
||||
// Don't keep the event loop alive solely for this timer.
|
||||
(_a = timeoutHandle.unref) === null || _a === void 0 ? void 0 : _a.call(timeoutHandle);
|
||||
}),
|
||||
]);
|
||||
}
|
||||
finally {
|
||||
if (timeoutHandle) {
|
||||
clearTimeout(timeoutHandle);
|
||||
}
|
||||
}
|
||||
}
|
||||
async disconnect(wait = true) {
|
||||
const client = await this.client;
|
||||
if (client.status !== 'end') {
|
||||
let _resolve, _reject;
|
||||
if (!wait) {
|
||||
return client.disconnect();
|
||||
}
|
||||
const disconnecting = new Promise((resolve, reject) => {
|
||||
increaseMaxListeners(client, 2);
|
||||
client.once('end', resolve);
|
||||
client.once('error', reject);
|
||||
_resolve = resolve;
|
||||
_reject = reject;
|
||||
});
|
||||
client.disconnect();
|
||||
try {
|
||||
await disconnecting;
|
||||
}
|
||||
finally {
|
||||
decreaseMaxListeners(client, 2);
|
||||
client.removeListener('end', _resolve);
|
||||
client.removeListener('error', _reject);
|
||||
}
|
||||
}
|
||||
}
|
||||
async reconnect() {
|
||||
const client = await this.client;
|
||||
for (;;) {
|
||||
if (client.status === 'ready' ||
|
||||
(client.status === 'connect' && isRedisCluster(client))) {
|
||||
return;
|
||||
}
|
||||
if (client.status === 'wait' || client.status === 'end') {
|
||||
return client.connect();
|
||||
}
|
||||
try {
|
||||
await RedisConnection.waitUntilReady(client);
|
||||
}
|
||||
catch (error) {
|
||||
if (!['end', 'connecting', 'connect', 'reconnecting'].includes(client.status)) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
async close(force = false) {
|
||||
var _a;
|
||||
if (!this.closing) {
|
||||
const status = this.status;
|
||||
this.status = 'closing';
|
||||
this.closing = true;
|
||||
this.disableBlockingClusterReconnect();
|
||||
try {
|
||||
if (status === 'ready') {
|
||||
// Not sure if we need to wait for this
|
||||
await this.initializing;
|
||||
}
|
||||
if (!this.extraOptions.shared) {
|
||||
if (status == 'initializing' || force) {
|
||||
// If we have not still connected to Redis, we need to disconnect.
|
||||
this._client.disconnect();
|
||||
// Suppress any rejection from the in-flight init() so it doesn't
|
||||
// become an unhandled rejection after we close the connection.
|
||||
(_a = this.initializing) === null || _a === void 0 ? void 0 : _a.catch(() => { });
|
||||
}
|
||||
else {
|
||||
await this._client.quit();
|
||||
}
|
||||
// As IORedis does not update this status properly, we do it ourselves.
|
||||
this._client['status'] = 'end';
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
if (isNotConnectionError(error)) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
finally {
|
||||
this.releaseBlockingClusterClientPatch();
|
||||
this._client.off('error', this.handleClientError);
|
||||
this._client.off('close', this.handleClientClose);
|
||||
this._client.off('ready', this.handleClientReady);
|
||||
decreaseMaxListeners(this._client, 3);
|
||||
this.removeAllListeners();
|
||||
this.status = 'closed';
|
||||
}
|
||||
}
|
||||
}
|
||||
async getRedisVersionAndType() {
|
||||
if (this.skipVersionCheck) {
|
||||
return {
|
||||
version: RedisConnection.minimumVersion,
|
||||
databaseType: 'redis',
|
||||
};
|
||||
}
|
||||
const doc = await this._client.info();
|
||||
const redisPrefix = 'redis_version:';
|
||||
const maxMemoryPolicyPrefix = 'maxmemory_policy:';
|
||||
const lines = doc.split(/\r?\n/);
|
||||
let redisVersion;
|
||||
let databaseType = 'redis';
|
||||
// Detect database type from server info
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
// Check for Dragonfly
|
||||
if (line.includes('dragonfly_version:') ||
|
||||
line.includes('server:Dragonfly')) {
|
||||
databaseType = 'dragonfly';
|
||||
// For Dragonfly, extract version from dragonfly_version field
|
||||
if (line.indexOf('dragonfly_version:') === 0) {
|
||||
redisVersion = line.substr('dragonfly_version:'.length);
|
||||
}
|
||||
}
|
||||
// Check for Valkey
|
||||
else if (line.includes('valkey_version:') ||
|
||||
line.includes('server:Valkey')) {
|
||||
databaseType = 'valkey';
|
||||
// For Valkey, extract version from valkey_version field
|
||||
if (line.indexOf('valkey_version:') === 0) {
|
||||
redisVersion = line.substr('valkey_version:'.length);
|
||||
}
|
||||
}
|
||||
// Standard Redis version detection
|
||||
else if (line.indexOf(redisPrefix) === 0) {
|
||||
redisVersion = line.substr(redisPrefix.length);
|
||||
// Keep Redis as default unless we find evidence of other databases above
|
||||
if (databaseType === 'redis') {
|
||||
databaseType = 'redis';
|
||||
}
|
||||
}
|
||||
if (line.indexOf(maxMemoryPolicyPrefix) === 0) {
|
||||
const maxMemoryPolicy = line.substr(maxMemoryPolicyPrefix.length);
|
||||
if (maxMemoryPolicy !== 'noeviction') {
|
||||
console.warn(`IMPORTANT! Eviction policy is ${maxMemoryPolicy}. It should be "noeviction"`);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Fallback version detection if specific database version field wasn't found
|
||||
if (!redisVersion) {
|
||||
// Try to find any version field as fallback
|
||||
for (const line of lines) {
|
||||
if (line.includes('version:')) {
|
||||
const parts = line.split(':');
|
||||
if (parts.length >= 2) {
|
||||
redisVersion = parts[1];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return {
|
||||
version: redisVersion || RedisConnection.minimumVersion,
|
||||
databaseType,
|
||||
};
|
||||
}
|
||||
get redisVersion() {
|
||||
return this.version;
|
||||
}
|
||||
get databaseType() {
|
||||
return this.dbType;
|
||||
}
|
||||
}
|
||||
RedisConnection.minimumVersion = '5.0.0';
|
||||
RedisConnection.recommendedMinimumVersion = '6.2.0';
|
||||
436
node_modules/bullmq/dist/esm/classes/redis-queue-backend.d.ts
generated
vendored
Normal file
436
node_modules/bullmq/dist/esm/classes/redis-queue-backend.d.ts
generated
vendored
Normal file
@@ -0,0 +1,436 @@
|
||||
/**
|
||||
* Includes all the scripts needed by the queue and jobs.
|
||||
*/
|
||||
import { EventEmitter } from 'events';
|
||||
import { DependenciesOpts, IQueueBackend, JobJson, MinimalJob, MoveToWaitingChildrenOpts, ParentKeyOpts, RedisClient, MoveToDelayedOpts, RepeatableOptions, RetryJobOpts, RetryOptions, ScriptQueueContext, StreamReadRaw } from '../interfaces';
|
||||
import { JobsOptions, JobState, JobType, FinishedStatus, FinishedPropValAttribute, KeepJobs, JobProgress } from '../types';
|
||||
import { IRedisTransaction } from '../interfaces';
|
||||
import { QueueBaseOptions } from '../interfaces';
|
||||
import { KeysMap } from './queue-keys';
|
||||
import { RedisConnection } from './redis-connection';
|
||||
export type JobData = [JobJson | number, string?];
|
||||
export declare class RedisQueueBackend extends EventEmitter implements IQueueBackend {
|
||||
connection: RedisConnection;
|
||||
protected readonly name: string;
|
||||
blockingConnection?: RedisConnection;
|
||||
protected ownsConnection: boolean;
|
||||
protected version: string;
|
||||
moveToFinishedKeys: (string | undefined)[];
|
||||
/**
|
||||
* Resolves once a close has been initiated. Owned by the backend (it owns the
|
||||
* underlying connection(s)).
|
||||
*/
|
||||
closing: Promise<void> | undefined;
|
||||
/**
|
||||
* Internal Redis access context (client, version, keys, …). Built from the
|
||||
* owned connection(s); kept private to this adapter.
|
||||
*/
|
||||
protected queue: ScriptQueueContext;
|
||||
/**
|
||||
* The resolved key prefix (defaults to `bull`). A Redis-specific concept used
|
||||
* to namespace this queue's keys, qualified name and client name.
|
||||
*/
|
||||
protected readonly redisPrefix: string;
|
||||
constructor(connection: RedisConnection, name: string, keys: KeysMap, toKey: (type: string) => string, opts: QueueBaseOptions, blockingConnection?: RedisConnection, ownsConnection?: boolean);
|
||||
/**
|
||||
* Returns a sibling backend bound to a different queue that shares this
|
||||
* backend's connection(s). Used by {@link FlowProducer} to operate on the
|
||||
* many queues that a flow may span over a single connection. The sibling
|
||||
* does not own the connection, so its `close`/`disconnect` are no-ops.
|
||||
*/
|
||||
forQueue(queueName: string, prefix?: string): IQueueBackend;
|
||||
/**
|
||||
* The queue's fully-qualified name (`"<prefix>:<queue>"`). This is the
|
||||
* cross-backend logical identifier (e.g. used as a flow parent reference).
|
||||
*/
|
||||
get qualifiedName(): string;
|
||||
/**
|
||||
* The concrete Redis keys for this queue (wait, active, events, …).
|
||||
*/
|
||||
get keys(): KeysMap;
|
||||
/**
|
||||
* Builds a namespaced Redis sub-key of the given `type`
|
||||
* (`"<prefix>:<queue>:<type>"`).
|
||||
*/
|
||||
toKey(type: string): string;
|
||||
/**
|
||||
* Parses a Redis flow child key (`"<prefix>:<queue>:<id>"`) into its
|
||||
* components. Inverse of {@link toKey}.
|
||||
*/
|
||||
parseNodeKey(key: string): {
|
||||
prefix: string;
|
||||
queueName: string;
|
||||
id: string;
|
||||
};
|
||||
/**
|
||||
* Builds the Redis client name (`"<prefix>:<base64(queue)><suffix>"`), used
|
||||
* for `CLIENT SETNAME` and worker/queue discovery via `CLIENT LIST`.
|
||||
*/
|
||||
clientName(suffix?: string): string;
|
||||
/**
|
||||
* Normalizes the events of the owned connection(s) into the backend's own
|
||||
* `'ready' | 'error' | 'close'` events.
|
||||
*/
|
||||
private forwardConnectionEvents;
|
||||
/**
|
||||
* Resolves once the backend's underlying connection(s) are ready.
|
||||
*/
|
||||
waitUntilReady(): Promise<void>;
|
||||
/**
|
||||
* Closes the backend and its underlying connection(s).
|
||||
*
|
||||
* The dedicated blocking connection (if any) is closed first so that an
|
||||
* in-flight blocking command (e.g. `bzpopmin`) is interrupted before the
|
||||
* main connection is closed.
|
||||
*/
|
||||
close(force?: boolean): Promise<void>;
|
||||
/**
|
||||
* Forcibly disconnects the backend's underlying connection(s).
|
||||
*/
|
||||
disconnect(): Promise<void>;
|
||||
/**
|
||||
* Sets a human-readable name on the underlying connection (CLIENT SETNAME).
|
||||
* Unsupported-command and shutdown errors are swallowed.
|
||||
*/
|
||||
setName(name: string): Promise<void>;
|
||||
/**
|
||||
* The raw Redis client. Redis-specific escape hatch (used e.g. by
|
||||
* `Queue.client`); not part of {@link IQueueBackend}.
|
||||
*/
|
||||
get client(): Promise<RedisClient>;
|
||||
/**
|
||||
* The raw blocking Redis client (a dedicated connection used for the
|
||||
* blocking `waitForJob` primitive), if this backend was created with one.
|
||||
* Redis-specific escape hatch; not part of {@link IQueueBackend}.
|
||||
*/
|
||||
get blockingClient(): Promise<RedisClient> | undefined;
|
||||
/**
|
||||
* The detected Redis server version. Redis-specific escape hatch; not part
|
||||
* of {@link IQueueBackend}.
|
||||
*/
|
||||
get redisVersion(): string;
|
||||
/**
|
||||
* The detected datastore flavour (`redis`, `dragonfly`, `valkey`, …).
|
||||
* Redis-specific escape hatch; not part of {@link IQueueBackend}.
|
||||
*/
|
||||
get databaseType(): string;
|
||||
/**
|
||||
* Smallest meaningful block timeout (seconds) given the blocking
|
||||
* connection's capabilities.
|
||||
*/
|
||||
get minimumBlockTimeout(): number;
|
||||
/**
|
||||
* Interrupts the in-flight blocking wait by disconnecting the dedicated
|
||||
* blocking connection. No-op if there is none.
|
||||
*/
|
||||
disconnectBlocking(wait?: boolean): Promise<void>;
|
||||
/**
|
||||
* Re-establishes the dedicated blocking connection after an interrupt.
|
||||
*/
|
||||
reconnectBlocking(): Promise<void>;
|
||||
/**
|
||||
* Executes a registered Lua script on the given Redis client, resolving the
|
||||
* versioned command name (e.g. `addJob:<packageVersion>`) so the script
|
||||
* belonging to the current BullMQ version is invoked.
|
||||
*
|
||||
* @param client - The Redis client or pipeline/transaction on which to run the command.
|
||||
* @param commandName - The base name of the Lua script (without version suffix).
|
||||
* @param args - Positional arguments forwarded to the Lua script (keys followed by argv).
|
||||
* @returns The raw result produced by the Lua script.
|
||||
*
|
||||
* @private
|
||||
*/
|
||||
execCommand(client: RedisClient | IRedisTransaction, commandName: string, args: any[]): any;
|
||||
/**
|
||||
* Checks whether a job with the given id is present in the provided queue
|
||||
* state.
|
||||
*/
|
||||
isJobInState(state: string, jobId: string): Promise<boolean>;
|
||||
protected addDelayedJobArgs(job: JobJson, encodedOpts: any, args: (string | number | Record<string, any>)[], keysMap?: KeysMap): (string | Buffer)[];
|
||||
protected addDelayedJob(client: RedisClient | IRedisTransaction, job: JobJson, encodedOpts: any, args: (string | number | Record<string, any>)[], keys?: KeysMap): Promise<string | number>;
|
||||
protected addPrioritizedJobArgs(job: JobJson, encodedOpts: any, args: (string | number | Record<string, any>)[], keysMap?: KeysMap): (string | Buffer)[];
|
||||
protected addPrioritizedJob(client: RedisClient | IRedisTransaction, job: JobJson, encodedOpts: any, args: (string | number | Record<string, any>)[], keys?: KeysMap): Promise<string | number>;
|
||||
protected addParentJobArgs(job: JobJson, encodedOpts: any, args: (string | number | Record<string, any>)[], keysMap?: KeysMap): (string | Buffer)[];
|
||||
protected addParentJob(client: RedisClient | IRedisTransaction, job: JobJson, encodedOpts: any, args: (string | number | Record<string, any>)[], keys?: KeysMap): Promise<string | number>;
|
||||
protected addStandardJobArgs(job: JobJson, encodedOpts: any, args: (string | number | Record<string, any>)[], keysMap?: KeysMap): (string | Buffer)[];
|
||||
protected addStandardJob(client: RedisClient | IRedisTransaction, job: JobJson, encodedOpts: any, args: (string | number | Record<string, any>)[], keys?: KeysMap): Promise<string | number>;
|
||||
/**
|
||||
* Low-level Redis adapter helper: queues/executes a single job insert on the
|
||||
* provided client or transaction (pipeline/multi). This is the only place
|
||||
* that needs a connection handle; the public {@link addJob} / {@link addJobs}
|
||||
* operations obtain it from the backend itself.
|
||||
*
|
||||
* Kept public (but outside {@link IQueueBackend}) so that flow producers can
|
||||
* batch inserts across queues onto a shared transaction.
|
||||
*/
|
||||
addJobToTransaction(client: RedisClient | IRedisTransaction, job: JobJson, jobId: string, parentKeyOpts?: ParentKeyOpts, keys?: KeysMap): Promise<string>;
|
||||
addJob(job: JobJson, jobId: string, parentKeyOpts?: ParentKeyOpts): Promise<string>;
|
||||
addJobs(entries: {
|
||||
job: JobJson;
|
||||
jobId: string;
|
||||
parentKeyOpts?: ParentKeyOpts;
|
||||
}[]): Promise<string[]>;
|
||||
/**
|
||||
* Atomically inserts a whole flow (tree) of jobs that may span multiple
|
||||
* queues, returning one `[error, idOrCode]` tuple per entry in the same
|
||||
* order they were provided. For the Redis adapter this is a single `MULTI`
|
||||
* transaction; another backend would use a single SQL transaction.
|
||||
*
|
||||
* Each entry is self-describing (it carries its own queue `prefix` and
|
||||
* `queueName`), so the operation does not need to be bound to a single
|
||||
* queue's key map.
|
||||
*/
|
||||
addFlow(entries: {
|
||||
jobData: JobJson;
|
||||
jobId: string;
|
||||
parentKeyOpts: ParentKeyOpts;
|
||||
prefix: string;
|
||||
queueName: string;
|
||||
}[]): Promise<[Error | null, string | number][]>;
|
||||
protected pauseArgs(pause: boolean, emitEvent?: boolean): (string | number)[];
|
||||
pause(pause: boolean): Promise<void>;
|
||||
/**
|
||||
* Removes a deduplication key from Redis so that a new job with the same
|
||||
* deduplication id can be enqueued again. The key is only removed if it
|
||||
* currently maps to the provided `jobId`, preventing races between
|
||||
* producers and finishing jobs.
|
||||
*
|
||||
* @param deduplicationId - The deduplication id whose key should be cleared.
|
||||
* @param jobId - The id of the job that currently owns the dedup key.
|
||||
* @returns `1` if the key was removed, `0` otherwise.
|
||||
*
|
||||
* @private
|
||||
*/
|
||||
removeDeduplicationKey(deduplicationId: string, jobId: string): Promise<number>;
|
||||
/**
|
||||
* Registers a job scheduler and enqueues its next delayed iteration.
|
||||
* The scheduler stores the template data/options so subsequent iterations
|
||||
* can be produced automatically based on the repeat options.
|
||||
*
|
||||
* @param jobSchedulerId - The id that uniquely identifies this scheduler.
|
||||
* @param nextMillis - Timestamp (ms since epoch) for the next iteration.
|
||||
* @param templateData - Serialized template data reused for every iteration.
|
||||
* @param templateOpts - Redis-encoded job options applied to every iteration.
|
||||
* @param opts - Repeat options describing the scheduling pattern.
|
||||
* @param delayedJobOpts - Options applied to the next delayed job that is produced.
|
||||
* @param producerId - Optional id of the job that produced this iteration, used to prevent duplicates.
|
||||
* @returns A tuple of `[jobId, delay]`, where `delay` is the computed delay in milliseconds
|
||||
* for the next iteration. When `delay` is `0`, the job is enqueued immediately.
|
||||
* @throws An error resolved from `finishedErrors` when the Lua script returns a negative status code.
|
||||
*
|
||||
* @private
|
||||
*/
|
||||
addJobScheduler(jobSchedulerId: string, nextMillis: number, templateData: string, templateOpts: JobsOptions, opts: RepeatableOptions, delayedJobOpts: JobsOptions, producerId?: string): Promise<[string, number]>;
|
||||
updateJobSchedulerNextMillis(jobSchedulerId: string, nextMillis: number, templateData: string, delayedJobOpts: JobsOptions, producerId?: string): Promise<string | null>;
|
||||
removeJobScheduler(jobSchedulerId: string): Promise<number>;
|
||||
protected removeArgs(jobId: string, removeChildren: boolean): (string | number)[];
|
||||
remove(jobId: string, removeChildren: boolean): Promise<number>;
|
||||
removeUnprocessedChildren(jobId: string): Promise<void>;
|
||||
extendLock(jobId: string, token: string, duration: number, client?: RedisClient | IRedisTransaction): Promise<number>;
|
||||
extendLocks(jobIds: string[], tokens: string[], duration: number): Promise<string[]>;
|
||||
updateData<T = any, R = any, N extends string = string>(job: MinimalJob<T, R, N>, data: T): Promise<void>;
|
||||
updateProgress(jobId: string, progress: JobProgress): Promise<void>;
|
||||
addLog(jobId: string, logRow: string, keepLogs?: number): Promise<number>;
|
||||
protected moveToFinishedArgs<T = any, R = any, N extends string = string>(job: MinimalJob<T, R, N>, val: any, propVal: FinishedPropValAttribute, shouldRemove: undefined | boolean | number | KeepJobs, target: FinishedStatus, token: string, timestamp: number, fetchNext?: boolean, fieldsToUpdate?: Record<string, any>): (string | number | boolean | Buffer)[];
|
||||
protected getKeepJobs(shouldRemove: undefined | boolean | number | KeepJobs, workerKeepJobs: undefined | KeepJobs): KeepJobs;
|
||||
moveToFinished(jobId: string, args: (string | number | boolean | Buffer)[]): Promise<any[]>;
|
||||
private drainArgs;
|
||||
drain(delayed: boolean): Promise<void>;
|
||||
private removeChildDependencyArgs;
|
||||
removeChildDependency(jobId: string, parentKey: string): Promise<boolean>;
|
||||
private getRangesArgs;
|
||||
getRanges(types: JobType[], start?: number, end?: number, asc?: boolean): Promise<[string][]>;
|
||||
private getJobsArgs;
|
||||
/**
|
||||
* Fetches job ids and their job hashes for the provided states in a single
|
||||
* script, skipping ids whose job hash is missing (for example the deprecated
|
||||
* wait list marker or jobs removed after their id was read). Each returned
|
||||
* entry is a `[jobId, jobHashFields]` tuple grouped per requested type.
|
||||
*/
|
||||
getJobs(types: JobType[], start?: number, end?: number, asc?: boolean): Promise<[string, string[]][][]>;
|
||||
private getCountsArgs;
|
||||
getCounts(types: JobType[]): Promise<number[]>;
|
||||
protected getCountsPerPriorityArgs(priorities: number[]): (string | number)[];
|
||||
getCountsPerPriority(priorities: number[]): Promise<number[]>;
|
||||
protected getDependencyCountsArgs(jobId: string, types: string[]): (string | number)[];
|
||||
getDependencyCounts(jobId: string, types: string[]): Promise<number[]>;
|
||||
moveToCompletedArgs<T = any, R = any, N extends string = string>(job: MinimalJob<T, R, N>, returnvalue: R, removeOnComplete: boolean | number | KeepJobs, token: string, fetchNext?: boolean): (string | number | boolean | Buffer)[];
|
||||
moveToFailedArgs<T = any, R = any, N extends string = string>(job: MinimalJob<T, R, N>, failedReason: string, removeOnFailed: boolean | number | KeepJobs, token: string, fetchNext?: boolean, fieldsToUpdate?: Record<string, any>): (string | number | boolean | Buffer)[];
|
||||
isFinished(jobId: string, returnValue?: boolean): Promise<number | [number, string]>;
|
||||
getState(jobId: string): Promise<JobState | 'unknown'>;
|
||||
/**
|
||||
* Change delay of a delayed job.
|
||||
*
|
||||
* Reschedules a delayed job by setting a new delay from the current time.
|
||||
* For example, calling changeDelay(5000) will reschedule the job to execute
|
||||
* 5000 milliseconds (5 seconds) from now, regardless of the original delay.
|
||||
*
|
||||
* @param jobId - the ID of the job to change the delay for.
|
||||
* @param delay - milliseconds from now when the job should be processed.
|
||||
* @returns delay in milliseconds.
|
||||
* @throws JobNotExist
|
||||
* This exception is thrown if jobId is missing.
|
||||
* @throws JobNotInState
|
||||
* This exception is thrown if job is not in delayed state.
|
||||
*/
|
||||
changeDelay(jobId: string, delay: number): Promise<void>;
|
||||
private changeDelayArgs;
|
||||
changePriority(jobId: string, priority?: number, lifo?: boolean): Promise<void>;
|
||||
protected changePriorityArgs(jobId: string, priority?: number, lifo?: boolean): (string | number)[];
|
||||
moveToDelayedArgs(jobId: string, timestamp: number, token: string, delay: number, opts?: MoveToDelayedOpts): (string | number | Buffer)[];
|
||||
moveToWaitingChildrenArgs(jobId: string, token: string, opts?: MoveToWaitingChildrenOpts): (string | number)[];
|
||||
isMaxedArgs(): string[];
|
||||
isMaxed(): Promise<boolean>;
|
||||
moveToDelayed(jobId: string, timestamp: number, delay: number, token?: string, opts?: MoveToDelayedOpts): Promise<void | any[]>;
|
||||
/**
|
||||
* Move parent job to waiting-children state.
|
||||
*
|
||||
* @returns true if job is successfully moved, false if there are pending dependencies.
|
||||
* @throws JobNotExist
|
||||
* This exception is thrown if jobId is missing.
|
||||
* @throws JobLockNotExist
|
||||
* This exception is thrown if job lock is missing.
|
||||
* @throws JobNotInState
|
||||
* This exception is thrown if job is not in active state.
|
||||
*/
|
||||
moveToWaitingChildren(jobId: string, token: string, opts?: MoveToWaitingChildrenOpts): Promise<boolean>;
|
||||
getRateLimitTtlArgs(maxJobs?: number): (string | number)[];
|
||||
getRateLimitTtl(maxJobs?: number): Promise<number>;
|
||||
/**
|
||||
* Remove jobs in a specific state.
|
||||
*
|
||||
* @returns Id jobs from the deleted records.
|
||||
*/
|
||||
cleanJobsByState(state: string, timestamp: number, limit?: number): Promise<string[]>;
|
||||
getJobSchedulerArgs(id: string): string[];
|
||||
getJobScheduler(id: string): Promise<[any, string | null]>;
|
||||
isJobScheduler(id: string): Promise<boolean>;
|
||||
getJobSchedulerData(key: string): Promise<Record<string, string>>;
|
||||
getJobSchedulersRange(start: number, end: number, asc: boolean): Promise<string[]>;
|
||||
getJobSchedulersCount(): Promise<number>;
|
||||
retryJobArgs(jobId: string, lifo: boolean, token: string, opts?: MoveToDelayedOpts): (string | number | Buffer)[];
|
||||
retryJob(jobId: string, lifo: boolean, token?: string, opts?: RetryJobOpts): Promise<void>;
|
||||
protected moveJobsToWaitArgs(state: FinishedStatus | 'delayed', count: number, timestamp: number): (string | number)[];
|
||||
retryFinishedJobs(state?: FinishedStatus, count?: number, timestamp?: number): Promise<number>;
|
||||
promoteJobs(count?: number): Promise<number>;
|
||||
/**
|
||||
* Attempts to reprocess a job
|
||||
*
|
||||
* @param job - The job to reprocess
|
||||
* @param state - The expected job state. If the job is not found
|
||||
* on the provided state, then it's not reprocessed. Supported states: 'failed', 'completed'
|
||||
*
|
||||
* @returns A promise that resolves when the job has been successfully moved to the wait queue.
|
||||
* @throws Will throw an error with a code property indicating the failure reason:
|
||||
* - code 0: Job does not exist
|
||||
* - code -1: Job is currently locked and can't be retried
|
||||
* - code -2: Job was not found in the expected set
|
||||
*/
|
||||
retryFinishedJob<T = any, R = any, N extends string = string>(job: MinimalJob<T, R, N>, state: 'failed' | 'completed', opts?: RetryOptions): Promise<void>;
|
||||
getMetrics(type: 'completed' | 'failed', start?: number, end?: number): Promise<[string[], string[], number]>;
|
||||
getClientList(): Promise<string[]>;
|
||||
moveToActive(token: string, name?: string): Promise<any[]>;
|
||||
promote(jobId: string): Promise<void>;
|
||||
protected moveStalledJobsToWaitArgs(): (string | number)[];
|
||||
/**
|
||||
* Looks for unlocked jobs in the active queue.
|
||||
*
|
||||
* The job was being worked on, but the worker process died and it failed to renew the lock.
|
||||
* We call these jobs 'stalled'. This is the most common case. We resolve these by moving them
|
||||
* back to wait to be re-processed. To prevent jobs from cycling endlessly between active and wait,
|
||||
* (e.g. if the job handler keeps crashing),
|
||||
* we limit the number stalled job recoveries to settings.maxStalledCount.
|
||||
*/
|
||||
moveStalledJobsToWait(): Promise<string[]>;
|
||||
/**
|
||||
* Moves a job back from Active to Wait.
|
||||
* This script is used when a job has been manually rate limited and needs
|
||||
* to be moved back to wait from active status.
|
||||
*
|
||||
* @param client - Redis client
|
||||
* @param jobId - Job id
|
||||
* @returns
|
||||
*/
|
||||
moveJobFromActiveToWait(jobId: string, token?: string): Promise<any>;
|
||||
obliterate(opts: {
|
||||
force: boolean;
|
||||
count: number;
|
||||
}): Promise<number>;
|
||||
/**
|
||||
* Paginate a set or hash keys.
|
||||
* @param opts - options to define the pagination behaviour
|
||||
*
|
||||
*/
|
||||
paginate(key: string, opts: {
|
||||
start: number;
|
||||
end: number;
|
||||
fetchJobs?: boolean;
|
||||
}): Promise<{
|
||||
cursor: string;
|
||||
items: {
|
||||
id: string;
|
||||
v?: any;
|
||||
err?: string;
|
||||
}[];
|
||||
total: number;
|
||||
jobs?: JobJson[];
|
||||
}>;
|
||||
finishedErrors({ code, jobId, parentKey, command, state, }: {
|
||||
code: number;
|
||||
jobId?: string;
|
||||
parentKey?: string;
|
||||
command: string;
|
||||
state?: string;
|
||||
}): Error;
|
||||
/**
|
||||
* Low-level Redis adapter helper: atomically check-and-delete a single batch
|
||||
* of candidate orphaned jobs. Driven by {@link removeOrphanedJobs}.
|
||||
*/
|
||||
protected removeOrphanedJobsBatch(candidateJobIds: string[], stateKeySuffixes: string[], jobSubKeySuffixes: string[]): Promise<number>;
|
||||
removeOrphanedJobs(count?: number, limit?: number): Promise<number>;
|
||||
moveToCompleted<T = any, R = any, N extends string = string>(job: MinimalJob<T, R, N>, returnValue: R, removeOnComplete: boolean | number | KeepJobs, token: string, fetchNext: boolean): Promise<{
|
||||
result: void | any[];
|
||||
finishedOn: number;
|
||||
}>;
|
||||
moveToFailed<T = any, R = any, N extends string = string>(job: MinimalJob<T, R, N>, failedReason: string, removeOnFail: boolean | number | KeepJobs, token: string, fetchNext: boolean, fieldsToUpdate?: Record<string, any>): Promise<{
|
||||
result: void | any[];
|
||||
finishedOn: number;
|
||||
}>;
|
||||
getJobData(jobId: string): Promise<JobJson | undefined>;
|
||||
getDeduplicationJobId(deduplicationId: string): Promise<string | null>;
|
||||
getJobLogs(jobId: string, start: number, end: number, asc: boolean): Promise<{
|
||||
logs: string[];
|
||||
count: number;
|
||||
}>;
|
||||
clearLogs(jobId: string, keepLogs?: number): Promise<void>;
|
||||
getProcessedChildrenValues(jobId: string): Promise<Record<string, string>>;
|
||||
getIgnoredChildrenFailures(jobId: string): Promise<Record<string, string>>;
|
||||
getDependencies(jobId: string, opts?: DependenciesOpts): Promise<{
|
||||
nextFailedCursor?: number;
|
||||
failed?: string[];
|
||||
nextIgnoredCursor?: number;
|
||||
ignored?: Record<string, any>;
|
||||
nextProcessedCursor?: number;
|
||||
processed?: Record<string, any>;
|
||||
nextUnprocessedCursor?: number;
|
||||
unprocessed?: string[];
|
||||
}>;
|
||||
setQueueMeta(values: Record<string, string | number>): Promise<number>;
|
||||
getQueueMetaField(field: string): Promise<string | null>;
|
||||
getQueueMetaFields(fields: string[]): Promise<(string | null)[]>;
|
||||
getQueueMeta(): Promise<Record<string, string>>;
|
||||
removeQueueMetaFields(fields: string[]): Promise<number>;
|
||||
hasQueueMetaField(field: string): Promise<boolean>;
|
||||
setRateLimit(expireTimeMs: number): Promise<void>;
|
||||
removeRateLimitKey(): Promise<number>;
|
||||
removeDeprecatedPriorityKey(): Promise<number>;
|
||||
deleteDeduplicationKey(deduplicationId: string): Promise<number>;
|
||||
trimEvents(maxLength: number): Promise<number>;
|
||||
waitForJob(blockTimeout: number): Promise<{
|
||||
member: string;
|
||||
score: number;
|
||||
} | null>;
|
||||
publishEvent(fields: Record<string, string | number>, maxEvents: number): Promise<string>;
|
||||
readEvents(id: string, blockTimeout: number): Promise<StreamReadRaw>;
|
||||
}
|
||||
export declare function raw2NextJobData(raw: any[]): any[];
|
||||
2047
node_modules/bullmq/dist/esm/classes/redis-queue-backend.js
generated
vendored
Normal file
2047
node_modules/bullmq/dist/esm/classes/redis-queue-backend.js
generated
vendored
Normal file
File diff suppressed because it is too large
Load Diff
4
node_modules/bullmq/dist/esm/classes/sandbox.d.ts
generated
vendored
Normal file
4
node_modules/bullmq/dist/esm/classes/sandbox.d.ts
generated
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
import { ChildPool } from './child-pool';
|
||||
import { Job } from './job';
|
||||
declare const sandbox: <T, R, N extends string>(processFile: any, childPool: ChildPool) => (job: Job<T, R, N>, token?: string, signal?: AbortSignal) => Promise<R>;
|
||||
export default sandbox;
|
||||
159
node_modules/bullmq/dist/esm/classes/sandbox.js
generated
vendored
Normal file
159
node_modules/bullmq/dist/esm/classes/sandbox.js
generated
vendored
Normal file
@@ -0,0 +1,159 @@
|
||||
import { ChildCommand, ParentCommand } from '../enums';
|
||||
const sandbox = (processFile, childPool) => {
|
||||
return async function process(job, token, signal) {
|
||||
let child;
|
||||
let msgHandler;
|
||||
let exitHandler;
|
||||
let abortHandler;
|
||||
try {
|
||||
const done = new Promise((resolve, reject) => {
|
||||
const initChild = async () => {
|
||||
try {
|
||||
exitHandler = (exitCode, signal) => {
|
||||
reject(new Error('Unexpected exit code: ' + exitCode + ' signal: ' + signal));
|
||||
};
|
||||
child = await childPool.retain(processFile);
|
||||
child.on('exit', exitHandler);
|
||||
msgHandler = async (msg) => {
|
||||
var _a, _b, _c, _d, _e, _f;
|
||||
try {
|
||||
switch (msg.cmd) {
|
||||
case ParentCommand.Completed:
|
||||
resolve(msg.value);
|
||||
break;
|
||||
case ParentCommand.Failed:
|
||||
case ParentCommand.Error: {
|
||||
const err = new Error();
|
||||
// ParentCommand.Failed carries the error under `value`,
|
||||
// while ParentCommand.Error carries it under `err`. Read
|
||||
// from either key so the failure reason is never lost.
|
||||
Object.assign(err, (_a = msg.value) !== null && _a !== void 0 ? _a : msg.err);
|
||||
reject(err);
|
||||
break;
|
||||
}
|
||||
case ParentCommand.Progress:
|
||||
await job.updateProgress(msg.value);
|
||||
break;
|
||||
case ParentCommand.Log:
|
||||
await job.log(msg.value);
|
||||
break;
|
||||
case ParentCommand.MoveToDelayed:
|
||||
await job.moveToDelayed((_b = msg.value) === null || _b === void 0 ? void 0 : _b.timestamp, (_c = msg.value) === null || _c === void 0 ? void 0 : _c.token);
|
||||
break;
|
||||
case ParentCommand.MoveToWait:
|
||||
await job.moveToWait((_d = msg.value) === null || _d === void 0 ? void 0 : _d.token);
|
||||
break;
|
||||
case ParentCommand.MoveToWaitingChildren:
|
||||
{
|
||||
const value = await job.moveToWaitingChildren((_e = msg.value) === null || _e === void 0 ? void 0 : _e.token, (_f = msg.value) === null || _f === void 0 ? void 0 : _f.opts);
|
||||
child.send({
|
||||
requestId: msg.requestId,
|
||||
cmd: ChildCommand.MoveToWaitingChildrenResponse,
|
||||
value,
|
||||
});
|
||||
}
|
||||
break;
|
||||
case ParentCommand.Update:
|
||||
await job.updateData(msg.value);
|
||||
break;
|
||||
case ParentCommand.GetChildrenValues:
|
||||
{
|
||||
const value = await job.getChildrenValues();
|
||||
child.send({
|
||||
requestId: msg.requestId,
|
||||
cmd: ChildCommand.GetChildrenValuesResponse,
|
||||
value,
|
||||
});
|
||||
}
|
||||
break;
|
||||
case ParentCommand.GetIgnoredChildrenFailures:
|
||||
{
|
||||
const value = await job.getIgnoredChildrenFailures();
|
||||
child.send({
|
||||
requestId: msg.requestId,
|
||||
cmd: ChildCommand.GetIgnoredChildrenFailuresResponse,
|
||||
value,
|
||||
});
|
||||
}
|
||||
break;
|
||||
case ParentCommand.GetDependenciesCount:
|
||||
{
|
||||
const value = await job.getDependenciesCount(msg.value);
|
||||
child.send({
|
||||
requestId: msg.requestId,
|
||||
cmd: ChildCommand.GetDependenciesCountResponse,
|
||||
value,
|
||||
});
|
||||
}
|
||||
break;
|
||||
case ParentCommand.GetDependencies:
|
||||
{
|
||||
const value = await job.getDependencies(msg.value);
|
||||
child.send({
|
||||
requestId: msg.requestId,
|
||||
cmd: ChildCommand.GetDependenciesResponse,
|
||||
value,
|
||||
});
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
reject(err);
|
||||
}
|
||||
};
|
||||
child.on('message', msgHandler);
|
||||
child.send({
|
||||
cmd: ChildCommand.Start,
|
||||
job: job.asJSONSandbox(),
|
||||
token,
|
||||
});
|
||||
if (signal) {
|
||||
abortHandler = () => {
|
||||
try {
|
||||
child.send({
|
||||
cmd: ChildCommand.Cancel,
|
||||
value: signal.reason,
|
||||
});
|
||||
}
|
||||
catch (_a) {
|
||||
// Child process may have already exited
|
||||
}
|
||||
};
|
||||
if (signal.aborted) {
|
||||
abortHandler();
|
||||
}
|
||||
else {
|
||||
signal.addEventListener('abort', abortHandler, { once: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
reject(error);
|
||||
}
|
||||
};
|
||||
initChild();
|
||||
});
|
||||
await done;
|
||||
return done;
|
||||
}
|
||||
finally {
|
||||
// Note: There is a potential race where the signal is aborted between
|
||||
// `await done` and this cleanup. This is safe because:
|
||||
// 1. abortHandler has a try-catch for child process already exited
|
||||
// 2. The listener is added with `once: true`, so it fires at most once
|
||||
// 3. removeEventListener here is defensive cleanup only
|
||||
if (signal && abortHandler) {
|
||||
signal.removeEventListener('abort', abortHandler);
|
||||
}
|
||||
if (child) {
|
||||
child.off('message', msgHandler);
|
||||
child.off('exit', exitHandler);
|
||||
if (child.exitCode === null && child.signalCode === null) {
|
||||
childPool.release(child);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
};
|
||||
export default sandbox;
|
||||
2
node_modules/bullmq/dist/esm/classes/valkey-glide-client.d.ts
generated
vendored
Normal file
2
node_modules/bullmq/dist/esm/classes/valkey-glide-client.d.ts
generated
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
import { IRedisClient } from '../interfaces/redis-client';
|
||||
export declare function createValkeyGlideClient(client: unknown): IRedisClient;
|
||||
976
node_modules/bullmq/dist/esm/classes/valkey-glide-client.js
generated
vendored
Normal file
976
node_modules/bullmq/dist/esm/classes/valkey-glide-client.js
generated
vendored
Normal file
@@ -0,0 +1,976 @@
|
||||
import { createHash } from 'crypto';
|
||||
import { EventEmitter } from 'events';
|
||||
import { Readable } from 'stream';
|
||||
import { ConnectionClosedError } from './errors/connection-closed-error';
|
||||
const GLIDE_STRING_DECODER = 1;
|
||||
function createCompatibilityBatch() {
|
||||
return {
|
||||
commands: [],
|
||||
customCommand(args) {
|
||||
this.commands.push(args);
|
||||
},
|
||||
};
|
||||
}
|
||||
function createGlideBatch(isAtomic) {
|
||||
try {
|
||||
const { Batch } = require('@valkey/valkey-glide');
|
||||
return typeof Batch === 'function' ? new Batch(isAtomic) : null;
|
||||
}
|
||||
catch (_a) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
function isBuffer(value) {
|
||||
return Buffer.isBuffer(value);
|
||||
}
|
||||
function toGlideArg(value) {
|
||||
if (isBuffer(value)) {
|
||||
return value;
|
||||
}
|
||||
if (value === undefined || value === null) {
|
||||
return '';
|
||||
}
|
||||
return String(value);
|
||||
}
|
||||
function toStringValue(value) {
|
||||
if (isBuffer(value)) {
|
||||
return value.toString();
|
||||
}
|
||||
return String(value);
|
||||
}
|
||||
function isKeyValueArray(value) {
|
||||
return (Array.isArray(value) &&
|
||||
value.every(item => item && typeof item === 'object' && 'key' in item && 'value' in item));
|
||||
}
|
||||
function normalizeScriptArgs(args) {
|
||||
return args.length === 1 && Array.isArray(args[0]) ? args[0] : args;
|
||||
}
|
||||
function normalizeHashReply(reply) {
|
||||
if (!reply) {
|
||||
return {};
|
||||
}
|
||||
if (isKeyValueArray(reply)) {
|
||||
return reply.reduce((acc, item) => {
|
||||
acc[toStringValue(item.key)] = toStringValue(item.value);
|
||||
return acc;
|
||||
}, {});
|
||||
}
|
||||
if (reply instanceof Map) {
|
||||
const out = {};
|
||||
for (const [key, value] of reply.entries()) {
|
||||
out[toStringValue(key)] = toStringValue(value);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
if (Array.isArray(reply)) {
|
||||
const out = {};
|
||||
for (let i = 0; i < reply.length; i += 2) {
|
||||
out[toStringValue(reply[i])] = toStringValue(reply[i + 1]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
if (typeof reply === 'object') {
|
||||
const out = {};
|
||||
for (const [key, value] of Object.entries(reply)) {
|
||||
out[key] = toStringValue(value);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
return {};
|
||||
}
|
||||
function flattenFieldPairs(fields) {
|
||||
if (!fields) {
|
||||
return [];
|
||||
}
|
||||
if (Array.isArray(fields)) {
|
||||
if (isKeyValueArray(fields)) {
|
||||
return fields.flatMap(pair => [
|
||||
toStringValue(pair.key),
|
||||
toStringValue(pair.value),
|
||||
]);
|
||||
}
|
||||
if (fields.every(item => Array.isArray(item) && item.length === 2)) {
|
||||
return fields.flatMap(pair => {
|
||||
const [field, value] = pair;
|
||||
return [toStringValue(field), toStringValue(value)];
|
||||
});
|
||||
}
|
||||
return fields.map(item => toStringValue(item));
|
||||
}
|
||||
if (typeof fields === 'object') {
|
||||
return Object.entries(fields).flatMap(([field, value]) => [field, toStringValue(value)]);
|
||||
}
|
||||
return [];
|
||||
}
|
||||
function normalizeXReadReply(reply) {
|
||||
if (!reply) {
|
||||
return null;
|
||||
}
|
||||
if (Array.isArray(reply) &&
|
||||
reply.every(stream => Array.isArray(stream) &&
|
||||
stream.length === 2 &&
|
||||
typeof stream[0] !== 'undefined' &&
|
||||
Array.isArray(stream[1]))) {
|
||||
return reply;
|
||||
}
|
||||
if (!isKeyValueArray(reply)) {
|
||||
return reply;
|
||||
}
|
||||
return reply.map(streamEntry => {
|
||||
const streamName = toStringValue(streamEntry.key);
|
||||
const messages = isKeyValueArray(streamEntry.value)
|
||||
? streamEntry.value.map(message => [
|
||||
toStringValue(message.key),
|
||||
flattenFieldPairs(message.value),
|
||||
])
|
||||
: [];
|
||||
return [streamName, messages];
|
||||
});
|
||||
}
|
||||
function normalizeScanReply(reply) {
|
||||
if (Array.isArray(reply) && reply.length >= 2) {
|
||||
const [cursor, keys] = reply;
|
||||
const normalizedKeys = Array.isArray(keys)
|
||||
? keys.map(key => toStringValue(key))
|
||||
: [];
|
||||
return [toStringValue(cursor), normalizedKeys];
|
||||
}
|
||||
if (reply && typeof reply === 'object') {
|
||||
const result = reply;
|
||||
if (result.cursor !== undefined && Array.isArray(result.keys)) {
|
||||
return [
|
||||
toStringValue(result.cursor),
|
||||
result.keys.map(key => toStringValue(key)),
|
||||
];
|
||||
}
|
||||
}
|
||||
return ['0', []];
|
||||
}
|
||||
function normalizeXRangeReply(reply) {
|
||||
if (!reply) {
|
||||
return [];
|
||||
}
|
||||
if (Array.isArray(reply) &&
|
||||
reply.every(entry => Array.isArray(entry) &&
|
||||
entry.length === 2 &&
|
||||
typeof entry[0] !== 'undefined' &&
|
||||
Array.isArray(entry[1]))) {
|
||||
return reply.map(([id, fields]) => [
|
||||
toStringValue(id),
|
||||
fields.map((field) => toStringValue(field)),
|
||||
]);
|
||||
}
|
||||
if (!isKeyValueArray(reply)) {
|
||||
return [];
|
||||
}
|
||||
return reply.map(entry => [
|
||||
toStringValue(entry.key),
|
||||
flattenFieldPairs(entry.value),
|
||||
]);
|
||||
}
|
||||
function normalizeSortedSetReply(reply, withScores = false) {
|
||||
if (!Array.isArray(reply)) {
|
||||
return [];
|
||||
}
|
||||
if (!withScores) {
|
||||
return reply.map(item => toStringValue(item));
|
||||
}
|
||||
if (isKeyValueArray(reply)) {
|
||||
return reply.flatMap(entry => [
|
||||
toStringValue(entry.key),
|
||||
toStringValue(entry.value),
|
||||
]);
|
||||
}
|
||||
if (reply.every(item => Array.isArray(item) && item.length >= 2 && item[0] !== undefined)) {
|
||||
return reply.flatMap(item => [
|
||||
toStringValue(item[0]),
|
||||
toStringValue(item[1]),
|
||||
]);
|
||||
}
|
||||
return reply.map(item => toStringValue(item));
|
||||
}
|
||||
function isBlockingCommand(args) {
|
||||
const [command, ...rest] = args;
|
||||
const name = toStringValue(command).toUpperCase();
|
||||
if (name === 'BZPOPMIN') {
|
||||
return true;
|
||||
}
|
||||
if (name !== 'XREAD') {
|
||||
return false;
|
||||
}
|
||||
for (let i = 0; i < rest.length; i += 2) {
|
||||
if (toStringValue(rest[i]).toUpperCase() === 'BLOCK') {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
export function createValkeyGlideClient(client) {
|
||||
return new ValkeyGlideAdapter(client);
|
||||
}
|
||||
class ValkeyGlideAdapter extends EventEmitter {
|
||||
constructor(rawOrPromise, connectionName) {
|
||||
super();
|
||||
this.connectionName = connectionName;
|
||||
this.scripts = new Map();
|
||||
this.scriptsBySha = new Map();
|
||||
this.scriptLoadPromises = new Map();
|
||||
this.readyEmitted = false;
|
||||
this.closed = false;
|
||||
this.operationChain = Promise.resolve();
|
||||
this.activeBlockingCommands = 0;
|
||||
if (rawOrPromise instanceof Promise) {
|
||||
this.rawPromise = rawOrPromise;
|
||||
void this.connect().catch(error => this.emit('error', error));
|
||||
}
|
||||
else {
|
||||
this.raw = rawOrPromise;
|
||||
}
|
||||
}
|
||||
get status() {
|
||||
if (this.statusOverride) {
|
||||
return this.statusOverride;
|
||||
}
|
||||
if (this.closed) {
|
||||
return 'end';
|
||||
}
|
||||
if (this.readyEmitted) {
|
||||
return 'ready';
|
||||
}
|
||||
return 'wait';
|
||||
}
|
||||
set status(value) {
|
||||
if (value === 'end') {
|
||||
this.disconnect();
|
||||
}
|
||||
this.statusOverride = value;
|
||||
}
|
||||
get isCluster() {
|
||||
var _a, _b, _c;
|
||||
const name = (_c = (_b = (_a = this.raw) === null || _a === void 0 ? void 0 : _a.constructor) === null || _b === void 0 ? void 0 : _b.name) !== null && _c !== void 0 ? _c : '';
|
||||
return name.includes('Cluster');
|
||||
}
|
||||
get options() {
|
||||
var _a, _b, _c, _d;
|
||||
return (_d = (_b = (_a = this.raw) === null || _a === void 0 ? void 0 : _a.config) !== null && _b !== void 0 ? _b : (_c = this.raw) === null || _c === void 0 ? void 0 : _c.options) !== null && _d !== void 0 ? _d : {};
|
||||
}
|
||||
set options(_value) {
|
||||
// no-op
|
||||
}
|
||||
ensureOpen() {
|
||||
if (this.closed) {
|
||||
throw new ConnectionClosedError();
|
||||
}
|
||||
}
|
||||
normalizeError(error) {
|
||||
if (error instanceof ConnectionClosedError ||
|
||||
(error instanceof Error && error.name === 'ClosingError')) {
|
||||
throw new ConnectionClosedError(error.message, error);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
async ensureRaw() {
|
||||
if (this.raw) {
|
||||
return this.raw;
|
||||
}
|
||||
if (!this.rawPromise) {
|
||||
throw new Error('BullMQ: Valkey Glide client not initialized - missing raw client ' +
|
||||
'and client promise. Please report this as a bug.');
|
||||
}
|
||||
this.raw = await this.rawPromise;
|
||||
return this.raw;
|
||||
}
|
||||
async runSerialized(fn) {
|
||||
this.ensureOpen();
|
||||
const previous = this.operationChain;
|
||||
let release;
|
||||
this.operationChain = new Promise(resolve => {
|
||||
release = resolve;
|
||||
});
|
||||
await previous;
|
||||
this.ensureOpen();
|
||||
try {
|
||||
const raw = await this.ensureRaw();
|
||||
this.ensureOpen();
|
||||
return await fn(raw);
|
||||
}
|
||||
catch (error) {
|
||||
this.normalizeError(error);
|
||||
}
|
||||
finally {
|
||||
release();
|
||||
}
|
||||
}
|
||||
async runRawCommand(args, options) {
|
||||
const blockingCommand = isBlockingCommand(args);
|
||||
return this.runSerialized(async (raw) => {
|
||||
if (blockingCommand) {
|
||||
this.activeBlockingCommands++;
|
||||
}
|
||||
try {
|
||||
return await raw.customCommand(args, options);
|
||||
}
|
||||
finally {
|
||||
if (blockingCommand) {
|
||||
this.activeBlockingCommands--;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
ensureScriptLoaded(script) {
|
||||
let pendingLoad = this.scriptLoadPromises.get(script.sha);
|
||||
if (!pendingLoad) {
|
||||
pendingLoad = this.runRawCommand(['SCRIPT', 'LOAD', script.lua])
|
||||
.then(() => { })
|
||||
.catch(() => {
|
||||
// Ignore script preload errors here – runCommand has NOSCRIPT fallback
|
||||
// for non-transactional usage, and transactional callers will surface
|
||||
// the underlying Redis error if the script still is not available.
|
||||
});
|
||||
this.scriptLoadPromises.set(script.sha, pendingLoad);
|
||||
}
|
||||
return pendingLoad;
|
||||
}
|
||||
async applyConnectionNameIfNeeded() {
|
||||
if (!this.connectionName) {
|
||||
return;
|
||||
}
|
||||
await this.runRawCommand(['CLIENT', 'SETNAME', this.connectionName], {
|
||||
decoder: GLIDE_STRING_DECODER,
|
||||
});
|
||||
}
|
||||
async clearConnectionNameIfNeeded(raw) {
|
||||
if (!this.connectionName) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await raw.customCommand(['CLIENT', 'SETNAME', ''], {
|
||||
decoder: GLIDE_STRING_DECODER,
|
||||
});
|
||||
}
|
||||
catch (_a) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
async recreateRaw() {
|
||||
var _a, _b;
|
||||
const raw = await this.ensureRaw();
|
||||
const clientConstructor = raw.constructor;
|
||||
const createClient = (_a = clientConstructor === null || clientConstructor === void 0 ? void 0 : clientConstructor.createClient) === null || _a === void 0 ? void 0 : _a.bind(clientConstructor);
|
||||
const config = (_b = raw.config) !== null && _b !== void 0 ? _b : raw.options;
|
||||
if (!createClient || !config) {
|
||||
throw new Error('BullMQ: Cannot recreate Valkey Glide client: missing createClient() method or ' +
|
||||
'config object. Ensure the client was created via GlideClient.createClient() ' +
|
||||
'or GlideClusterClient.createClient().');
|
||||
}
|
||||
this.raw = await createClient(config);
|
||||
// Glide closes clients permanently, so a recreated raw connection starts
|
||||
// with an empty server-side script cache.
|
||||
this.scriptLoadPromises.clear();
|
||||
}
|
||||
async connect() {
|
||||
if (this.connecting) {
|
||||
return this.connecting;
|
||||
}
|
||||
this.connecting = (async () => {
|
||||
if (this.closed && this.raw) {
|
||||
await this.recreateRaw();
|
||||
}
|
||||
this.closed = false;
|
||||
this.statusOverride = undefined;
|
||||
await this.ensureRaw();
|
||||
try {
|
||||
await this.applyConnectionNameIfNeeded();
|
||||
}
|
||||
catch (error) {
|
||||
if (this.closed && error instanceof ConnectionClosedError) {
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (this.closed) {
|
||||
return;
|
||||
}
|
||||
this.readyEmitted = true;
|
||||
this.emit('ready');
|
||||
})().finally(() => {
|
||||
this.connecting = undefined;
|
||||
});
|
||||
return this.connecting;
|
||||
}
|
||||
closeRaw() {
|
||||
if (!this.closingPromise) {
|
||||
const closeRaw = () => {
|
||||
if (!this.raw) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
this.raw.close();
|
||||
}
|
||||
catch (_a) {
|
||||
// ignore
|
||||
}
|
||||
};
|
||||
this.closingPromise = (this.activeBlockingCommands > 0
|
||||
? Promise.resolve().then(closeRaw)
|
||||
: this.operationChain
|
||||
.catch(() => {
|
||||
// ignore
|
||||
})
|
||||
.then(async () => {
|
||||
if (!this.closed) {
|
||||
return;
|
||||
}
|
||||
if (this.raw) {
|
||||
await this.clearConnectionNameIfNeeded(this.raw);
|
||||
}
|
||||
closeRaw();
|
||||
})).finally(() => {
|
||||
this.closingPromise = undefined;
|
||||
});
|
||||
}
|
||||
return this.closingPromise;
|
||||
}
|
||||
disconnect(reconnect = false) {
|
||||
if (this.closed && !reconnect) {
|
||||
return;
|
||||
}
|
||||
this.closed = true;
|
||||
this.readyEmitted = false;
|
||||
this.statusOverride = reconnect ? undefined : 'end';
|
||||
this.emit('close');
|
||||
if (reconnect) {
|
||||
void this.closeRaw()
|
||||
.then(() => {
|
||||
this.emit('reconnecting');
|
||||
return this.connect();
|
||||
})
|
||||
.catch(error => this.emit('error', error));
|
||||
return;
|
||||
}
|
||||
void this.closeRaw();
|
||||
this.emit('end');
|
||||
}
|
||||
async quit() {
|
||||
if (this.closed) {
|
||||
setImmediate(() => {
|
||||
this.emit('end');
|
||||
this.emit('close');
|
||||
});
|
||||
return 'OK';
|
||||
}
|
||||
this.closed = true;
|
||||
this.readyEmitted = false;
|
||||
this.statusOverride = 'end';
|
||||
void this.closeRaw();
|
||||
setImmediate(() => {
|
||||
this.emit('end');
|
||||
this.emit('close');
|
||||
});
|
||||
return 'OK';
|
||||
}
|
||||
duplicate(...args) {
|
||||
var _a;
|
||||
const options = ((_a = args[0]) !== null && _a !== void 0 ? _a : {});
|
||||
const duplicated = (async () => {
|
||||
var _a, _b;
|
||||
const raw = await this.ensureRaw();
|
||||
const clientConstructor = raw.constructor;
|
||||
const createClient = (_a = clientConstructor === null || clientConstructor === void 0 ? void 0 : clientConstructor.createClient) === null || _a === void 0 ? void 0 : _a.bind(clientConstructor);
|
||||
const config = (_b = raw.config) !== null && _b !== void 0 ? _b : raw.options;
|
||||
if (!createClient || !config) {
|
||||
throw new Error('BullMQ: Cannot duplicate Valkey Glide client: missing createClient() or ' +
|
||||
'config. Ensure the client was created via GlideClient.createClient()/' +
|
||||
'GlideClusterClient.createClient().');
|
||||
}
|
||||
return createClient(config);
|
||||
})();
|
||||
return new ValkeyGlideAdapter(duplicated, options.connectionName);
|
||||
}
|
||||
defineCommand(name, definition) {
|
||||
const sha = createHash('sha1').update(definition.lua).digest('hex');
|
||||
const script = {
|
||||
sha,
|
||||
lua: definition.lua,
|
||||
numberOfKeys: definition.numberOfKeys,
|
||||
};
|
||||
this.scripts.set(name, script);
|
||||
this.scriptsBySha.set(sha, script);
|
||||
this[name] = (...args) => this.runCommand(name, args);
|
||||
void this.ensureScriptLoaded(script);
|
||||
}
|
||||
async runCommand(name, args) {
|
||||
const script = this.scripts.get(name);
|
||||
if (!script) {
|
||||
throw new Error(`BullMQ: command "${name}" is not defined. Use defineCommand() before runCommand().`);
|
||||
}
|
||||
const normalizedArgs = normalizeScriptArgs(args);
|
||||
const keys = normalizedArgs.slice(0, script.numberOfKeys).map(toGlideArg);
|
||||
const argv = normalizedArgs.slice(script.numberOfKeys).map(toGlideArg);
|
||||
const evalShaArgs = [
|
||||
'EVALSHA',
|
||||
script.sha,
|
||||
String(script.numberOfKeys),
|
||||
...keys,
|
||||
...argv,
|
||||
];
|
||||
try {
|
||||
return await this.runRawCommand(evalShaArgs);
|
||||
}
|
||||
catch (err) {
|
||||
if (typeof (err === null || err === void 0 ? void 0 : err.message) === 'string' &&
|
||||
err.message.toLowerCase().includes('noscript')) {
|
||||
return this.runRawCommand([
|
||||
'EVAL',
|
||||
script.lua,
|
||||
String(script.numberOfKeys),
|
||||
...keys,
|
||||
...argv,
|
||||
]);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
multi() {
|
||||
return new ValkeyGlideTransaction(this, this.scripts);
|
||||
}
|
||||
pipeline() {
|
||||
return this.multi();
|
||||
}
|
||||
async hgetall(key) {
|
||||
return normalizeHashReply(await this.runRawCommand(['HGETALL', key]));
|
||||
}
|
||||
async hget(key, field) {
|
||||
const value = await this.runRawCommand(['HGET', key, field]);
|
||||
return value == null ? null : toStringValue(value);
|
||||
}
|
||||
async hmget(key, ...fields) {
|
||||
const value = await this.runRawCommand(['HMGET', key, ...fields]);
|
||||
return Array.isArray(value)
|
||||
? value.map(item => (item == null ? null : toStringValue(item)))
|
||||
: [];
|
||||
}
|
||||
async hset(key, data) {
|
||||
const args = ['HSET', key];
|
||||
for (const [field, value] of Object.entries(data)) {
|
||||
args.push(field, toGlideArg(value));
|
||||
}
|
||||
return Number(await this.runRawCommand(args));
|
||||
}
|
||||
async hdel(key, ...fields) {
|
||||
return Number(await this.runRawCommand(['HDEL', key, ...fields]));
|
||||
}
|
||||
async hexists(key, field) {
|
||||
const result = await this.runRawCommand(['HEXISTS', key, field]);
|
||||
if (typeof result === 'boolean') {
|
||||
return result ? 1 : 0;
|
||||
}
|
||||
return Number(result);
|
||||
}
|
||||
async get(key) {
|
||||
const value = await this.runRawCommand(['GET', key]);
|
||||
return value == null ? null : toStringValue(value);
|
||||
}
|
||||
async set(key, value, options) {
|
||||
const args = ['SET', key, toGlideArg(value)];
|
||||
if ((options === null || options === void 0 ? void 0 : options.PX) != null) {
|
||||
args.push('PX', String(options.PX));
|
||||
}
|
||||
else if ((options === null || options === void 0 ? void 0 : options.EX) != null) {
|
||||
args.push('EX', String(options.EX));
|
||||
}
|
||||
const result = await this.runRawCommand(args);
|
||||
return result == null ? null : toStringValue(result);
|
||||
}
|
||||
async del(...keys) {
|
||||
if (keys.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
return Number(await this.runRawCommand(['DEL', ...keys]));
|
||||
}
|
||||
async zrange(key, start, end, options) {
|
||||
const args = ['ZRANGE', key, String(start), String(end)];
|
||||
if (options === null || options === void 0 ? void 0 : options.WITHSCORES) {
|
||||
args.push('WITHSCORES');
|
||||
}
|
||||
const result = await this.runRawCommand(args);
|
||||
return normalizeSortedSetReply(result, options === null || options === void 0 ? void 0 : options.WITHSCORES);
|
||||
}
|
||||
async zrevrange(key, start, end, options) {
|
||||
const args = ['ZREVRANGE', key, String(start), String(end)];
|
||||
if (options === null || options === void 0 ? void 0 : options.WITHSCORES) {
|
||||
args.push('WITHSCORES');
|
||||
}
|
||||
const result = await this.runRawCommand(args);
|
||||
return normalizeSortedSetReply(result, options === null || options === void 0 ? void 0 : options.WITHSCORES);
|
||||
}
|
||||
async zcard(key) {
|
||||
return Number(await this.runRawCommand(['ZCARD', key]));
|
||||
}
|
||||
async zscore(key, member) {
|
||||
const result = await this.runRawCommand(['ZSCORE', key, member]);
|
||||
return result == null ? null : toStringValue(result);
|
||||
}
|
||||
async lrange(key, start, end) {
|
||||
const result = await this.runRawCommand([
|
||||
'LRANGE',
|
||||
key,
|
||||
String(start),
|
||||
String(end),
|
||||
]);
|
||||
return Array.isArray(result) ? result.map(toStringValue) : [];
|
||||
}
|
||||
async llen(key) {
|
||||
return Number(await this.runRawCommand(['LLEN', key]));
|
||||
}
|
||||
async ltrim(key, start, end) {
|
||||
const result = await this.runRawCommand([
|
||||
'LTRIM',
|
||||
key,
|
||||
String(start),
|
||||
String(end),
|
||||
]);
|
||||
return result == null ? 'OK' : toStringValue(result);
|
||||
}
|
||||
async lpos(key, value) {
|
||||
const result = await this.runRawCommand(['LPOS', key, value]);
|
||||
return result == null ? null : Number(result);
|
||||
}
|
||||
async smembers(key) {
|
||||
const result = await this.runRawCommand(['SMEMBERS', key]);
|
||||
return Array.isArray(result) ? result.map(toStringValue) : [];
|
||||
}
|
||||
async xadd(key, id, fields, options) {
|
||||
const args = ['XADD', key];
|
||||
if ((options === null || options === void 0 ? void 0 : options.MAXLEN) != null) {
|
||||
args.push('MAXLEN');
|
||||
if (options.approximate !== false) {
|
||||
args.push('~');
|
||||
}
|
||||
args.push(String(options.MAXLEN));
|
||||
}
|
||||
args.push(id);
|
||||
for (const [field, value] of Object.entries(fields)) {
|
||||
args.push(field, toGlideArg(value));
|
||||
}
|
||||
return toStringValue(await this.runRawCommand(args));
|
||||
}
|
||||
async xread(streams, options) {
|
||||
const args = ['XREAD'];
|
||||
if ((options === null || options === void 0 ? void 0 : options.BLOCK) != null) {
|
||||
args.push('BLOCK', String(options.BLOCK));
|
||||
}
|
||||
if ((options === null || options === void 0 ? void 0 : options.COUNT) != null) {
|
||||
args.push('COUNT', String(options.COUNT));
|
||||
}
|
||||
args.push('STREAMS');
|
||||
for (const stream of streams) {
|
||||
args.push(stream.key);
|
||||
}
|
||||
for (const stream of streams) {
|
||||
args.push(stream.id);
|
||||
}
|
||||
return normalizeXReadReply(await this.runRawCommand(args));
|
||||
}
|
||||
async xtrim(key, strategy, threshold, options) {
|
||||
const args = ['XTRIM', key, strategy];
|
||||
if ((options === null || options === void 0 ? void 0 : options.approximate) !== false) {
|
||||
args.push('~');
|
||||
}
|
||||
args.push(String(threshold));
|
||||
return Number(await this.runRawCommand(args));
|
||||
}
|
||||
async bzpopmin(key, timeout) {
|
||||
const result = await this.runRawCommand(['BZPOPMIN', key, String(timeout)]);
|
||||
if (!result) {
|
||||
return null;
|
||||
}
|
||||
if (Array.isArray(result) && result.length >= 3) {
|
||||
return [
|
||||
toStringValue(result[0]),
|
||||
toStringValue(result[1]),
|
||||
toStringValue(result[2]),
|
||||
];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
async info() {
|
||||
return toStringValue(await this.runRawCommand(['INFO']));
|
||||
}
|
||||
async clientSetName(name) {
|
||||
return this.runRawCommand(['CLIENT', 'SETNAME', name], {
|
||||
decoder: GLIDE_STRING_DECODER,
|
||||
});
|
||||
}
|
||||
async clientList() {
|
||||
return toStringValue(await this.runRawCommand(['CLIENT', 'LIST'], {
|
||||
decoder: GLIDE_STRING_DECODER,
|
||||
}));
|
||||
}
|
||||
async scan(cursor, options) {
|
||||
const args = ['SCAN', String(cursor)];
|
||||
if (options === null || options === void 0 ? void 0 : options.MATCH) {
|
||||
args.push('MATCH', options.MATCH);
|
||||
}
|
||||
if ((options === null || options === void 0 ? void 0 : options.COUNT) != null) {
|
||||
args.push('COUNT', String(options.COUNT));
|
||||
}
|
||||
return normalizeScanReply(await this.runRawCommand(args));
|
||||
}
|
||||
scanStream(options) {
|
||||
let cursor = '0';
|
||||
let running = false;
|
||||
const stream = new Readable({
|
||||
objectMode: true,
|
||||
read: () => {
|
||||
if (running) {
|
||||
return;
|
||||
}
|
||||
running = true;
|
||||
(async () => {
|
||||
do {
|
||||
const [nextCursor, keys] = await this.scan(cursor, {
|
||||
MATCH: options.match,
|
||||
COUNT: options.count,
|
||||
});
|
||||
cursor = nextCursor;
|
||||
if (keys.length > 0 && !stream.push(keys)) {
|
||||
return;
|
||||
}
|
||||
} while (cursor !== '0');
|
||||
stream.push(null);
|
||||
})()
|
||||
.catch(err => stream.destroy(err))
|
||||
.finally(() => {
|
||||
running = false;
|
||||
});
|
||||
},
|
||||
});
|
||||
return stream;
|
||||
}
|
||||
async keys(pattern) {
|
||||
const result = await this.runRawCommand(['KEYS', pattern]);
|
||||
return Array.isArray(result) ? result.map(toStringValue) : [];
|
||||
}
|
||||
async exists(...keys) {
|
||||
if (keys.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
const result = await this.runRawCommand(['EXISTS', ...keys]);
|
||||
if (typeof result === 'boolean') {
|
||||
return result ? 1 : 0;
|
||||
}
|
||||
return Number(result);
|
||||
}
|
||||
async zadd(key, ...args) {
|
||||
const commandArgs = ['ZADD', key];
|
||||
for (let i = 0; i < args.length; i += 2) {
|
||||
commandArgs.push(toGlideArg(args[i]), toGlideArg(args[i + 1]));
|
||||
}
|
||||
return Number(await this.runRawCommand(commandArgs));
|
||||
}
|
||||
async zrem(key, ...members) {
|
||||
return Number(await this.runRawCommand(['ZREM', key, ...members]));
|
||||
}
|
||||
async xlen(key) {
|
||||
return Number(await this.runRawCommand(['XLEN', key]));
|
||||
}
|
||||
async xrevrange(key, end, start, ...rest) {
|
||||
const args = ['XREVRANGE', key, end, start];
|
||||
if (rest[0] === 'COUNT') {
|
||||
args.push('COUNT', toGlideArg(rest[1]));
|
||||
}
|
||||
return normalizeXRangeReply(await this.runRawCommand(args));
|
||||
}
|
||||
async sadd(key, ...members) {
|
||||
return Number(await this.runRawCommand([
|
||||
'SADD',
|
||||
key,
|
||||
...members.map(member => toGlideArg(member)),
|
||||
]));
|
||||
}
|
||||
async scard(key) {
|
||||
return Number(await this.runRawCommand(['SCARD', key]));
|
||||
}
|
||||
async lpush(key, ...values) {
|
||||
return Number(await this.runRawCommand(['LPUSH', key, ...values]));
|
||||
}
|
||||
async rpop(key) {
|
||||
const result = await this.runRawCommand(['RPOP', key]);
|
||||
return result == null ? null : toStringValue(result);
|
||||
}
|
||||
async incr(key) {
|
||||
return Number(await this.runRawCommand(['INCR', key]));
|
||||
}
|
||||
async incrby(key, increment) {
|
||||
return Number(await this.runRawCommand(['INCRBY', key, String(increment)]));
|
||||
}
|
||||
async flushall() {
|
||||
const result = await this.runRawCommand(['FLUSHALL']);
|
||||
return result == null ? 'OK' : toStringValue(result);
|
||||
}
|
||||
async execQueuedCommands(commands) {
|
||||
const requiredScripts = commands
|
||||
.map(command => String(command.args[0]).toUpperCase() === 'EVALSHA'
|
||||
? this.scriptsBySha.get(toStringValue(command.args[1]))
|
||||
: undefined)
|
||||
.filter((script) => Boolean(script));
|
||||
if (requiredScripts.length > 0) {
|
||||
await Promise.all(requiredScripts.map(script => this.ensureScriptLoaded(script)));
|
||||
}
|
||||
return this.runSerialized(async (raw) => {
|
||||
if (typeof raw.exec === 'function') {
|
||||
const nativeBatch = createGlideBatch(true);
|
||||
const batch = nativeBatch !== null && nativeBatch !== void 0 ? nativeBatch : createCompatibilityBatch();
|
||||
if (batch) {
|
||||
for (const command of commands) {
|
||||
batch.customCommand(command.args);
|
||||
}
|
||||
let execResult;
|
||||
try {
|
||||
execResult = await raw.exec(batch, false);
|
||||
}
|
||||
catch (err) {
|
||||
if (nativeBatch) {
|
||||
throw err;
|
||||
}
|
||||
execResult = null;
|
||||
}
|
||||
if (!execResult) {
|
||||
if (nativeBatch) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
else {
|
||||
return execResult.map((value, index) => {
|
||||
var _a;
|
||||
if (value instanceof Error) {
|
||||
return [value, null];
|
||||
}
|
||||
const transform = (_a = commands[index]) === null || _a === void 0 ? void 0 : _a.transform;
|
||||
return [null, transform ? transform(value) : value];
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
await raw.customCommand(['MULTI']);
|
||||
try {
|
||||
for (const command of commands) {
|
||||
// During MULTI queueing Redis replies with "QUEUED". For commands such
|
||||
// as HGETALL, Glide's default decoder expects a structured reply and
|
||||
// throws if it receives this simple string, so force string decoding.
|
||||
await raw.customCommand(command.args, {
|
||||
decoder: GLIDE_STRING_DECODER,
|
||||
});
|
||||
}
|
||||
const execResult = await raw.customCommand(['EXEC']);
|
||||
if (!execResult) {
|
||||
return null;
|
||||
}
|
||||
const results = Array.isArray(execResult) ? execResult : [execResult];
|
||||
return results.map((value, index) => {
|
||||
var _a;
|
||||
if (value instanceof Error) {
|
||||
return [value, null];
|
||||
}
|
||||
const transform = (_a = commands[index]) === null || _a === void 0 ? void 0 : _a.transform;
|
||||
return [null, transform ? transform(value) : value];
|
||||
});
|
||||
}
|
||||
catch (err) {
|
||||
try {
|
||||
await raw.customCommand(['DISCARD']);
|
||||
}
|
||||
catch (_a) {
|
||||
// ignore
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
class ValkeyGlideTransaction {
|
||||
constructor(adapter, scripts) {
|
||||
this.adapter = adapter;
|
||||
this.scripts = scripts;
|
||||
this.commands = [];
|
||||
}
|
||||
queueCommand(args, transform) {
|
||||
this.commands.push({ args, transform });
|
||||
return this;
|
||||
}
|
||||
hgetall(key) {
|
||||
return this.queueCommand(['HGETALL', key], normalizeHashReply);
|
||||
}
|
||||
hset(key, data) {
|
||||
const args = ['HSET', key];
|
||||
for (const [field, value] of Object.entries(data)) {
|
||||
args.push(field, toGlideArg(value));
|
||||
}
|
||||
return this.queueCommand(args);
|
||||
}
|
||||
hscan(key, cursor, options) {
|
||||
const args = ['HSCAN', key, String(cursor)];
|
||||
if ((options === null || options === void 0 ? void 0 : options.COUNT) != null) {
|
||||
args.push('COUNT', String(options.COUNT));
|
||||
}
|
||||
return this.queueCommand(args, value => {
|
||||
if (Array.isArray(value) && value.length >= 2) {
|
||||
return [toStringValue(value[0]), flattenFieldPairs(value[1])];
|
||||
}
|
||||
return ['0', []];
|
||||
});
|
||||
}
|
||||
smembers(key) {
|
||||
return this.queueCommand(['SMEMBERS', key], value => Array.isArray(value) ? value.map(item => toStringValue(item)) : []);
|
||||
}
|
||||
sscan(key, cursor, options) {
|
||||
const args = ['SSCAN', key, String(cursor)];
|
||||
if ((options === null || options === void 0 ? void 0 : options.COUNT) != null) {
|
||||
args.push('COUNT', String(options.COUNT));
|
||||
}
|
||||
return this.queueCommand(args, value => {
|
||||
if (Array.isArray(value) && value.length >= 2) {
|
||||
const members = Array.isArray(value[1])
|
||||
? value[1].map(item => toStringValue(item))
|
||||
: [];
|
||||
return [toStringValue(value[0]), members];
|
||||
}
|
||||
return ['0', []];
|
||||
});
|
||||
}
|
||||
zrange(key, start, end) {
|
||||
return this.queueCommand(['ZRANGE', key, String(start), String(end)]);
|
||||
}
|
||||
lrange(key, start, end) {
|
||||
return this.queueCommand(['LRANGE', key, String(start), String(end)]);
|
||||
}
|
||||
llen(key) {
|
||||
return this.queueCommand(['LLEN', key]);
|
||||
}
|
||||
del(...keys) {
|
||||
if (keys.length > 0) {
|
||||
this.queueCommand(['DEL', ...keys]);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
runCommand(name, args) {
|
||||
const script = this.scripts.get(name);
|
||||
if (!script) {
|
||||
throw new Error(`BullMQ: command "${name}" is not defined. Use defineCommand() before adding it to transactions.`);
|
||||
}
|
||||
const normalizedArgs = normalizeScriptArgs(args);
|
||||
const keys = normalizedArgs.slice(0, script.numberOfKeys).map(toGlideArg);
|
||||
const argv = normalizedArgs.slice(script.numberOfKeys).map(toGlideArg);
|
||||
return this.queueCommand([
|
||||
'EVALSHA',
|
||||
script.sha,
|
||||
String(script.numberOfKeys),
|
||||
...keys,
|
||||
...argv,
|
||||
]);
|
||||
}
|
||||
exec() {
|
||||
return this.adapter.execQueuedCommands(this.commands);
|
||||
}
|
||||
}
|
||||
288
node_modules/bullmq/dist/esm/classes/worker.d.ts
generated
vendored
Normal file
288
node_modules/bullmq/dist/esm/classes/worker.d.ts
generated
vendored
Normal file
@@ -0,0 +1,288 @@
|
||||
import { URL } from 'url';
|
||||
import { AbortController } from './abort-controller';
|
||||
import { BackendFactory, GetNextJobOptions, IoredisListener, IQueueBackend, JobJson, Span, WorkerOptions } from '../interfaces';
|
||||
import { JobProgress } from '../types';
|
||||
import { Processor } from '../types/processor';
|
||||
import { QueueBase } from './queue-base';
|
||||
import { RedisQueueBackend } from './redis-queue-backend';
|
||||
import { Job } from './job';
|
||||
import { JobScheduler } from './job-scheduler';
|
||||
import { LockManager } from './lock-manager';
|
||||
export interface WorkerListener<DataType = any, ResultType = any, NameType extends string = string> extends IoredisListener {
|
||||
/**
|
||||
* Listen to 'active' event.
|
||||
*
|
||||
* This event is triggered when a job enters the 'active' state.
|
||||
*/
|
||||
active: (job: Job<DataType, ResultType, NameType>, prev: string) => void;
|
||||
/**
|
||||
* Listen to 'closed' event.
|
||||
*
|
||||
* This event is triggered when the worker is closed.
|
||||
*/
|
||||
closed: () => void;
|
||||
/**
|
||||
* Listen to 'closing' event.
|
||||
*
|
||||
* This event is triggered when the worker is closing.
|
||||
*/
|
||||
closing: (msg: string) => void;
|
||||
/**
|
||||
* Listen to 'completed' event.
|
||||
*
|
||||
* This event is triggered when a job has successfully completed.
|
||||
*/
|
||||
completed: (job: Job<DataType, ResultType, NameType>, result: ResultType, prev: string) => void;
|
||||
/**
|
||||
* Listen to 'drained' event.
|
||||
*
|
||||
* This event is triggered when the queue has drained the waiting list.
|
||||
* Note that there could still be delayed jobs waiting their timers to expire
|
||||
* and this event will still be triggered as long as the waiting list has emptied.
|
||||
*/
|
||||
drained: () => void;
|
||||
/**
|
||||
* Listen to 'error' event.
|
||||
*
|
||||
* This event is triggered when an error is throw.
|
||||
*/
|
||||
error: (failedReason: Error) => void;
|
||||
/**
|
||||
* Listen to 'failed' event.
|
||||
*
|
||||
* This event is triggered when a job has thrown an exception.
|
||||
* Note: job parameter could be received as undefined when an stalled job
|
||||
* reaches the stalled limit and it is deleted by the removeOnFail option.
|
||||
*/
|
||||
failed: (job: Job<DataType, ResultType, NameType> | undefined, error: Error, prev: string) => void;
|
||||
/**
|
||||
* Listen to 'paused' event.
|
||||
*
|
||||
* This event is triggered when the queue is paused.
|
||||
*/
|
||||
paused: () => void;
|
||||
/**
|
||||
* Listen to 'progress' event.
|
||||
*
|
||||
* This event is triggered when a job updates it progress, i.e. the
|
||||
* Job##updateProgress() method is called. This is useful to notify
|
||||
* progress or any other data from within a processor to the rest of the
|
||||
* world.
|
||||
*/
|
||||
progress: (job: Job<DataType, ResultType, NameType>, progress: JobProgress) => void;
|
||||
/**
|
||||
* Listen to 'ready' event.
|
||||
*
|
||||
* This event is triggered when blockingConnection is ready.
|
||||
*/
|
||||
ready: () => void;
|
||||
/**
|
||||
* Listen to 'resumed' event.
|
||||
*
|
||||
* This event is triggered when the queue is resumed.
|
||||
*/
|
||||
resumed: () => void;
|
||||
/**
|
||||
* Listen to 'stalled' event.
|
||||
*
|
||||
* This event is triggered when a job has stalled and
|
||||
* has been moved back to the wait list.
|
||||
*/
|
||||
stalled: (jobId: string, prev: string) => void;
|
||||
/**
|
||||
* Listen to 'lockRenewalFailed' event.
|
||||
*
|
||||
* This event is triggered when lock renewal fails for one or more jobs.
|
||||
*/
|
||||
lockRenewalFailed: (jobIds: string[]) => void;
|
||||
/**
|
||||
* Listen to 'locksRenewed' event.
|
||||
*
|
||||
* This event is triggered when locks are successfully renewed.
|
||||
*/
|
||||
locksRenewed: (data: {
|
||||
count: number;
|
||||
jobIds: string[];
|
||||
}) => void;
|
||||
}
|
||||
/**
|
||||
*
|
||||
* This class represents a worker that is able to process jobs from the queue.
|
||||
* As soon as the class is instantiated and a connection to Redis is established
|
||||
* it will start processing jobs.
|
||||
*
|
||||
*/
|
||||
export declare class Worker<DataType = any, ResultType = any, NameType extends string = string, B extends IQueueBackend = RedisQueueBackend> extends QueueBase<B> {
|
||||
readonly opts: WorkerOptions;
|
||||
readonly id: string;
|
||||
private abortDelayController;
|
||||
private blockUntil;
|
||||
private _concurrency;
|
||||
private childPool;
|
||||
private drained;
|
||||
private limitUntil;
|
||||
protected lockManager: LockManager;
|
||||
private processorAcceptsSignal;
|
||||
private stalledCheckerRunning;
|
||||
private stalledCheckStopper?;
|
||||
private waiting;
|
||||
protected _jobScheduler: JobScheduler;
|
||||
protected paused: boolean;
|
||||
protected processFn: Processor<DataType, ResultType, NameType>;
|
||||
protected running: boolean;
|
||||
protected mainLoopRunning: Promise<void> | null;
|
||||
static RateLimitError(): Error;
|
||||
constructor(name: string, processor?: string | URL | null | Processor<DataType, ResultType, NameType>, opts?: WorkerOptions, backendFactory?: BackendFactory<B>);
|
||||
/**
|
||||
* Builds the worker's backend, which owns both the regular connection and a
|
||||
* dedicated blocking connection used by the `waitForJob` primitive.
|
||||
*/
|
||||
protected createBackend(): void;
|
||||
/**
|
||||
* Creates and configures the lock manager for processing jobs.
|
||||
* This method can be overridden in subclasses to customize lock manager behavior.
|
||||
*/
|
||||
protected createLockManager(): void;
|
||||
/**
|
||||
* Creates and configures the sandbox for processing jobs.
|
||||
* This method can be overridden in subclasses to customize sandbox behavior.
|
||||
*
|
||||
* @param processor - The processor file path, URL, or function to be sandboxed
|
||||
*/
|
||||
protected createSandbox(processor: string | URL | null | Processor<DataType, ResultType, NameType>): void;
|
||||
/**
|
||||
* Public accessor method for LockManager to extend locks.
|
||||
* This delegates to the protected scripts object.
|
||||
*/
|
||||
extendJobLocks(jobIds: string[], tokens: string[], duration: number): Promise<string[]>;
|
||||
emit<U extends keyof WorkerListener<DataType, ResultType, NameType>>(event: U, ...args: Parameters<WorkerListener<DataType, ResultType, NameType>[U]>): boolean;
|
||||
off<U extends keyof WorkerListener<DataType, ResultType, NameType>>(eventName: U, listener: WorkerListener<DataType, ResultType, NameType>[U]): this;
|
||||
on<U extends keyof WorkerListener<DataType, ResultType, NameType>>(event: U, listener: WorkerListener<DataType, ResultType, NameType>[U]): this;
|
||||
once<U extends keyof WorkerListener<DataType, ResultType, NameType>>(event: U, listener: WorkerListener<DataType, ResultType, NameType>[U]): this;
|
||||
protected callProcessJob(job: Job<DataType, ResultType, NameType>, token: string, signal?: AbortSignal): Promise<ResultType>;
|
||||
protected createJob(data: JobJson, jobId: string): Job<DataType, ResultType, NameType>;
|
||||
/**
|
||||
*
|
||||
* Waits until the worker is ready to start processing jobs.
|
||||
* In general only useful when writing tests.
|
||||
*
|
||||
*/
|
||||
waitUntilReady(): Promise<void>;
|
||||
/**
|
||||
* Cancels a specific job currently being processed by this worker.
|
||||
* The job's processor function will receive an abort signal.
|
||||
*
|
||||
* @param jobId - The ID of the job to cancel
|
||||
* @param reason - Optional reason for the cancellation
|
||||
* @returns true if the job was found and cancelled, false otherwise
|
||||
*/
|
||||
cancelJob(jobId: string, reason?: string): boolean;
|
||||
/**
|
||||
* Cancels all jobs currently being processed by this worker.
|
||||
* All active job processor functions will receive abort signals.
|
||||
*
|
||||
* @param reason - Optional reason for the cancellation
|
||||
*/
|
||||
cancelAllJobs(reason?: string): void;
|
||||
set concurrency(concurrency: number);
|
||||
get concurrency(): number;
|
||||
get jobScheduler(): Promise<JobScheduler>;
|
||||
run(): Promise<void>;
|
||||
private waitForRateLimit;
|
||||
/**
|
||||
* This is the main loop in BullMQ. Its goals are to fetch jobs from the queue
|
||||
* as efficiently as possible, providing concurrency and minimal unnecessary calls
|
||||
* to Redis.
|
||||
*/
|
||||
private mainLoop;
|
||||
/**
|
||||
* Returns a promise that resolves to the next job in queue.
|
||||
* @param token - worker token to be assigned to retrieved job
|
||||
* @returns a Job or undefined if no job was available in the queue.
|
||||
*/
|
||||
getNextJob(token: string, { block }?: GetNextJobOptions): Promise<Job<DataType, ResultType, NameType>>;
|
||||
private _getNextJob;
|
||||
/**
|
||||
* Overrides the rate limit to be active for the next jobs.
|
||||
* @deprecated This method is deprecated and will be removed in v6. Use queue.rateLimit method instead.
|
||||
* @param expireTimeMs - expire time in ms of this rate limit.
|
||||
*/
|
||||
rateLimit(expireTimeMs: number): Promise<void>;
|
||||
get minimumBlockTimeout(): number;
|
||||
private isRateLimited;
|
||||
protected moveToActive(token: string, name?: string): Promise<Job<DataType, ResultType, NameType>>;
|
||||
private waitForJob;
|
||||
protected getBlockTimeout(blockUntil: number): number;
|
||||
protected getRateLimitDelay(delay: number): number;
|
||||
/**
|
||||
*
|
||||
* This function is exposed only for testing purposes.
|
||||
*/
|
||||
delay(milliseconds?: number, abortController?: AbortController): Promise<void>;
|
||||
private updateDelays;
|
||||
protected nextJobFromJobData(jobData?: JobJson, jobId?: string, token?: string): Promise<Job<DataType, ResultType, NameType>>;
|
||||
processJob(job: Job<DataType, ResultType, NameType>, token: string, fetchNextCallback?: () => boolean): Promise<void | Job<DataType, ResultType, NameType>>;
|
||||
private getUnrecoverableErrorMessage;
|
||||
protected handleCompleted(result: ResultType, job: Job<DataType, ResultType, NameType>, token: string, fetchNextCallback?: () => boolean, span?: Span): Promise<Job<DataType, ResultType, NameType>>;
|
||||
protected handleFailed(err: Error, job: Job<DataType, ResultType, NameType>, token: string, fetchNextCallback?: () => boolean, span?: Span): Promise<Job<DataType, ResultType, NameType>>;
|
||||
/**
|
||||
*
|
||||
* Pauses the processing of this queue only for this worker.
|
||||
*/
|
||||
pause(doNotWaitActive?: boolean): Promise<void>;
|
||||
/**
|
||||
*
|
||||
* Resumes processing of this worker (if paused).
|
||||
*/
|
||||
resume(): Promise<void>;
|
||||
/**
|
||||
*
|
||||
* Checks if worker is paused.
|
||||
*
|
||||
* @returns true if worker is paused, false otherwise.
|
||||
*/
|
||||
isPaused(): boolean;
|
||||
/**
|
||||
*
|
||||
* Checks if worker is currently running.
|
||||
*
|
||||
* @returns true if worker is running, false otherwise.
|
||||
*/
|
||||
isRunning(): boolean;
|
||||
/**
|
||||
*
|
||||
* Closes the worker and related redis connections.
|
||||
*
|
||||
* This method waits for current jobs to finalize before returning.
|
||||
*
|
||||
* @param force - Use force boolean parameter if you do not want to wait for
|
||||
* current jobs to be processed. When using telemetry, be mindful that it can
|
||||
* interfere with the proper closure of spans, potentially preventing them from being exported.
|
||||
*
|
||||
* @returns Promise that resolves when the worker has been closed.
|
||||
*/
|
||||
close(force?: boolean): Promise<void>;
|
||||
/**
|
||||
*
|
||||
* Manually starts the stalled checker.
|
||||
* The check will run once as soon as this method is called, and
|
||||
* then every opts.stalledInterval milliseconds until the worker is closed.
|
||||
* Note: Normally you do not need to call this method, since the stalled checker
|
||||
* is automatically started when the worker starts processing jobs after
|
||||
* calling run. However if you want to process the jobs manually you need
|
||||
* to call this method to start the stalled checker.
|
||||
*
|
||||
* @see {@link https://docs.bullmq.io/patterns/manually-fetching-jobs}
|
||||
*/
|
||||
startStalledCheckTimer(): Promise<void>;
|
||||
private stalledChecker;
|
||||
/**
|
||||
* Returns a promise that resolves when active jobs are cleared
|
||||
*
|
||||
* @returns
|
||||
*/
|
||||
private whenCurrentJobsFinished;
|
||||
private retryIfFailed;
|
||||
private moveStalledJobsToWait;
|
||||
private moveLimitedBackToWait;
|
||||
}
|
||||
888
node_modules/bullmq/dist/esm/classes/worker.js
generated
vendored
Normal file
888
node_modules/bullmq/dist/esm/classes/worker.js
generated
vendored
Normal file
@@ -0,0 +1,888 @@
|
||||
import * as fs from 'fs';
|
||||
import { URL } from 'url';
|
||||
import * as path from 'path';
|
||||
import { AbortController } from './abort-controller';
|
||||
import { delay, DELAY_TIME_1, isNotConnectionError, randomUUID, } from '../utils';
|
||||
import { QueueBase } from './queue-base';
|
||||
import { ChildPool } from './child-pool';
|
||||
import sandbox from './sandbox';
|
||||
import { AsyncFifoQueue } from './async-fifo-queue';
|
||||
import { DelayedError, RateLimitError, RATE_LIMIT_ERROR, WaitingChildrenError, WaitingError, UnrecoverableError, } from './errors';
|
||||
import { SpanKind, TelemetryAttributes } from '../enums';
|
||||
import { getLegacyRepeatableJobError, hasLegacyRepeatableKeyShape, JobScheduler, } from './job-scheduler';
|
||||
import { LockManager } from './lock-manager';
|
||||
// 10 seconds is the maximum time a BZPOPMIN can block.
|
||||
const maximumBlockTimeout = 10;
|
||||
/**
|
||||
*
|
||||
* This class represents a worker that is able to process jobs from the queue.
|
||||
* As soon as the class is instantiated and a connection to Redis is established
|
||||
* it will start processing jobs.
|
||||
*
|
||||
*/
|
||||
export class Worker extends QueueBase {
|
||||
static RateLimitError() {
|
||||
return new RateLimitError();
|
||||
}
|
||||
constructor(name, processor, opts, backendFactory) {
|
||||
super(name, Object.assign(Object.assign({ drainDelay: 5, concurrency: 1, lockDuration: 30000, maximumRateLimitDelay: 30000, maxStalledCount: 1, stalledInterval: 30000, autorun: true, runRetryDelay: 15000 }, opts), { blockingConnection: true }), backendFactory);
|
||||
this.abortDelayController = null;
|
||||
this.blockUntil = 0;
|
||||
this.drained = false;
|
||||
this.limitUntil = 0;
|
||||
this.processorAcceptsSignal = false;
|
||||
this.stalledCheckerRunning = false;
|
||||
this.waiting = null;
|
||||
this.running = false;
|
||||
this.mainLoopRunning = null;
|
||||
if (!opts || !opts.connection) {
|
||||
throw new Error('Worker requires a connection');
|
||||
}
|
||||
if (typeof this.opts.maxStalledCount !== 'number' ||
|
||||
this.opts.maxStalledCount < 0) {
|
||||
throw new Error('maxStalledCount must be greater or equal than 0');
|
||||
}
|
||||
if (typeof this.opts.maxStartedAttempts === 'number' &&
|
||||
this.opts.maxStartedAttempts < 0) {
|
||||
throw new Error('maxStartedAttempts must be greater or equal than 0');
|
||||
}
|
||||
if (typeof this.opts.stalledInterval !== 'number' ||
|
||||
this.opts.stalledInterval <= 0) {
|
||||
throw new Error('stalledInterval must be greater than 0');
|
||||
}
|
||||
if (typeof this.opts.drainDelay !== 'number' || this.opts.drainDelay <= 0) {
|
||||
throw new Error('drainDelay must be greater than 0');
|
||||
}
|
||||
this.concurrency = this.opts.concurrency;
|
||||
this.opts.lockRenewTime =
|
||||
this.opts.lockRenewTime || this.opts.lockDuration / 2;
|
||||
this.id = randomUUID();
|
||||
this.createLockManager();
|
||||
if (processor) {
|
||||
if (typeof processor === 'function') {
|
||||
this.processFn = processor;
|
||||
// Check if processor accepts signal parameter (3rd parameter)
|
||||
this.processorAcceptsSignal = processor.length >= 3;
|
||||
}
|
||||
else {
|
||||
// SANDBOXED
|
||||
if (processor instanceof URL) {
|
||||
if (!fs.existsSync(processor)) {
|
||||
throw new Error(`URL ${processor} does not exist in the local file system`);
|
||||
}
|
||||
processor = processor.href;
|
||||
}
|
||||
else {
|
||||
const supportedFileTypes = ['.js', '.ts', '.flow', '.cjs', '.mjs'];
|
||||
const processorFile = processor +
|
||||
(supportedFileTypes.includes(path.extname(processor)) ? '' : '.js');
|
||||
if (!fs.existsSync(processorFile)) {
|
||||
throw new Error(`File ${processorFile} does not exist`);
|
||||
}
|
||||
}
|
||||
// Separate paths so that bundling tools can resolve dependencies easier
|
||||
const dirname = path.dirname(module.filename || __filename);
|
||||
const workerThreadsMainFile = path.join(dirname, 'main-worker.js');
|
||||
const spawnProcessMainFile = path.join(dirname, 'main.js');
|
||||
let mainFilePath = this.opts.useWorkerThreads
|
||||
? workerThreadsMainFile
|
||||
: spawnProcessMainFile;
|
||||
try {
|
||||
fs.statSync(mainFilePath); // would throw if file not exists
|
||||
}
|
||||
catch (_) {
|
||||
const mainFile = this.opts.useWorkerThreads
|
||||
? 'main-worker.js'
|
||||
: 'main.js';
|
||||
mainFilePath = path.join(process.cwd(), `dist/cjs/classes/${mainFile}`);
|
||||
fs.statSync(mainFilePath);
|
||||
}
|
||||
this.childPool = new ChildPool({
|
||||
mainFile: mainFilePath,
|
||||
useWorkerThreads: this.opts.useWorkerThreads,
|
||||
workerForkOptions: this.opts.workerForkOptions,
|
||||
workerThreadsOptions: this.opts.workerThreadsOptions,
|
||||
});
|
||||
this.createSandbox(processor);
|
||||
this.processorAcceptsSignal = true;
|
||||
}
|
||||
if (this.opts.autorun) {
|
||||
this.run().catch(error => this.emit('error', error));
|
||||
}
|
||||
}
|
||||
// The Worker is only "ready" once its dedicated blocking connection (used
|
||||
// by the blocking `waitForJob` primitive) is ready. The backend forwards a
|
||||
// 'ready' event from *either* of its connections, so we cannot rely on that
|
||||
// event to gate on blocking readiness. Instead emit a single 'ready' once
|
||||
// `backend.waitUntilReady()` resolves, which only happens after both the
|
||||
// main and the blocking connections are ready. Connection failures are
|
||||
// surfaced via the 'error' event wired up in QueueBase.
|
||||
this.backend
|
||||
.waitUntilReady()
|
||||
.then(() => setTimeout(() => this.emit('ready'), 0))
|
||||
.catch(() => { });
|
||||
}
|
||||
/**
|
||||
* Builds the worker's backend, which owns both the regular connection and a
|
||||
* dedicated blocking connection used by the `waitForJob` primitive.
|
||||
*/
|
||||
createBackend() {
|
||||
this.backend = this.backendFactory(this.name, this.opts, {
|
||||
withBlockingConnection: true,
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Creates and configures the lock manager for processing jobs.
|
||||
* This method can be overridden in subclasses to customize lock manager behavior.
|
||||
*/
|
||||
createLockManager() {
|
||||
this.lockManager = new LockManager(this, {
|
||||
lockRenewTime: this.opts.lockRenewTime,
|
||||
lockDuration: this.opts.lockDuration,
|
||||
workerId: this.id,
|
||||
workerName: this.opts.name,
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Creates and configures the sandbox for processing jobs.
|
||||
* This method can be overridden in subclasses to customize sandbox behavior.
|
||||
*
|
||||
* @param processor - The processor file path, URL, or function to be sandboxed
|
||||
*/
|
||||
createSandbox(processor) {
|
||||
this.processFn = sandbox(processor, this.childPool).bind(this);
|
||||
}
|
||||
/**
|
||||
* Public accessor method for LockManager to extend locks.
|
||||
* This delegates to the protected scripts object.
|
||||
*/
|
||||
async extendJobLocks(jobIds, tokens, duration) {
|
||||
return this.backend.extendLocks(jobIds, tokens, duration);
|
||||
}
|
||||
emit(event, ...args) {
|
||||
return super.emit(event, ...args);
|
||||
}
|
||||
off(eventName, listener) {
|
||||
super.off(eventName, listener);
|
||||
return this;
|
||||
}
|
||||
on(event, listener) {
|
||||
super.on(event, listener);
|
||||
return this;
|
||||
}
|
||||
once(event, listener) {
|
||||
super.once(event, listener);
|
||||
return this;
|
||||
}
|
||||
callProcessJob(job, token, signal) {
|
||||
return this.processFn(job, token, signal);
|
||||
}
|
||||
createJob(data, jobId) {
|
||||
return this.Job.fromJSON(this, data, jobId);
|
||||
}
|
||||
/**
|
||||
*
|
||||
* Waits until the worker is ready to start processing jobs.
|
||||
* In general only useful when writing tests.
|
||||
*
|
||||
*/
|
||||
async waitUntilReady() {
|
||||
await super.waitUntilReady();
|
||||
}
|
||||
/**
|
||||
* Cancels a specific job currently being processed by this worker.
|
||||
* The job's processor function will receive an abort signal.
|
||||
*
|
||||
* @param jobId - The ID of the job to cancel
|
||||
* @param reason - Optional reason for the cancellation
|
||||
* @returns true if the job was found and cancelled, false otherwise
|
||||
*/
|
||||
cancelJob(jobId, reason) {
|
||||
return this.lockManager.cancelJob(jobId, reason);
|
||||
}
|
||||
/**
|
||||
* Cancels all jobs currently being processed by this worker.
|
||||
* All active job processor functions will receive abort signals.
|
||||
*
|
||||
* @param reason - Optional reason for the cancellation
|
||||
*/
|
||||
cancelAllJobs(reason) {
|
||||
this.lockManager.cancelAllJobs(reason);
|
||||
}
|
||||
set concurrency(concurrency) {
|
||||
if (typeof concurrency !== 'number' ||
|
||||
concurrency < 1 ||
|
||||
!isFinite(concurrency)) {
|
||||
throw new Error('concurrency must be a finite number greater than 0');
|
||||
}
|
||||
this._concurrency = concurrency;
|
||||
}
|
||||
get concurrency() {
|
||||
return this._concurrency;
|
||||
}
|
||||
get jobScheduler() {
|
||||
return new Promise(async (resolve) => {
|
||||
if (!this._jobScheduler) {
|
||||
// Share the worker's backend (same queue) with the scheduler.
|
||||
this._jobScheduler = new JobScheduler(this.name, this.opts, () => this.backend);
|
||||
this._jobScheduler.on('error', this.emit.bind(this, 'error'));
|
||||
}
|
||||
resolve(this._jobScheduler);
|
||||
});
|
||||
}
|
||||
async run() {
|
||||
if (!this.processFn) {
|
||||
throw new Error('No process function is defined.');
|
||||
}
|
||||
if (this.running) {
|
||||
throw new Error('Worker is already running.');
|
||||
}
|
||||
try {
|
||||
this.running = true;
|
||||
if (this.closing || this.paused) {
|
||||
return;
|
||||
}
|
||||
await this.startStalledCheckTimer();
|
||||
if (!this.opts.skipLockRenewal) {
|
||||
this.lockManager.start();
|
||||
}
|
||||
this.mainLoopRunning = this.mainLoop();
|
||||
// We must await here or finally will be called too early.
|
||||
await this.mainLoopRunning;
|
||||
}
|
||||
finally {
|
||||
this.running = false;
|
||||
}
|
||||
}
|
||||
async waitForRateLimit() {
|
||||
var _a;
|
||||
const limitUntil = this.limitUntil;
|
||||
if (limitUntil > Date.now()) {
|
||||
(_a = this.abortDelayController) === null || _a === void 0 ? void 0 : _a.abort();
|
||||
this.abortDelayController = new AbortController();
|
||||
const delay = this.getRateLimitDelay(limitUntil - Date.now());
|
||||
await this.delay(delay, this.abortDelayController);
|
||||
this.drained = false;
|
||||
this.limitUntil = 0;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* This is the main loop in BullMQ. Its goals are to fetch jobs from the queue
|
||||
* as efficiently as possible, providing concurrency and minimal unnecessary calls
|
||||
* to Redis.
|
||||
*/
|
||||
async mainLoop() {
|
||||
const asyncFifoQueue = new AsyncFifoQueue();
|
||||
let tokenPostfix = 0;
|
||||
while ((!this.closing && !this.paused) || asyncFifoQueue.numTotal() > 0) {
|
||||
/**
|
||||
* This inner loop tries to fetch jobs concurrently, but if we are waiting for a job
|
||||
* to arrive at the queue we should not try to fetch more jobs (as it would be pointless)
|
||||
*/
|
||||
while (!this.closing &&
|
||||
!this.paused &&
|
||||
!this.waiting &&
|
||||
asyncFifoQueue.numTotal() < this._concurrency &&
|
||||
!this.isRateLimited()) {
|
||||
const token = `${this.id}:${tokenPostfix++}`;
|
||||
const fetchedJob = this.retryIfFailed(() => this._getNextJob(token, { block: true }), {
|
||||
delayInMs: this.opts.runRetryDelay,
|
||||
onlyEmitError: true,
|
||||
});
|
||||
asyncFifoQueue.add(fetchedJob);
|
||||
if (this.waiting && asyncFifoQueue.numTotal() > 1) {
|
||||
// We are waiting for jobs but we have others that we could start processing already
|
||||
break;
|
||||
}
|
||||
// We await here so that we fetch jobs in sequence, this is important to avoid unnecessary calls
|
||||
// to Redis in high concurrency scenarios.
|
||||
const job = await fetchedJob;
|
||||
// No more jobs waiting but we have others that could start processing already
|
||||
if (!job && asyncFifoQueue.numTotal() > 1) {
|
||||
break;
|
||||
}
|
||||
// If there are potential jobs to be processed and blockUntil is set, we should exit to avoid waiting
|
||||
// for processing this job.
|
||||
if (this.blockUntil) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Since there can be undefined jobs in the queue (when a job fails or queue is empty)
|
||||
// we iterate until we find a job.
|
||||
let job;
|
||||
do {
|
||||
job = await asyncFifoQueue.fetch();
|
||||
} while (!job && asyncFifoQueue.numQueued() > 0);
|
||||
if (job) {
|
||||
const token = job.token;
|
||||
asyncFifoQueue.add(this.processJob(job, token, () => asyncFifoQueue.numTotal() <= this._concurrency));
|
||||
}
|
||||
else if (asyncFifoQueue.numQueued() === 0) {
|
||||
await this.waitForRateLimit();
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Returns a promise that resolves to the next job in queue.
|
||||
* @param token - worker token to be assigned to retrieved job
|
||||
* @returns a Job or undefined if no job was available in the queue.
|
||||
*/
|
||||
async getNextJob(token, { block = true } = {}) {
|
||||
var _a, _b;
|
||||
const nextJob = await this._getNextJob(token, { block });
|
||||
return this.trace(SpanKind.INTERNAL, 'getNextJob', this.name, async (span) => {
|
||||
span === null || span === void 0 ? void 0 : span.setAttributes({
|
||||
[TelemetryAttributes.WorkerId]: this.id,
|
||||
[TelemetryAttributes.QueueName]: this.name,
|
||||
[TelemetryAttributes.WorkerName]: this.opts.name,
|
||||
[TelemetryAttributes.WorkerOptions]: JSON.stringify({ block }),
|
||||
[TelemetryAttributes.JobId]: nextJob === null || nextJob === void 0 ? void 0 : nextJob.id,
|
||||
});
|
||||
return nextJob;
|
||||
}, (_b = (_a = nextJob === null || nextJob === void 0 ? void 0 : nextJob.opts) === null || _a === void 0 ? void 0 : _a.telemetry) === null || _b === void 0 ? void 0 : _b.metadata);
|
||||
}
|
||||
async _getNextJob(token, { block = true } = {}) {
|
||||
if (this.paused) {
|
||||
return;
|
||||
}
|
||||
if (this.closing) {
|
||||
return;
|
||||
}
|
||||
let job;
|
||||
if (this.drained && block && !this.limitUntil && !this.waiting) {
|
||||
this.waiting = this.waitForJob(this.blockUntil);
|
||||
try {
|
||||
this.blockUntil = await this.waiting;
|
||||
if (this.blockUntil <= 0 || this.blockUntil - Date.now() < 1) {
|
||||
job = await this.moveToActive(token, this.opts.name);
|
||||
}
|
||||
}
|
||||
finally {
|
||||
this.waiting = null;
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (!this.isRateLimited()) {
|
||||
job = await this.moveToActive(token, this.opts.name);
|
||||
}
|
||||
}
|
||||
return job;
|
||||
}
|
||||
/**
|
||||
* Overrides the rate limit to be active for the next jobs.
|
||||
* @deprecated This method is deprecated and will be removed in v6. Use queue.rateLimit method instead.
|
||||
* @param expireTimeMs - expire time in ms of this rate limit.
|
||||
*/
|
||||
async rateLimit(expireTimeMs) {
|
||||
await this.trace(SpanKind.INTERNAL, 'rateLimit', this.name, async (span) => {
|
||||
span === null || span === void 0 ? void 0 : span.setAttributes({
|
||||
[TelemetryAttributes.WorkerId]: this.id,
|
||||
[TelemetryAttributes.WorkerRateLimit]: expireTimeMs,
|
||||
});
|
||||
await this.backend.setRateLimit(expireTimeMs);
|
||||
});
|
||||
}
|
||||
get minimumBlockTimeout() {
|
||||
return this.backend.minimumBlockTimeout;
|
||||
}
|
||||
isRateLimited() {
|
||||
return this.limitUntil > Date.now();
|
||||
}
|
||||
async moveToActive(token, name) {
|
||||
const [jobData, id, rateLimitDelay, delayUntil] = await this.backend.moveToActive(token, name);
|
||||
this.updateDelays(rateLimitDelay, delayUntil);
|
||||
return this.nextJobFromJobData(jobData, id, token);
|
||||
}
|
||||
async waitForJob(blockUntil) {
|
||||
if (this.paused) {
|
||||
return Infinity;
|
||||
}
|
||||
try {
|
||||
if (!this.closing && !this.isRateLimited()) {
|
||||
const blockTimeout = this.getBlockTimeout(blockUntil);
|
||||
if (blockTimeout > 0) {
|
||||
this.updateDelays(); // reset delays to avoid reusing same values in next iteration
|
||||
// Markers should only be used for un-blocking, so we will handle them
|
||||
// in this function only. Capability-based timeout rounding and the
|
||||
// stuck-connection watchdog are owned by the backend's waitForJob.
|
||||
const result = await this.backend.waitForJob(blockTimeout);
|
||||
if (result) {
|
||||
const newBlockUntil = result.score;
|
||||
// Use by pro version as rate limited groups could generate lower blockUntil values
|
||||
// markers only return delays for delayed jobs
|
||||
if (blockUntil && newBlockUntil > blockUntil) {
|
||||
return blockUntil;
|
||||
}
|
||||
return newBlockUntil;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
if (isNotConnectionError(error)) {
|
||||
this.emit('error', error);
|
||||
}
|
||||
// The watchdog must abort every overdue blocking command, but doing so
|
||||
// during socketless Sentinel resolution can leave the blocking transport
|
||||
// disconnected. Ask the backend to recover it before retrying.
|
||||
if (!this.closing) {
|
||||
try {
|
||||
await this.backend.reconnectBlocking();
|
||||
}
|
||||
catch (reconnectError) {
|
||||
if (isNotConnectionError(reconnectError)) {
|
||||
this.emit('error', reconnectError);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!this.closing) {
|
||||
await this.delay();
|
||||
}
|
||||
}
|
||||
return Infinity;
|
||||
}
|
||||
getBlockTimeout(blockUntil) {
|
||||
const opts = this.opts;
|
||||
// when there are delayed jobs
|
||||
if (blockUntil) {
|
||||
const blockDelay = blockUntil - Date.now();
|
||||
// when we reach the time to get new jobs
|
||||
if (blockDelay <= 0) {
|
||||
return blockDelay;
|
||||
}
|
||||
else if (blockDelay < this.minimumBlockTimeout * 1000) {
|
||||
return this.minimumBlockTimeout;
|
||||
}
|
||||
else {
|
||||
// We restrict the maximum block timeout to 10 second to avoid
|
||||
// blocking the connection for too long in the case of reconnections
|
||||
// reference: https://github.com/taskforcesh/bullmq/issues/1658
|
||||
return Math.min(blockDelay / 1000, maximumBlockTimeout);
|
||||
}
|
||||
}
|
||||
else {
|
||||
return Math.max(opts.drainDelay, this.minimumBlockTimeout);
|
||||
}
|
||||
}
|
||||
getRateLimitDelay(delay) {
|
||||
// We restrict the maximum limit delay to the configured maximumRateLimitDelay
|
||||
// to be able to promote delayed jobs while the queue is rate limited
|
||||
return Math.min(delay, this.opts.maximumRateLimitDelay);
|
||||
}
|
||||
/**
|
||||
*
|
||||
* This function is exposed only for testing purposes.
|
||||
*/
|
||||
async delay(milliseconds, abortController) {
|
||||
await delay(milliseconds || DELAY_TIME_1, abortController);
|
||||
}
|
||||
updateDelays(limitDelay = 0, delayUntil = 0) {
|
||||
const clampedLimit = Math.max(limitDelay, 0);
|
||||
if (clampedLimit > 0) {
|
||||
this.limitUntil = Date.now() + clampedLimit;
|
||||
}
|
||||
else {
|
||||
this.limitUntil = 0;
|
||||
}
|
||||
this.blockUntil = Math.max(delayUntil, 0) || 0;
|
||||
}
|
||||
async nextJobFromJobData(jobData, jobId, token) {
|
||||
if (!jobData) {
|
||||
if (!this.drained) {
|
||||
this.emit('drained');
|
||||
this.drained = true;
|
||||
}
|
||||
}
|
||||
else {
|
||||
this.drained = false;
|
||||
const job = this.createJob(jobData, jobId);
|
||||
job.token = token;
|
||||
try {
|
||||
const shouldScheduleRepeat = await this.retryIfFailed(async () => {
|
||||
// We need to distinguish between new job schedulers and legacy
|
||||
// repeatable jobs. We first use a key-shape heuristic to detect
|
||||
// legacy repeat keys (`name:id:endDate:tz:<cron|every>`), then
|
||||
// probe the per-id scheduler metadata hash (`repeat:<id>` with
|
||||
// the `ic` field) via `JobScheduler.isJobScheduler()` for
|
||||
// disambiguation when needed (see issue #3828).
|
||||
const hasRepeatJobKey = !!job.repeatJobKey;
|
||||
const hasLegacyKeyShape = hasRepeatJobKey && hasLegacyRepeatableKeyShape(job.repeatJobKey);
|
||||
let isJobScheduler = hasRepeatJobKey && !hasLegacyKeyShape;
|
||||
if (hasLegacyKeyShape) {
|
||||
const jobScheduler = await this.jobScheduler;
|
||||
isJobScheduler = await jobScheduler.isJobScheduler(job.repeatJobKey);
|
||||
}
|
||||
if (isJobScheduler) {
|
||||
const jobScheduler = await this.jobScheduler;
|
||||
await jobScheduler.upsertJobScheduler(
|
||||
// Most of these arguments are not really needed
|
||||
// anymore as we read them from the job scheduler itself
|
||||
job.repeatJobKey, job.opts.repeat, job.name, job.data, job.opts, { override: false, producerId: job.id });
|
||||
}
|
||||
return !hasLegacyKeyShape || isJobScheduler;
|
||||
}, { delayInMs: this.opts.runRetryDelay });
|
||||
if (job.repeatJobKey && !shouldScheduleRepeat) {
|
||||
const schedulingError = new Error(`Failed to add repeatable job for next iteration: ${getLegacyRepeatableJobError(job.repeatJobKey).message}`);
|
||||
this.emit('error', schedulingError);
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
// Emit error but don't throw to avoid breaking current job completion
|
||||
// Note: This means the next repeatable job will not be scheduled
|
||||
const errorMessage = err instanceof Error ? err.message : String(err);
|
||||
const schedulingError = new Error(`Failed to add repeatable job for next iteration: ${errorMessage}`);
|
||||
this.emit('error', schedulingError);
|
||||
// Return undefined to indicate no next job is available
|
||||
return undefined;
|
||||
}
|
||||
this.emit('active', job, 'waiting');
|
||||
return job;
|
||||
}
|
||||
}
|
||||
async processJob(job, token, fetchNextCallback = () => true) {
|
||||
var _a, _b;
|
||||
const srcPropagationMetadata = (_b = (_a = job.opts) === null || _a === void 0 ? void 0 : _a.telemetry) === null || _b === void 0 ? void 0 : _b.metadata;
|
||||
return this.trace(SpanKind.CONSUMER, 'process', this.name, async (span) => {
|
||||
span === null || span === void 0 ? void 0 : span.setAttributes({
|
||||
[TelemetryAttributes.WorkerId]: this.id,
|
||||
[TelemetryAttributes.WorkerName]: this.opts.name,
|
||||
[TelemetryAttributes.JobId]: job.id,
|
||||
[TelemetryAttributes.JobName]: job.name,
|
||||
});
|
||||
const abortController = this.lockManager.trackJob(job.id, token, job.processedOn, this.processorAcceptsSignal);
|
||||
try {
|
||||
const unrecoverableErrorMessage = this.getUnrecoverableErrorMessage(job);
|
||||
if (unrecoverableErrorMessage) {
|
||||
const failed = await this.retryIfFailed(() => {
|
||||
this.lockManager.untrackJob(job.id);
|
||||
return this.handleFailed(new UnrecoverableError(unrecoverableErrorMessage), job, token, fetchNextCallback, span);
|
||||
}, { delayInMs: this.opts.runRetryDelay, span });
|
||||
return failed;
|
||||
}
|
||||
const result = await this.callProcessJob(job, token, abortController
|
||||
? abortController.signal
|
||||
: undefined);
|
||||
return await this.retryIfFailed(() => {
|
||||
this.lockManager.untrackJob(job.id);
|
||||
return this.handleCompleted(result, job, token, fetchNextCallback, span);
|
||||
}, { delayInMs: this.opts.runRetryDelay, span });
|
||||
}
|
||||
catch (err) {
|
||||
const failed = await this.retryIfFailed(() => {
|
||||
this.lockManager.untrackJob(job.id);
|
||||
return this.handleFailed(err, job, token, fetchNextCallback, span);
|
||||
}, { delayInMs: this.opts.runRetryDelay, span, onlyEmitError: true });
|
||||
return failed;
|
||||
}
|
||||
finally {
|
||||
this.lockManager.untrackJob(job.id);
|
||||
const now = Date.now();
|
||||
span === null || span === void 0 ? void 0 : span.setAttributes({
|
||||
[TelemetryAttributes.JobAttemptFinishedTimestamp]: job.finishedOn || now,
|
||||
[TelemetryAttributes.JobProcessedTimestamp]: job.processedOn,
|
||||
});
|
||||
}
|
||||
}, srcPropagationMetadata);
|
||||
}
|
||||
getUnrecoverableErrorMessage(job) {
|
||||
if (job.deferredFailure) {
|
||||
return job.deferredFailure;
|
||||
}
|
||||
if (this.opts.maxStartedAttempts &&
|
||||
this.opts.maxStartedAttempts < job.attemptsStarted) {
|
||||
return 'job started more than allowable limit';
|
||||
}
|
||||
}
|
||||
async handleCompleted(result, job, token, fetchNextCallback = () => true, span) {
|
||||
if (!this.backend.closing) {
|
||||
const completed = await job.moveToCompleted(result, token, fetchNextCallback() && !(this.closing || this.paused));
|
||||
this.emit('completed', job, result, 'active');
|
||||
span === null || span === void 0 ? void 0 : span.addEvent('job completed', {
|
||||
[TelemetryAttributes.JobResult]: JSON.stringify(result),
|
||||
});
|
||||
span === null || span === void 0 ? void 0 : span.setAttributes({
|
||||
[TelemetryAttributes.JobAttemptsMade]: job.attemptsMade,
|
||||
});
|
||||
if (Array.isArray(completed)) {
|
||||
const [jobData, jobId, rateLimitDelay, delayUntil] = completed;
|
||||
this.updateDelays(rateLimitDelay, delayUntil);
|
||||
return this.nextJobFromJobData(jobData, jobId, token);
|
||||
}
|
||||
}
|
||||
}
|
||||
async handleFailed(err, job, token, fetchNextCallback = () => true, span) {
|
||||
if (!this.backend.closing) {
|
||||
// Check if the job was manually rate-limited
|
||||
if (err.message === RATE_LIMIT_ERROR) {
|
||||
const rateLimitTtl = await this.moveLimitedBackToWait(job, token);
|
||||
this.limitUntil = rateLimitTtl > 0 ? Date.now() + rateLimitTtl : 0;
|
||||
return;
|
||||
}
|
||||
const fetchNext = fetchNextCallback() && !(this.closing || this.paused);
|
||||
if (err instanceof DelayedError ||
|
||||
err.name == 'DelayedError' ||
|
||||
err instanceof WaitingError ||
|
||||
err.name == 'WaitingError' ||
|
||||
err instanceof WaitingChildrenError ||
|
||||
err.name == 'WaitingChildrenError') {
|
||||
if (!fetchNext) {
|
||||
return;
|
||||
}
|
||||
return this.moveToActive(token, this.opts.name);
|
||||
}
|
||||
const result = await job.moveToFailed(err, token, fetchNext);
|
||||
this.emit('failed', job, err, 'active');
|
||||
span === null || span === void 0 ? void 0 : span.addEvent('job failed', {
|
||||
[TelemetryAttributes.JobFailedReason]: err.message,
|
||||
});
|
||||
span === null || span === void 0 ? void 0 : span.setAttributes({
|
||||
[TelemetryAttributes.JobAttemptsMade]: job.attemptsMade,
|
||||
});
|
||||
// Note: result can be undefined if moveToFailed fails (e.g., lock was lost)
|
||||
if (Array.isArray(result)) {
|
||||
const [jobData, jobId, rateLimitDelay, delayUntil] = result;
|
||||
this.updateDelays(rateLimitDelay, delayUntil);
|
||||
return this.nextJobFromJobData(jobData, jobId, token);
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
*
|
||||
* Pauses the processing of this queue only for this worker.
|
||||
*/
|
||||
async pause(doNotWaitActive) {
|
||||
await this.trace(SpanKind.INTERNAL, 'pause', this.name, async (span) => {
|
||||
var _a;
|
||||
span === null || span === void 0 ? void 0 : span.setAttributes({
|
||||
[TelemetryAttributes.WorkerId]: this.id,
|
||||
[TelemetryAttributes.WorkerName]: this.opts.name,
|
||||
[TelemetryAttributes.WorkerDoNotWaitActive]: doNotWaitActive,
|
||||
});
|
||||
if (!this.paused) {
|
||||
this.paused = true;
|
||||
if (!doNotWaitActive) {
|
||||
await this.whenCurrentJobsFinished();
|
||||
}
|
||||
(_a = this.stalledCheckStopper) === null || _a === void 0 ? void 0 : _a.call(this);
|
||||
this.emit('paused');
|
||||
}
|
||||
});
|
||||
}
|
||||
/**
|
||||
*
|
||||
* Resumes processing of this worker (if paused).
|
||||
*/
|
||||
async resume() {
|
||||
try {
|
||||
if (!this.running || this.paused) {
|
||||
await this.trace(SpanKind.INTERNAL, 'resume', this.name, async (span) => {
|
||||
span === null || span === void 0 ? void 0 : span.setAttributes({
|
||||
[TelemetryAttributes.WorkerId]: this.id,
|
||||
[TelemetryAttributes.WorkerName]: this.opts.name,
|
||||
});
|
||||
this.paused = false;
|
||||
if (!this.running) {
|
||||
if (this.processFn) {
|
||||
this.run();
|
||||
}
|
||||
}
|
||||
else {
|
||||
// Main loop is still running (pause was called with doNotWaitActive=true).
|
||||
// Restart the stalled checker since pause() stopped it.
|
||||
await this.startStalledCheckTimer();
|
||||
}
|
||||
this.emit('resumed');
|
||||
});
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
this.emit('error', error);
|
||||
}
|
||||
}
|
||||
/**
|
||||
*
|
||||
* Checks if worker is paused.
|
||||
*
|
||||
* @returns true if worker is paused, false otherwise.
|
||||
*/
|
||||
isPaused() {
|
||||
return !!this.paused;
|
||||
}
|
||||
/**
|
||||
*
|
||||
* Checks if worker is currently running.
|
||||
*
|
||||
* @returns true if worker is running, false otherwise.
|
||||
*/
|
||||
isRunning() {
|
||||
return this.running;
|
||||
}
|
||||
/**
|
||||
*
|
||||
* Closes the worker and related redis connections.
|
||||
*
|
||||
* This method waits for current jobs to finalize before returning.
|
||||
*
|
||||
* @param force - Use force boolean parameter if you do not want to wait for
|
||||
* current jobs to be processed. When using telemetry, be mindful that it can
|
||||
* interfere with the proper closure of spans, potentially preventing them from being exported.
|
||||
*
|
||||
* @returns Promise that resolves when the worker has been closed.
|
||||
*/
|
||||
async close(force = false) {
|
||||
if (this.closing) {
|
||||
return this.closing;
|
||||
}
|
||||
this.closing = (async () => {
|
||||
await this.trace(SpanKind.INTERNAL, 'close', this.name, async (span) => {
|
||||
var _a, _b;
|
||||
span === null || span === void 0 ? void 0 : span.setAttributes({
|
||||
[TelemetryAttributes.WorkerId]: this.id,
|
||||
[TelemetryAttributes.WorkerName]: this.opts.name,
|
||||
[TelemetryAttributes.WorkerForceClose]: force,
|
||||
});
|
||||
this.emit('closing', 'closing queue');
|
||||
(_a = this.abortDelayController) === null || _a === void 0 ? void 0 : _a.abort();
|
||||
// Define the async cleanup functions
|
||||
const asyncCleanups = [
|
||||
() => {
|
||||
return force || this.whenCurrentJobsFinished(false);
|
||||
},
|
||||
() => this.lockManager.close(),
|
||||
() => { var _a; return (_a = this.childPool) === null || _a === void 0 ? void 0 : _a.clean(); },
|
||||
() => this.backend.close(force),
|
||||
];
|
||||
// Run cleanup functions sequentially and make sure all are run despite any errors
|
||||
for (const cleanup of asyncCleanups) {
|
||||
try {
|
||||
await cleanup();
|
||||
}
|
||||
catch (err) {
|
||||
this.emit('error', err);
|
||||
}
|
||||
}
|
||||
(_b = this.stalledCheckStopper) === null || _b === void 0 ? void 0 : _b.call(this);
|
||||
this.closed = true;
|
||||
this.emit('closed');
|
||||
});
|
||||
})();
|
||||
return await this.closing;
|
||||
}
|
||||
/**
|
||||
*
|
||||
* Manually starts the stalled checker.
|
||||
* The check will run once as soon as this method is called, and
|
||||
* then every opts.stalledInterval milliseconds until the worker is closed.
|
||||
* Note: Normally you do not need to call this method, since the stalled checker
|
||||
* is automatically started when the worker starts processing jobs after
|
||||
* calling run. However if you want to process the jobs manually you need
|
||||
* to call this method to start the stalled checker.
|
||||
*
|
||||
* @see {@link https://docs.bullmq.io/patterns/manually-fetching-jobs}
|
||||
*/
|
||||
async startStalledCheckTimer() {
|
||||
if (!this.opts.skipStalledCheck) {
|
||||
if (!this.closing && !this.stalledCheckerRunning) {
|
||||
await this.trace(SpanKind.INTERNAL, 'startStalledCheckTimer', this.name, async (span) => {
|
||||
span === null || span === void 0 ? void 0 : span.setAttributes({
|
||||
[TelemetryAttributes.WorkerId]: this.id,
|
||||
[TelemetryAttributes.WorkerName]: this.opts.name,
|
||||
});
|
||||
this.stalledCheckerRunning = true;
|
||||
this.stalledChecker()
|
||||
.catch(err => {
|
||||
this.emit('error', err);
|
||||
})
|
||||
.finally(() => {
|
||||
this.stalledCheckerRunning = false;
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
async stalledChecker() {
|
||||
while (!(this.closing || this.paused)) {
|
||||
await this.checkConnectionError(() => this.moveStalledJobsToWait());
|
||||
await new Promise(resolve => {
|
||||
const timeout = setTimeout(resolve, this.opts.stalledInterval);
|
||||
this.stalledCheckStopper = () => {
|
||||
clearTimeout(timeout);
|
||||
resolve();
|
||||
};
|
||||
});
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Returns a promise that resolves when active jobs are cleared
|
||||
*
|
||||
* @returns
|
||||
*/
|
||||
async whenCurrentJobsFinished(reconnect = true) {
|
||||
// The blocking connection is dedicated to bzpopmin, so it is safe to
|
||||
// always disconnect it whenever the main loop is running. Waiting for the
|
||||
// actual disconnect ('end' event) is required to avoid a race where the
|
||||
// bzpopmin call is still in flight when the main loop awaits its result.
|
||||
if (this.mainLoopRunning) {
|
||||
await this.backend.disconnectBlocking(true);
|
||||
await this.mainLoopRunning;
|
||||
}
|
||||
else {
|
||||
reconnect = false;
|
||||
}
|
||||
reconnect && (await this.backend.reconnectBlocking());
|
||||
}
|
||||
async retryIfFailed(fn, opts) {
|
||||
var _a;
|
||||
let retry = 0;
|
||||
const maxRetries = opts.maxRetries || Infinity;
|
||||
do {
|
||||
try {
|
||||
return await fn();
|
||||
}
|
||||
catch (err) {
|
||||
(_a = opts.span) === null || _a === void 0 ? void 0 : _a.recordException(err.message);
|
||||
if (isNotConnectionError(err)) {
|
||||
// Emit error when not paused or closing; optionally swallow (no throw) when opts.onlyEmitError is set.
|
||||
if (!this.paused && !this.closing) {
|
||||
this.emit('error', err);
|
||||
}
|
||||
if (opts.onlyEmitError) {
|
||||
return;
|
||||
}
|
||||
else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (opts.delayInMs && !this.closing && !this.closed) {
|
||||
await this.delay(opts.delayInMs, this.abortDelayController);
|
||||
}
|
||||
if (retry + 1 >= maxRetries) {
|
||||
// If we've reached max retries, throw the last error
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
} while (++retry < maxRetries);
|
||||
}
|
||||
async moveStalledJobsToWait() {
|
||||
await this.trace(SpanKind.INTERNAL, 'moveStalledJobsToWait', this.name, async (span) => {
|
||||
const stalled = await this.backend.moveStalledJobsToWait();
|
||||
span === null || span === void 0 ? void 0 : span.setAttributes({
|
||||
[TelemetryAttributes.WorkerId]: this.id,
|
||||
[TelemetryAttributes.WorkerName]: this.opts.name,
|
||||
[TelemetryAttributes.WorkerStalledJobs]: stalled,
|
||||
});
|
||||
stalled.forEach((jobId) => {
|
||||
span === null || span === void 0 ? void 0 : span.addEvent('job stalled', {
|
||||
[TelemetryAttributes.JobId]: jobId,
|
||||
});
|
||||
this.emit('stalled', jobId, 'active');
|
||||
});
|
||||
});
|
||||
}
|
||||
moveLimitedBackToWait(job, token) {
|
||||
return job.moveToWait(token);
|
||||
}
|
||||
}
|
||||
108
node_modules/bullmq/dist/esm/commands/addDelayedJob-6.lua
generated
vendored
Normal file
108
node_modules/bullmq/dist/esm/commands/addDelayedJob-6.lua
generated
vendored
Normal file
@@ -0,0 +1,108 @@
|
||||
--[[
|
||||
Adds a delayed job to the queue by doing the following:
|
||||
- Increases the job counter if needed.
|
||||
- Creates a new job key with the job data.
|
||||
|
||||
- computes timestamp.
|
||||
- adds to delayed zset.
|
||||
- Emits a global event 'delayed' if the job is delayed.
|
||||
|
||||
Input:
|
||||
KEYS[1] 'marker',
|
||||
KEYS[2] 'meta'
|
||||
KEYS[3] 'id'
|
||||
KEYS[4] 'delayed'
|
||||
KEYS[5] 'completed'
|
||||
KEYS[6] events stream key
|
||||
|
||||
ARGV[1] msgpacked arguments array
|
||||
[1] key prefix,
|
||||
[2] custom id (use custom instead of one generated automatically)
|
||||
[3] name
|
||||
[4] timestamp
|
||||
[5] parentKey?
|
||||
[6] parent dependencies key.
|
||||
[7] parent? {id, queueKey}
|
||||
[8] repeat job key
|
||||
[9] deduplication key
|
||||
|
||||
ARGV[2] Json stringified job data
|
||||
ARGV[3] msgpacked options
|
||||
|
||||
Output:
|
||||
jobId - OK
|
||||
-5 - Missing parent key
|
||||
]]
|
||||
local metaKey = KEYS[2]
|
||||
local idKey = KEYS[3]
|
||||
local delayedKey = KEYS[4]
|
||||
|
||||
local completedKey = KEYS[5]
|
||||
local eventsKey = KEYS[6]
|
||||
|
||||
local jobId
|
||||
local jobIdKey
|
||||
local rcall = redis.call
|
||||
|
||||
local args = cmsgpack.unpack(ARGV[1])
|
||||
|
||||
local data = ARGV[2]
|
||||
|
||||
local parentKey = args[5]
|
||||
local parent = args[7]
|
||||
local repeatJobKey = args[8]
|
||||
local deduplicationKey = args[9]
|
||||
local parentData
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/addDelayedJob"
|
||||
--- @include "includes/deduplicateJob"
|
||||
--- @include "includes/getOrSetMaxEvents"
|
||||
--- @include "includes/handleDuplicatedJob"
|
||||
--- @include "includes/storeJob"
|
||||
|
||||
if parentKey ~= nil then
|
||||
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
|
||||
|
||||
parentData = cjson.encode(parent)
|
||||
end
|
||||
|
||||
local jobCounter = rcall("INCR", idKey)
|
||||
|
||||
local maxEvents = getOrSetMaxEvents(metaKey)
|
||||
local opts = cmsgpack.unpack(ARGV[3])
|
||||
|
||||
local parentDependenciesKey = args[6]
|
||||
local timestamp = args[4]
|
||||
|
||||
if args[2] == "" then
|
||||
jobId = jobCounter .. "" -- convert to string
|
||||
jobIdKey = args[1] .. jobId
|
||||
else
|
||||
jobId = args[2]
|
||||
jobIdKey = args[1] .. jobId
|
||||
if rcall("EXISTS", jobIdKey) == 1 then
|
||||
return handleDuplicatedJob(jobIdKey, jobId, parentKey, parent,
|
||||
parentData, parentDependenciesKey, completedKey, eventsKey,
|
||||
maxEvents, timestamp)
|
||||
end
|
||||
end
|
||||
|
||||
local deduplicationJobId = deduplicateJob(opts['de'], jobId, delayedKey, deduplicationKey,
|
||||
eventsKey, maxEvents, args[1], args[3], ARGV[2], opts,
|
||||
parentKey, parentData, parentDependenciesKey, repeatJobKey)
|
||||
if deduplicationJobId then
|
||||
return deduplicationJobId
|
||||
end
|
||||
|
||||
local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
|
||||
opts, timestamp, parentKey, parentData, repeatJobKey)
|
||||
|
||||
addDelayedJob(jobId, delayedKey, eventsKey, timestamp, maxEvents, KEYS[1], delay)
|
||||
|
||||
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
||||
if parentDependenciesKey ~= nil then
|
||||
rcall("SADD", parentDependenciesKey, jobIdKey)
|
||||
end
|
||||
|
||||
return jobId
|
||||
202
node_modules/bullmq/dist/esm/commands/addJobScheduler-11.lua
generated
vendored
Normal file
202
node_modules/bullmq/dist/esm/commands/addJobScheduler-11.lua
generated
vendored
Normal file
@@ -0,0 +1,202 @@
|
||||
--[[
|
||||
Adds a job scheduler, i.e. a job factory that creates jobs based on a given schedule (repeat options).
|
||||
|
||||
Input:
|
||||
KEYS[1] 'repeat' key
|
||||
KEYS[2] 'delayed' key
|
||||
KEYS[3] 'wait' key
|
||||
KEYS[4] 'paused' key
|
||||
KEYS[5] 'meta' key
|
||||
KEYS[6] 'prioritized' key
|
||||
KEYS[7] 'marker' key
|
||||
KEYS[8] 'id' key
|
||||
KEYS[9] 'events' key
|
||||
KEYS[10] 'pc' priority counter
|
||||
KEYS[11] 'active' key
|
||||
|
||||
ARGV[1] next milliseconds
|
||||
ARGV[2] msgpacked options
|
||||
[1] name
|
||||
[2] tz?
|
||||
[3] pattern?
|
||||
[4] endDate?
|
||||
[5] every?
|
||||
ARGV[3] jobs scheduler id
|
||||
ARGV[4] Json stringified template data
|
||||
ARGV[5] mspacked template opts
|
||||
ARGV[6] msgpacked delayed opts
|
||||
ARGV[7] timestamp
|
||||
ARGV[8] prefix key
|
||||
ARGV[9] producer key
|
||||
|
||||
Output:
|
||||
repeatableKey - OK
|
||||
]] local rcall = redis.call
|
||||
local repeatKey = KEYS[1]
|
||||
local delayedKey = KEYS[2]
|
||||
local waitKey = KEYS[3]
|
||||
local pausedKey = KEYS[4]
|
||||
local metaKey = KEYS[5]
|
||||
local prioritizedKey = KEYS[6]
|
||||
local eventsKey = KEYS[9]
|
||||
|
||||
local nextMillis = ARGV[1]
|
||||
local jobSchedulerId = ARGV[3]
|
||||
local templateOpts = cmsgpack.unpack(ARGV[5])
|
||||
local now = tonumber(ARGV[7])
|
||||
local prefixKey = ARGV[8]
|
||||
local jobOpts = cmsgpack.unpack(ARGV[6])
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/addJobFromScheduler"
|
||||
--- @include "includes/getOrSetMaxEvents"
|
||||
--- @include "includes/isQueuePaused"
|
||||
--- @include "includes/removeJob"
|
||||
--- @include "includes/storeJobScheduler"
|
||||
--- @include "includes/getJobSchedulerEveryNextMillis"
|
||||
|
||||
-- If we are overriding a repeatable job we must delete the delayed job for
|
||||
-- the next iteration.
|
||||
local schedulerKey = repeatKey .. ":" .. jobSchedulerId
|
||||
local maxEvents = getOrSetMaxEvents(metaKey)
|
||||
|
||||
local templateData = ARGV[4]
|
||||
|
||||
local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
|
||||
if prevMillis then
|
||||
prevMillis = tonumber(prevMillis)
|
||||
end
|
||||
local schedulerOpts = cmsgpack.unpack(ARGV[2])
|
||||
|
||||
local every = schedulerOpts['every']
|
||||
|
||||
-- For backwards compatibility we also check the offset from the job itself.
|
||||
-- could be removed in future major versions.
|
||||
local jobOffset = jobOpts['repeat'] and jobOpts['repeat']['offset'] or 0
|
||||
local offset = schedulerOpts['offset'] or jobOffset or 0
|
||||
local newOffset = offset
|
||||
|
||||
local updatedEvery = false
|
||||
if every then
|
||||
-- if we changed the 'every' value we need to reset millis to nil
|
||||
local millis = prevMillis
|
||||
if prevMillis then
|
||||
local prevEvery = tonumber(rcall("HGET", schedulerKey, "every"))
|
||||
if prevEvery ~= every then
|
||||
millis = nil
|
||||
updatedEvery = true
|
||||
end
|
||||
end
|
||||
|
||||
local startDate = schedulerOpts['startDate']
|
||||
nextMillis, newOffset = getJobSchedulerEveryNextMillis(millis, every, now, offset, startDate)
|
||||
end
|
||||
|
||||
local function removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, waitKey, pausedKey, jobId, metaKey,
|
||||
eventsKey)
|
||||
if rcall("ZSCORE", delayedKey, jobId) then
|
||||
removeJob(jobId, true, prefixKey, true --[[remove debounce key]] )
|
||||
rcall("ZREM", delayedKey, jobId)
|
||||
return true
|
||||
elseif rcall("ZSCORE", prioritizedKey, jobId) then
|
||||
removeJob(jobId, true, prefixKey, true --[[remove debounce key]] )
|
||||
rcall("ZREM", prioritizedKey, jobId)
|
||||
return true
|
||||
else
|
||||
local pausedOrWaitKey = waitKey
|
||||
if isQueuePaused(metaKey) then
|
||||
pausedOrWaitKey = pausedKey
|
||||
end
|
||||
|
||||
if rcall("LREM", pausedOrWaitKey, 1, jobId) > 0 then
|
||||
removeJob(jobId, true, prefixKey, true --[[remove debounce key]] )
|
||||
return true
|
||||
end
|
||||
end
|
||||
|
||||
return false
|
||||
end
|
||||
|
||||
local removedPrevJob = false
|
||||
if prevMillis then
|
||||
local currentJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
|
||||
local currentJobKey = schedulerKey .. ":" .. prevMillis
|
||||
|
||||
-- In theory it should always exist the currentJobKey if there is a prevMillis unless something has
|
||||
-- gone really wrong.
|
||||
if rcall("EXISTS", currentJobKey) == 1 then
|
||||
removedPrevJob = removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, waitKey, pausedKey, currentJobId,
|
||||
metaKey, eventsKey)
|
||||
end
|
||||
end
|
||||
|
||||
if removedPrevJob then
|
||||
-- The jobs has been removed and we want to replace it, so lets use the same millis.
|
||||
if every and not updatedEvery then
|
||||
nextMillis = prevMillis
|
||||
end
|
||||
else
|
||||
-- Special case where no job was removed, and we need to add the next iteration.
|
||||
schedulerOpts['offset'] = newOffset
|
||||
end
|
||||
|
||||
-- Check for job ID collision with existing jobs (in any state)
|
||||
local jobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
|
||||
local jobKey = prefixKey .. jobId
|
||||
|
||||
-- If there's already a job with this ID, in a state
|
||||
-- that is not updatable (active, completed, failed) we must
|
||||
-- handle the collision
|
||||
local hasCollision = false
|
||||
if rcall("EXISTS", jobKey) == 1 then
|
||||
if every then
|
||||
-- For 'every' case: walk forward through subsequent slots
|
||||
-- until we find a free one. Stale completed/failed jobs from
|
||||
-- a previous scheduler under the same id can occupy several
|
||||
-- consecutive slots (issue #3063), so a single retry is not
|
||||
-- enough. The scan is bounded so we don't spin if the
|
||||
-- scheduler is genuinely contested.
|
||||
local maxSlotScans = 32
|
||||
local slotsScanned = 0
|
||||
local jobExists
|
||||
repeat
|
||||
nextMillis = nextMillis + every
|
||||
jobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
|
||||
jobKey = prefixKey .. jobId
|
||||
slotsScanned = slotsScanned + 1
|
||||
jobExists = rcall("EXISTS", jobKey)
|
||||
until jobExists == 0 or slotsScanned >= maxSlotScans
|
||||
|
||||
if jobExists == 1 then
|
||||
-- Every scanned slot still has a job, return error code
|
||||
return -11 -- SchedulerJobSlotsBusy
|
||||
end
|
||||
else
|
||||
hasCollision = true
|
||||
end
|
||||
end
|
||||
|
||||
local delay = nextMillis - now
|
||||
|
||||
-- Fast Clamp delay to minimum of 0
|
||||
if delay < 0 then
|
||||
delay = 0
|
||||
end
|
||||
|
||||
local nextJobKey = schedulerKey .. ":" .. nextMillis
|
||||
|
||||
if not hasCollision or removedPrevJob then
|
||||
-- jobId already calculated above during collision check
|
||||
|
||||
storeJobScheduler(jobSchedulerId, schedulerKey, repeatKey, nextMillis, schedulerOpts, templateData, templateOpts)
|
||||
|
||||
rcall("INCR", KEYS[8])
|
||||
|
||||
addJobFromScheduler(nextJobKey, jobId, jobOpts, waitKey, pausedKey, KEYS[11], metaKey, prioritizedKey, KEYS[10],
|
||||
delayedKey, KEYS[7], eventsKey, schedulerOpts['name'], maxEvents, now, templateData, jobSchedulerId, delay)
|
||||
elseif hasCollision then
|
||||
-- For 'pattern' case: return error code
|
||||
return -10 -- SchedulerJobIdCollision
|
||||
end
|
||||
|
||||
return {jobId .. "", delay}
|
||||
30
node_modules/bullmq/dist/esm/commands/addLog-2.lua
generated
vendored
Normal file
30
node_modules/bullmq/dist/esm/commands/addLog-2.lua
generated
vendored
Normal file
@@ -0,0 +1,30 @@
|
||||
--[[
|
||||
Add job log
|
||||
|
||||
Input:
|
||||
KEYS[1] job id key
|
||||
KEYS[2] job logs key
|
||||
|
||||
ARGV[1] id
|
||||
ARGV[2] log
|
||||
ARGV[3] keepLogs
|
||||
|
||||
Output:
|
||||
-1 - Missing job.
|
||||
]]
|
||||
local rcall = redis.call
|
||||
|
||||
if rcall("EXISTS", KEYS[1]) == 1 then -- // Make sure job exists
|
||||
local logCount = rcall("RPUSH", KEYS[2], ARGV[2])
|
||||
|
||||
if ARGV[3] ~= '' then
|
||||
local keepLogs = tonumber(ARGV[3])
|
||||
rcall("LTRIM", KEYS[2], -keepLogs, -1)
|
||||
|
||||
return math.min(keepLogs, logCount)
|
||||
end
|
||||
|
||||
return logCount
|
||||
else
|
||||
return -1
|
||||
end
|
||||
110
node_modules/bullmq/dist/esm/commands/addParentJob-6.lua
generated
vendored
Normal file
110
node_modules/bullmq/dist/esm/commands/addParentJob-6.lua
generated
vendored
Normal file
@@ -0,0 +1,110 @@
|
||||
--[[
|
||||
Adds a parent job to the queue by doing the following:
|
||||
- Increases the job counter if needed.
|
||||
- Creates a new job key with the job data.
|
||||
- adds the job to the waiting-children zset
|
||||
|
||||
Input:
|
||||
KEYS[1] 'meta'
|
||||
KEYS[2] 'id'
|
||||
KEYS[3] 'delayed'
|
||||
KEYS[4] 'waiting-children'
|
||||
KEYS[5] 'completed'
|
||||
KEYS[6] events stream key
|
||||
|
||||
ARGV[1] msgpacked arguments array
|
||||
[1] key prefix,
|
||||
[2] custom id (will not generate one automatically)
|
||||
[3] name
|
||||
[4] timestamp
|
||||
[5] parentKey?
|
||||
[6] parent dependencies key.
|
||||
[7] parent? {id, queueKey}
|
||||
[8] repeat job key
|
||||
[9] deduplication key
|
||||
|
||||
ARGV[2] Json stringified job data
|
||||
ARGV[3] msgpacked options
|
||||
|
||||
Output:
|
||||
jobId - OK
|
||||
-5 - Missing parent key
|
||||
]]
|
||||
local metaKey = KEYS[1]
|
||||
local idKey = KEYS[2]
|
||||
local delayedKey = KEYS[3]
|
||||
|
||||
local completedKey = KEYS[5]
|
||||
local eventsKey = KEYS[6]
|
||||
|
||||
local jobId
|
||||
local jobIdKey
|
||||
local rcall = redis.call
|
||||
|
||||
local args = cmsgpack.unpack(ARGV[1])
|
||||
|
||||
local data = ARGV[2]
|
||||
local opts = cmsgpack.unpack(ARGV[3])
|
||||
|
||||
local parentKey = args[5]
|
||||
local parent = args[7]
|
||||
local repeatJobKey = args[8]
|
||||
local deduplicationKey = args[9]
|
||||
local parentData
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/deduplicateJobWithoutReplace"
|
||||
--- @include "includes/getOrSetMaxEvents"
|
||||
--- @include "includes/handleDuplicatedJob"
|
||||
--- @include "includes/storeJob"
|
||||
|
||||
if parentKey ~= nil then
|
||||
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
|
||||
|
||||
parentData = cjson.encode(parent)
|
||||
end
|
||||
|
||||
local jobCounter = rcall("INCR", idKey)
|
||||
|
||||
local maxEvents = getOrSetMaxEvents(metaKey)
|
||||
|
||||
local parentDependenciesKey = args[6]
|
||||
local timestamp = args[4]
|
||||
if args[2] == "" then
|
||||
jobId = jobCounter .. "" -- convert to string
|
||||
jobIdKey = args[1] .. jobId
|
||||
else
|
||||
jobId = args[2]
|
||||
jobIdKey = args[1] .. jobId
|
||||
if rcall("EXISTS", jobIdKey) == 1 then
|
||||
return handleDuplicatedJob(jobIdKey, jobId, parentKey, parent,
|
||||
parentData, parentDependenciesKey, completedKey, eventsKey,
|
||||
maxEvents, timestamp)
|
||||
end
|
||||
end
|
||||
|
||||
local deduplicationId = opts['de'] and opts['de']['id']
|
||||
if deduplicationId then
|
||||
local deduplicationJobId = deduplicateJobWithoutReplace(deduplicationId, opts['de'],
|
||||
jobId, deduplicationKey, eventsKey, maxEvents, args[1], args[3], ARGV[2], opts,
|
||||
parentKey, parentData, parentDependenciesKey, repeatJobKey)
|
||||
if deduplicationJobId then
|
||||
return deduplicationJobId
|
||||
end
|
||||
end
|
||||
|
||||
-- Store the job.
|
||||
storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2], opts, timestamp,
|
||||
parentKey, parentData, repeatJobKey)
|
||||
|
||||
local waitChildrenKey = KEYS[4]
|
||||
rcall("ZADD", waitChildrenKey, timestamp, jobId)
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
||||
"waiting-children", "jobId", jobId)
|
||||
|
||||
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
||||
if parentDependenciesKey ~= nil then
|
||||
rcall("SADD", parentDependenciesKey, jobIdKey)
|
||||
end
|
||||
|
||||
return jobId
|
||||
118
node_modules/bullmq/dist/esm/commands/addPrioritizedJob-9.lua
generated
vendored
Normal file
118
node_modules/bullmq/dist/esm/commands/addPrioritizedJob-9.lua
generated
vendored
Normal file
@@ -0,0 +1,118 @@
|
||||
--[[
|
||||
Adds a prioritized job to the queue by doing the following:
|
||||
- Increases the job counter if needed.
|
||||
- Creates a new job key with the job data.
|
||||
- Adds the job to the "added" list so that workers gets notified.
|
||||
|
||||
Input:
|
||||
KEYS[1] 'marker',
|
||||
KEYS[2] 'meta'
|
||||
KEYS[3] 'id'
|
||||
KEYS[4] 'prioritized'
|
||||
KEYS[5] 'delayed'
|
||||
KEYS[6] 'completed'
|
||||
KEYS[7] 'active'
|
||||
KEYS[8] events stream key
|
||||
KEYS[9] 'pc' priority counter
|
||||
|
||||
ARGV[1] msgpacked arguments array
|
||||
[1] key prefix,
|
||||
[2] custom id (will not generate one automatically)
|
||||
[3] name
|
||||
[4] timestamp
|
||||
[5] parentKey?
|
||||
[6] parent dependencies key.
|
||||
[7] parent? {id, queueKey}
|
||||
[8] repeat job key
|
||||
[9] deduplication key
|
||||
|
||||
ARGV[2] Json stringified job data
|
||||
ARGV[3] msgpacked options
|
||||
|
||||
Output:
|
||||
jobId - OK
|
||||
-5 - Missing parent key
|
||||
]]
|
||||
local metaKey = KEYS[2]
|
||||
local idKey = KEYS[3]
|
||||
local priorityKey = KEYS[4]
|
||||
|
||||
local completedKey = KEYS[6]
|
||||
local activeKey = KEYS[7]
|
||||
local eventsKey = KEYS[8]
|
||||
local priorityCounterKey = KEYS[9]
|
||||
|
||||
local jobId
|
||||
local jobIdKey
|
||||
local rcall = redis.call
|
||||
|
||||
local args = cmsgpack.unpack(ARGV[1])
|
||||
|
||||
local data = ARGV[2]
|
||||
local opts = cmsgpack.unpack(ARGV[3])
|
||||
|
||||
local parentKey = args[5]
|
||||
local parent = args[7]
|
||||
local repeatJobKey = args[8]
|
||||
local deduplicationKey = args[9]
|
||||
local parentData
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/addJobWithPriority"
|
||||
--- @include "includes/deduplicateJob"
|
||||
--- @include "includes/storeJob"
|
||||
--- @include "includes/getOrSetMaxEvents"
|
||||
--- @include "includes/handleDuplicatedJob"
|
||||
--- @include "includes/isQueuePausedOrMaxed"
|
||||
|
||||
if parentKey ~= nil then
|
||||
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
|
||||
|
||||
parentData = cjson.encode(parent)
|
||||
end
|
||||
|
||||
local jobCounter = rcall("INCR", idKey)
|
||||
|
||||
local maxEvents = getOrSetMaxEvents(metaKey)
|
||||
|
||||
local parentDependenciesKey = args[6]
|
||||
local timestamp = args[4]
|
||||
if args[2] == "" then
|
||||
jobId = jobCounter .. "" -- convert to string
|
||||
jobIdKey = args[1] .. jobId
|
||||
else
|
||||
jobId = args[2]
|
||||
jobIdKey = args[1] .. jobId
|
||||
if rcall("EXISTS", jobIdKey) == 1 then
|
||||
return handleDuplicatedJob(jobIdKey, jobId, parentKey, parent,
|
||||
parentData, parentDependenciesKey, completedKey, eventsKey,
|
||||
maxEvents, timestamp)
|
||||
end
|
||||
end
|
||||
|
||||
local deduplicationJobId = deduplicateJob(opts['de'], jobId, KEYS[5],
|
||||
deduplicationKey, eventsKey, maxEvents, args[1], args[3], ARGV[2], opts,
|
||||
parentKey, parentData, parentDependenciesKey, repeatJobKey)
|
||||
if deduplicationJobId then
|
||||
return deduplicationJobId
|
||||
end
|
||||
|
||||
-- Store the job.
|
||||
local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
|
||||
opts, timestamp, parentKey, parentData,
|
||||
repeatJobKey)
|
||||
|
||||
-- Add the job to the prioritized set
|
||||
local isPausedOrMaxed = isQueuePausedOrMaxed(metaKey, activeKey)
|
||||
addJobWithPriority( KEYS[1], priorityKey, priority, jobId, priorityCounterKey, isPausedOrMaxed)
|
||||
|
||||
-- Emit waiting event
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting",
|
||||
"jobId", jobId)
|
||||
|
||||
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
||||
if parentDependenciesKey ~= nil then
|
||||
rcall("SADD", parentDependenciesKey, jobIdKey)
|
||||
end
|
||||
|
||||
return jobId
|
||||
125
node_modules/bullmq/dist/esm/commands/addStandardJob-9.lua
generated
vendored
Normal file
125
node_modules/bullmq/dist/esm/commands/addStandardJob-9.lua
generated
vendored
Normal file
@@ -0,0 +1,125 @@
|
||||
--[[
|
||||
Adds a job to the queue by doing the following:
|
||||
- Increases the job counter if needed.
|
||||
- Creates a new job key with the job data.
|
||||
|
||||
- if delayed:
|
||||
- computes timestamp.
|
||||
- adds to delayed zset.
|
||||
- Emits a global event 'delayed' if the job is delayed.
|
||||
- if not delayed
|
||||
- Adds the jobId to the wait/paused list in one of three ways:
|
||||
- LIFO
|
||||
- FIFO
|
||||
- prioritized.
|
||||
- Adds the job to the "added" list so that workers gets notified.
|
||||
|
||||
Input:
|
||||
KEYS[1] 'wait',
|
||||
KEYS[2] 'paused'
|
||||
KEYS[3] 'meta'
|
||||
KEYS[4] 'id'
|
||||
KEYS[5] 'completed'
|
||||
KEYS[6] 'delayed'
|
||||
KEYS[7] 'active'
|
||||
KEYS[8] events stream key
|
||||
KEYS[9] marker key
|
||||
|
||||
ARGV[1] msgpacked arguments array
|
||||
[1] key prefix,
|
||||
[2] custom id (will not generate one automatically)
|
||||
[3] name
|
||||
[4] timestamp
|
||||
[5] parentKey?
|
||||
[6] parent dependencies key.
|
||||
[7] parent? {id, queueKey}
|
||||
[8] repeat job key
|
||||
[9] deduplication key
|
||||
|
||||
ARGV[2] Json stringified job data
|
||||
ARGV[3] msgpacked options
|
||||
|
||||
Output:
|
||||
jobId - OK
|
||||
-5 - Missing parent key
|
||||
]]
|
||||
local waitKey = KEYS[1]
|
||||
local metaKey = KEYS[3]
|
||||
local activeKey = KEYS[7]
|
||||
local eventsKey = KEYS[8]
|
||||
|
||||
local jobId
|
||||
local jobIdKey
|
||||
local rcall = redis.call
|
||||
|
||||
local args = cmsgpack.unpack(ARGV[1])
|
||||
|
||||
local data = ARGV[2]
|
||||
local opts = cmsgpack.unpack(ARGV[3])
|
||||
|
||||
local parentKey = args[5]
|
||||
local parent = args[7]
|
||||
local repeatJobKey = args[8]
|
||||
local deduplicationKey = args[9]
|
||||
local parentData
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/addJobInTargetList"
|
||||
--- @include "includes/deduplicateJob"
|
||||
--- @include "includes/getOrSetMaxEvents"
|
||||
--- @include "includes/handleDuplicatedJob"
|
||||
--- @include "includes/isQueuePausedOrMaxed"
|
||||
--- @include "includes/storeJob"
|
||||
|
||||
if parentKey ~= nil then
|
||||
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
|
||||
|
||||
parentData = cjson.encode(parent)
|
||||
end
|
||||
|
||||
local jobCounter = rcall("INCR", KEYS[4])
|
||||
|
||||
local maxEvents = getOrSetMaxEvents(metaKey)
|
||||
|
||||
local parentDependenciesKey = args[6]
|
||||
local timestamp = args[4]
|
||||
if args[2] == "" then
|
||||
jobId = jobCounter .. "" -- convert to string
|
||||
jobIdKey = args[1] .. jobId
|
||||
else
|
||||
jobId = args[2]
|
||||
jobIdKey = args[1] .. jobId
|
||||
if rcall("EXISTS", jobIdKey) == 1 then
|
||||
return handleDuplicatedJob(jobIdKey, jobId, parentKey, parent,
|
||||
parentData, parentDependenciesKey, KEYS[5], eventsKey,
|
||||
maxEvents, timestamp)
|
||||
end
|
||||
end
|
||||
|
||||
local deduplicationJobId = deduplicateJob(opts['de'], jobId, KEYS[6],
|
||||
deduplicationKey, eventsKey, maxEvents, args[1], args[3], ARGV[2], opts,
|
||||
parentKey, parentData, parentDependenciesKey, repeatJobKey)
|
||||
if deduplicationJobId then
|
||||
return deduplicationJobId
|
||||
end
|
||||
|
||||
-- Store the job.
|
||||
storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2], opts, timestamp,
|
||||
parentKey, parentData, repeatJobKey)
|
||||
|
||||
local isPausedOrMaxed = isQueuePausedOrMaxed(metaKey, activeKey)
|
||||
|
||||
-- LIFO or FIFO
|
||||
local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH'
|
||||
addJobInTargetList(waitKey, KEYS[9], pushCmd, isPausedOrMaxed, jobId)
|
||||
|
||||
-- Emit waiting event
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting",
|
||||
"jobId", jobId)
|
||||
|
||||
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
||||
if parentDependenciesKey ~= nil then
|
||||
rcall("SADD", parentDependenciesKey, jobIdKey)
|
||||
end
|
||||
|
||||
return jobId
|
||||
55
node_modules/bullmq/dist/esm/commands/changeDelay-4.lua
generated
vendored
Normal file
55
node_modules/bullmq/dist/esm/commands/changeDelay-4.lua
generated
vendored
Normal file
@@ -0,0 +1,55 @@
|
||||
--[[
|
||||
Change job delay when it is in delayed set.
|
||||
Input:
|
||||
KEYS[1] delayed key
|
||||
KEYS[2] meta key
|
||||
KEYS[3] marker key
|
||||
KEYS[4] events stream
|
||||
|
||||
ARGV[1] delay
|
||||
ARGV[2] timestamp
|
||||
ARGV[3] the id of the job
|
||||
ARGV[4] job key
|
||||
|
||||
Output:
|
||||
0 - OK
|
||||
-1 - Missing job.
|
||||
-3 - Job not in delayed set.
|
||||
|
||||
Events:
|
||||
- delayed key.
|
||||
]]
|
||||
local rcall = redis.call
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/addDelayMarkerIfNeeded"
|
||||
--- @include "includes/getDelayedScore"
|
||||
--- @include "includes/getOrSetMaxEvents"
|
||||
|
||||
if rcall("EXISTS", ARGV[4]) == 1 then
|
||||
local jobId = ARGV[3]
|
||||
|
||||
local delay = tonumber(ARGV[1])
|
||||
local score, delayedTimestamp = getDelayedScore(KEYS[1], ARGV[2], delay)
|
||||
|
||||
local numRemovedElements = rcall("ZREM", KEYS[1], jobId)
|
||||
|
||||
if numRemovedElements < 1 then
|
||||
return -3
|
||||
end
|
||||
|
||||
rcall("HSET", ARGV[4], "delay", delay)
|
||||
rcall("ZADD", KEYS[1], score, jobId)
|
||||
|
||||
local maxEvents = getOrSetMaxEvents(KEYS[2])
|
||||
|
||||
rcall("XADD", KEYS[4], "MAXLEN", "~", maxEvents, "*", "event", "delayed",
|
||||
"jobId", jobId, "delay", delayedTimestamp)
|
||||
|
||||
-- mark that a delayed job is available
|
||||
addDelayMarkerIfNeeded(KEYS[3], KEYS[1])
|
||||
|
||||
return 0
|
||||
else
|
||||
return -1
|
||||
end
|
||||
71
node_modules/bullmq/dist/esm/commands/changePriority-7.lua
generated
vendored
Normal file
71
node_modules/bullmq/dist/esm/commands/changePriority-7.lua
generated
vendored
Normal file
@@ -0,0 +1,71 @@
|
||||
--[[
|
||||
Change job priority
|
||||
Input:
|
||||
KEYS[1] 'wait',
|
||||
KEYS[2] 'paused'
|
||||
KEYS[3] 'meta'
|
||||
KEYS[4] 'prioritized'
|
||||
KEYS[5] 'active'
|
||||
KEYS[6] 'pc' priority counter
|
||||
KEYS[7] 'marker'
|
||||
|
||||
ARGV[1] priority value
|
||||
ARGV[2] prefix key
|
||||
ARGV[3] job id
|
||||
ARGV[4] lifo
|
||||
|
||||
Output:
|
||||
0 - OK
|
||||
-1 - Missing job
|
||||
]]
|
||||
local jobId = ARGV[3]
|
||||
local jobKey = ARGV[2] .. jobId
|
||||
local priority = tonumber(ARGV[1])
|
||||
local rcall = redis.call
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/addJobInTargetList"
|
||||
--- @include "includes/addJobWithPriority"
|
||||
--- @include "includes/isQueuePausedOrMaxed"
|
||||
--- @include "includes/pushBackJobWithPriority"
|
||||
|
||||
local function reAddJobWithNewPriority( prioritizedKey, markerKey, waitKey,
|
||||
priorityCounter, lifo, priority, jobId, isPausedOrMaxed)
|
||||
if priority == 0 then
|
||||
local pushCmd = lifo and 'RPUSH' or 'LPUSH'
|
||||
addJobInTargetList(waitKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
|
||||
else
|
||||
if lifo then
|
||||
pushBackJobWithPriority(prioritizedKey, priority, jobId)
|
||||
else
|
||||
addJobWithPriority(markerKey, prioritizedKey, priority, jobId,
|
||||
priorityCounter, isPausedOrMaxed)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
if rcall("EXISTS", jobKey) == 1 then
|
||||
local metaKey = KEYS[3]
|
||||
local activeKey = KEYS[5]
|
||||
local waitKey = KEYS[1]
|
||||
local pausedKey = KEYS[2]
|
||||
local isPausedOrMaxed = isQueuePausedOrMaxed(metaKey, activeKey)
|
||||
local prioritizedKey = KEYS[4]
|
||||
local priorityCounterKey = KEYS[6]
|
||||
local markerKey = KEYS[7]
|
||||
|
||||
-- Re-add with the new priority
|
||||
if rcall("ZREM", prioritizedKey, jobId) > 0 then
|
||||
reAddJobWithNewPriority( prioritizedKey, markerKey, waitKey,
|
||||
priorityCounterKey, ARGV[4] == '1', priority, jobId, isPausedOrMaxed)
|
||||
elseif rcall("LREM", waitKey, -1, jobId) > 0 then
|
||||
reAddJobWithNewPriority( prioritizedKey, markerKey, waitKey,
|
||||
priorityCounterKey, ARGV[4] == '1', priority, jobId, isPausedOrMaxed)
|
||||
end
|
||||
|
||||
rcall("HSET", jobKey, "priority", priority)
|
||||
|
||||
return 0
|
||||
else
|
||||
return -1
|
||||
end
|
||||
59
node_modules/bullmq/dist/esm/commands/cleanJobsInSet-3.lua
generated
vendored
Normal file
59
node_modules/bullmq/dist/esm/commands/cleanJobsInSet-3.lua
generated
vendored
Normal file
@@ -0,0 +1,59 @@
|
||||
--[[
|
||||
Remove jobs from the specific set.
|
||||
|
||||
Input:
|
||||
KEYS[1] set key,
|
||||
KEYS[2] events stream key
|
||||
KEYS[3] repeat key
|
||||
|
||||
ARGV[1] jobKey prefix
|
||||
ARGV[2] timestamp
|
||||
ARGV[3] limit the number of jobs to be removed. 0 is unlimited
|
||||
ARGV[4] set name, can be any of 'wait', 'active', 'paused', 'delayed', 'completed', or 'failed'
|
||||
]]
|
||||
local rcall = redis.call
|
||||
local repeatKey = KEYS[3]
|
||||
local rangeStart = 0
|
||||
local rangeEnd = -1
|
||||
|
||||
local limit = tonumber(ARGV[3])
|
||||
|
||||
-- If we're only deleting _n_ items, avoid retrieving all items
|
||||
-- for faster performance
|
||||
--
|
||||
-- Start from the tail of the list, since that's where oldest elements
|
||||
-- are generally added for FIFO lists
|
||||
if limit > 0 then
|
||||
rangeStart = -1 - limit + 1
|
||||
rangeEnd = -1
|
||||
end
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/cleanList"
|
||||
--- @include "includes/cleanSet"
|
||||
|
||||
local result
|
||||
if ARGV[4] == "active" then
|
||||
result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], false --[[ hasFinished ]],
|
||||
repeatKey)
|
||||
elseif ARGV[4] == "delayed" then
|
||||
rangeEnd = "+inf"
|
||||
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
|
||||
{"processedOn", "timestamp"}, false --[[ hasFinished ]], repeatKey)
|
||||
elseif ARGV[4] == "prioritized" then
|
||||
rangeEnd = "+inf"
|
||||
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
|
||||
{"timestamp"}, false --[[ hasFinished ]], repeatKey)
|
||||
elseif ARGV[4] == "wait" or ARGV[4] == "paused" then
|
||||
result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], true --[[ hasFinished ]],
|
||||
repeatKey)
|
||||
else
|
||||
rangeEnd = ARGV[2]
|
||||
-- No need to pass repeat key as in that moment job won't be related to a job scheduler
|
||||
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
|
||||
{"finishedOn"}, true --[[ hasFinished ]])
|
||||
end
|
||||
|
||||
rcall("XADD", KEYS[2], "*", "event", "cleaned", "count", result[2])
|
||||
|
||||
return result[1]
|
||||
41
node_modules/bullmq/dist/esm/commands/drain-5.lua
generated
vendored
Normal file
41
node_modules/bullmq/dist/esm/commands/drain-5.lua
generated
vendored
Normal file
@@ -0,0 +1,41 @@
|
||||
--[[
|
||||
Drains the queue, removes all jobs that are waiting
|
||||
or delayed, but not active, completed or failed
|
||||
|
||||
Input:
|
||||
KEYS[1] 'wait',
|
||||
KEYS[2] 'paused'
|
||||
KEYS[3] 'delayed'
|
||||
KEYS[4] 'prioritized'
|
||||
KEYS[5] 'jobschedulers' (repeat)
|
||||
|
||||
ARGV[1] queue key prefix
|
||||
ARGV[2] should clean delayed jobs
|
||||
]]
|
||||
local rcall = redis.call
|
||||
local queueBaseKey = ARGV[1]
|
||||
|
||||
--- @include "includes/removeListJobs"
|
||||
--- @include "includes/removeZSetJobs"
|
||||
|
||||
-- We must not remove delayed jobs if they are associated to a job scheduler.
|
||||
local scheduledJobs = {}
|
||||
local jobSchedulers = rcall("ZRANGE", KEYS[5], 0, -1, "WITHSCORES")
|
||||
|
||||
-- For every job scheduler, get the current delayed job id.
|
||||
for i = 1, #jobSchedulers, 2 do
|
||||
local jobSchedulerId = jobSchedulers[i]
|
||||
local jobSchedulerMillis = jobSchedulers[i + 1]
|
||||
|
||||
local delayedJobId = "repeat:" .. jobSchedulerId .. ":" .. jobSchedulerMillis
|
||||
scheduledJobs[delayedJobId] = true
|
||||
end
|
||||
|
||||
removeListJobs(KEYS[1], true, queueBaseKey, 0, scheduledJobs) -- wait
|
||||
removeListJobs(KEYS[2], true, queueBaseKey, 0, scheduledJobs) -- paused
|
||||
|
||||
if ARGV[2] == "1" then
|
||||
removeZSetJobs(KEYS[3], true, queueBaseKey, 0, scheduledJobs) -- delayed
|
||||
end
|
||||
|
||||
removeZSetJobs(KEYS[4], true, queueBaseKey, 0, scheduledJobs) -- prioritized
|
||||
23
node_modules/bullmq/dist/esm/commands/extendLock-2.lua
generated
vendored
Normal file
23
node_modules/bullmq/dist/esm/commands/extendLock-2.lua
generated
vendored
Normal file
@@ -0,0 +1,23 @@
|
||||
--[[
|
||||
Extend lock and removes the job from the stalled set.
|
||||
|
||||
Input:
|
||||
KEYS[1] 'lock',
|
||||
KEYS[2] 'stalled'
|
||||
|
||||
ARGV[1] token
|
||||
ARGV[2] lock duration in milliseconds
|
||||
ARGV[3] jobid
|
||||
|
||||
Output:
|
||||
"1" if lock extended successfully.
|
||||
]]
|
||||
local rcall = redis.call
|
||||
if rcall("GET", KEYS[1]) == ARGV[1] then
|
||||
-- if rcall("SET", KEYS[1], ARGV[1], "PX", ARGV[2], "XX") then
|
||||
if rcall("SET", KEYS[1], ARGV[1], "PX", ARGV[2]) then
|
||||
rcall("SREM", KEYS[2], ARGV[3])
|
||||
return 1
|
||||
end
|
||||
end
|
||||
return 0
|
||||
48
node_modules/bullmq/dist/esm/commands/extendLocks-1.lua
generated
vendored
Normal file
48
node_modules/bullmq/dist/esm/commands/extendLocks-1.lua
generated
vendored
Normal file
@@ -0,0 +1,48 @@
|
||||
--[[
|
||||
Extend locks for multiple jobs and remove them from the stalled set if successful.
|
||||
Return the list of job IDs for which the operation failed.
|
||||
|
||||
KEYS[1] = stalled key
|
||||
|
||||
ARGV[1] = baseKey
|
||||
ARGV[2] = tokens
|
||||
ARGV[3] = jobIds
|
||||
ARGV[4] = lockDuration (ms)
|
||||
|
||||
Output:
|
||||
An array of failed job IDs. If empty, all succeeded.
|
||||
]]
|
||||
local rcall = redis.call
|
||||
|
||||
local stalledKey = KEYS[1]
|
||||
local baseKey = ARGV[1]
|
||||
local tokens = cmsgpack.unpack(ARGV[2])
|
||||
local jobIds = cmsgpack.unpack(ARGV[3])
|
||||
local lockDuration = ARGV[4]
|
||||
|
||||
local jobCount = #jobIds
|
||||
local failedJobs = {}
|
||||
|
||||
for i = 1, jobCount, 1 do
|
||||
local lockKey = baseKey .. jobIds[i] .. ':lock'
|
||||
local jobId = jobIds[i]
|
||||
local token = tokens[i]
|
||||
|
||||
local currentToken = rcall("GET", lockKey)
|
||||
if currentToken then
|
||||
if currentToken == token then
|
||||
local setResult = rcall("SET", lockKey, token, "PX", lockDuration)
|
||||
if setResult then
|
||||
rcall("SREM", stalledKey, jobId)
|
||||
else
|
||||
table.insert(failedJobs, jobId)
|
||||
end
|
||||
else
|
||||
table.insert(failedJobs, jobId)
|
||||
end
|
||||
else
|
||||
table.insert(failedJobs, jobId)
|
||||
end
|
||||
end
|
||||
|
||||
return failedJobs
|
||||
36
node_modules/bullmq/dist/esm/commands/getCounts-1.lua
generated
vendored
Normal file
36
node_modules/bullmq/dist/esm/commands/getCounts-1.lua
generated
vendored
Normal file
@@ -0,0 +1,36 @@
|
||||
--[[
|
||||
Get counts per provided states
|
||||
|
||||
Input:
|
||||
KEYS[1] 'prefix'
|
||||
|
||||
ARGV[1...] types
|
||||
]]
|
||||
local rcall = redis.call;
|
||||
local prefix = KEYS[1]
|
||||
local results = {}
|
||||
|
||||
for i = 1, #ARGV do
|
||||
local stateKey = prefix .. ARGV[i]
|
||||
if ARGV[i] == "wait" or ARGV[i] == "paused" then
|
||||
-- Markers in waitlist DEPRECATED in v5: Remove in v6.
|
||||
local marker = rcall("LINDEX", stateKey, -1)
|
||||
if marker and string.sub(marker, 1, 2) == "0:" then
|
||||
local count = rcall("LLEN", stateKey)
|
||||
if count > 1 then
|
||||
rcall("RPOP", stateKey)
|
||||
results[#results+1] = count-1
|
||||
else
|
||||
results[#results+1] = 0
|
||||
end
|
||||
else
|
||||
results[#results+1] = rcall("LLEN", stateKey)
|
||||
end
|
||||
elseif ARGV[i] == "active" then
|
||||
results[#results+1] = rcall("LLEN", stateKey)
|
||||
else
|
||||
results[#results+1] = rcall("ZCARD", stateKey)
|
||||
end
|
||||
end
|
||||
|
||||
return results
|
||||
28
node_modules/bullmq/dist/esm/commands/getCountsPerPriority-4.lua
generated
vendored
Normal file
28
node_modules/bullmq/dist/esm/commands/getCountsPerPriority-4.lua
generated
vendored
Normal file
@@ -0,0 +1,28 @@
|
||||
--[[
|
||||
Get counts per provided states
|
||||
|
||||
Input:
|
||||
KEYS[1] wait key
|
||||
KEYS[2] paused key
|
||||
KEYS[3] meta key
|
||||
KEYS[4] prioritized key
|
||||
|
||||
ARGV[1...] priorities
|
||||
]]
|
||||
local rcall = redis.call
|
||||
local results = {}
|
||||
local waitKey = KEYS[1]
|
||||
local pausedKey = KEYS[2]
|
||||
local prioritizedKey = KEYS[4]
|
||||
|
||||
for i = 1, #ARGV do
|
||||
local priority = tonumber(ARGV[i])
|
||||
if priority == 0 then
|
||||
results[#results+1] = rcall("LLEN", waitKey)
|
||||
else
|
||||
results[#results+1] = rcall("ZCOUNT", prioritizedKey,
|
||||
priority * 0x100000000, (priority + 1) * 0x100000000 - 1)
|
||||
end
|
||||
end
|
||||
|
||||
return results
|
||||
31
node_modules/bullmq/dist/esm/commands/getDependencyCounts-4.lua
generated
vendored
Normal file
31
node_modules/bullmq/dist/esm/commands/getDependencyCounts-4.lua
generated
vendored
Normal file
@@ -0,0 +1,31 @@
|
||||
--[[
|
||||
Get counts per child states
|
||||
|
||||
Input:
|
||||
KEYS[1] processed key
|
||||
KEYS[2] unprocessed key
|
||||
KEYS[3] ignored key
|
||||
KEYS[4] failed key
|
||||
|
||||
ARGV[1...] types
|
||||
]]
|
||||
local rcall = redis.call;
|
||||
local processedKey = KEYS[1]
|
||||
local unprocessedKey = KEYS[2]
|
||||
local ignoredKey = KEYS[3]
|
||||
local failedKey = KEYS[4]
|
||||
local results = {}
|
||||
|
||||
for i = 1, #ARGV do
|
||||
if ARGV[i] == "processed" then
|
||||
results[#results+1] = rcall("HLEN", processedKey)
|
||||
elseif ARGV[i] == "unprocessed" then
|
||||
results[#results+1] = rcall("SCARD", unprocessedKey)
|
||||
elseif ARGV[i] == "ignored" then
|
||||
results[#results+1] = rcall("HLEN", ignoredKey)
|
||||
else
|
||||
results[#results+1] = rcall("ZCARD", failedKey)
|
||||
end
|
||||
end
|
||||
|
||||
return results
|
||||
19
node_modules/bullmq/dist/esm/commands/getJobScheduler-1.lua
generated
vendored
Normal file
19
node_modules/bullmq/dist/esm/commands/getJobScheduler-1.lua
generated
vendored
Normal file
@@ -0,0 +1,19 @@
|
||||
--[[
|
||||
Get job scheduler record.
|
||||
|
||||
Input:
|
||||
KEYS[1] 'repeat' key
|
||||
|
||||
ARGV[1] id
|
||||
]]
|
||||
|
||||
local rcall = redis.call
|
||||
local jobSchedulerKey = KEYS[1] .. ":" .. ARGV[1]
|
||||
|
||||
local score = rcall("ZSCORE", KEYS[1], ARGV[1])
|
||||
|
||||
if score then
|
||||
return {rcall("HGETALL", jobSchedulerKey), score} -- get job data
|
||||
end
|
||||
|
||||
return {nil, nil}
|
||||
151
node_modules/bullmq/dist/esm/commands/getJobs-1.lua
generated
vendored
Normal file
151
node_modules/bullmq/dist/esm/commands/getJobs-1.lua
generated
vendored
Normal file
@@ -0,0 +1,151 @@
|
||||
--[[
|
||||
Get jobs (id + data) for the provided states.
|
||||
|
||||
Job ids and their hashes are read in the same script so that ids whose hash
|
||||
disappears after the id is read (but before the job is loaded) do not appear
|
||||
in the result set. Ids without a job hash (for example the deprecated wait
|
||||
list marker entry stored in the list) are skipped. For bounded ranges the
|
||||
script iterates forward using the range offset as a cursor to backfill
|
||||
skipped ids, preserving the requested page size when possible.
|
||||
|
||||
Input:
|
||||
KEYS[1] 'prefix'
|
||||
|
||||
ARGV[1] start
|
||||
ARGV[2] end
|
||||
ARGV[3] asc ('1' | '0')
|
||||
ARGV[4] max iterations (backfill bound)
|
||||
ARGV[5...] types
|
||||
|
||||
Output:
|
||||
results grouped per requested type; each entry is a
|
||||
{jobId, {field, value, ...}} tuple
|
||||
]]
|
||||
local rcall = redis.call
|
||||
local prefix = KEYS[1]
|
||||
local rangeStart = tonumber(ARGV[1])
|
||||
local rangeEnd = tonumber(ARGV[2])
|
||||
local asc = ARGV[3] == "1"
|
||||
local max_iterations = tonumber(ARGV[4])
|
||||
local results = {}
|
||||
|
||||
local function isListType(stateType)
|
||||
return stateType == "wait" or stateType == "paused" or stateType == "active"
|
||||
end
|
||||
|
||||
-- Fetch a slice of ids for the given state respecting the requested order.
|
||||
local function fetchIds(stateKey, stateType, sliceStart, sliceEnd, listLength)
|
||||
if isListType(stateType) then
|
||||
if asc then
|
||||
local modifiedRangeStart
|
||||
local modifiedRangeEnd
|
||||
if sliceStart == -1 then
|
||||
modifiedRangeStart = 0
|
||||
else
|
||||
modifiedRangeStart = -(sliceStart + 1)
|
||||
end
|
||||
|
||||
if sliceEnd == -1 then
|
||||
modifiedRangeEnd = 0
|
||||
else
|
||||
modifiedRangeEnd = -(sliceEnd + 1)
|
||||
end
|
||||
|
||||
-- Ascending list slices use negative indexes. When the whole window is
|
||||
-- beyond the list length Redis clamps both indexes to 0 and LRANGE would
|
||||
-- return the head element, so guard against out-of-range slices with LLEN.
|
||||
if listLength ~= nil and sliceStart >= 0 and sliceEnd >= 0 and sliceStart >= listLength then
|
||||
return {}
|
||||
end
|
||||
|
||||
local ids = rcall("LRANGE", stateKey, modifiedRangeEnd, modifiedRangeStart)
|
||||
local reversed = {}
|
||||
for i = #ids, 1, -1 do
|
||||
reversed[#reversed + 1] = ids[i]
|
||||
end
|
||||
return reversed
|
||||
else
|
||||
return rcall("LRANGE", stateKey, sliceStart, sliceEnd)
|
||||
end
|
||||
else
|
||||
if asc then
|
||||
return rcall("ZRANGE", stateKey, sliceStart, sliceEnd)
|
||||
else
|
||||
return rcall("ZREVRANGE", stateKey, sliceStart, sliceEnd)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
-- Fetch the job hash for an id and append it when present.
|
||||
local function appendJob(entries, jobId)
|
||||
local jobData = rcall("HGETALL", prefix .. jobId)
|
||||
if #jobData > 0 then
|
||||
entries[#entries + 1] = {jobId, jobData}
|
||||
end
|
||||
end
|
||||
|
||||
local function cleanupDeprecatedMarker(stateKey, stateType)
|
||||
if stateType == "wait" or stateType == "paused" then
|
||||
local marker = rcall("LINDEX", stateKey, -1)
|
||||
if marker and string.sub(marker, 1, 2) == "0:" then
|
||||
local count = rcall("LLEN", stateKey)
|
||||
if count > 1 then
|
||||
rcall("RPOP", stateKey)
|
||||
return count - 1
|
||||
end
|
||||
return 0
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
local function collectJobs(stateKey, stateType)
|
||||
local entries = {}
|
||||
local listLength
|
||||
local cleanedListLength = cleanupDeprecatedMarker(stateKey, stateType)
|
||||
|
||||
if asc and isListType(stateType) and rangeStart >= 0 and rangeEnd >= 0 then
|
||||
if cleanedListLength ~= nil then
|
||||
listLength = cleanedListLength
|
||||
else
|
||||
listLength = rcall("LLEN", stateKey)
|
||||
end
|
||||
end
|
||||
|
||||
-- Unbounded or negative ranges: fetch the exact window and skip missing ids.
|
||||
if rangeStart < 0 or rangeEnd < 0 then
|
||||
local ids = fetchIds(stateKey, stateType, rangeStart, rangeEnd, listLength)
|
||||
for i = 1, #ids do
|
||||
appendJob(entries, ids[i])
|
||||
end
|
||||
return entries
|
||||
end
|
||||
|
||||
-- Bounded range: iterate forward to backfill skipped ids.
|
||||
local needed = rangeEnd - rangeStart + 1
|
||||
local cursor = rangeStart
|
||||
local iterations = 0
|
||||
while #entries < needed and iterations < max_iterations do
|
||||
local ids = fetchIds(stateKey, stateType, cursor, cursor + needed - 1, listLength)
|
||||
if #ids == 0 then
|
||||
break
|
||||
end
|
||||
for i = 1, #ids do
|
||||
if #entries >= needed then
|
||||
break
|
||||
end
|
||||
appendJob(entries, ids[i])
|
||||
end
|
||||
cursor = cursor + #ids
|
||||
iterations = iterations + 1
|
||||
end
|
||||
|
||||
return entries
|
||||
end
|
||||
|
||||
for i = 5, #ARGV do
|
||||
local stateType = ARGV[i]
|
||||
local stateKey = prefix .. stateType
|
||||
results[#results + 1] = collectJobs(stateKey, stateType)
|
||||
end
|
||||
|
||||
return results
|
||||
19
node_modules/bullmq/dist/esm/commands/getMetrics-2.lua
generated
vendored
Normal file
19
node_modules/bullmq/dist/esm/commands/getMetrics-2.lua
generated
vendored
Normal file
@@ -0,0 +1,19 @@
|
||||
--[[
|
||||
Get metrics
|
||||
|
||||
Input:
|
||||
KEYS[1] 'metrics' key
|
||||
KEYS[2] 'metrics data' key
|
||||
|
||||
ARGV[1] start index
|
||||
ARGV[2] end index
|
||||
]]
|
||||
local rcall = redis.call;
|
||||
local metricsKey = KEYS[1]
|
||||
local dataKey = KEYS[2]
|
||||
|
||||
local metrics = rcall("HMGET", metricsKey, "count", "prevTS", "prevCount")
|
||||
local data = rcall("LRANGE", dataKey, tonumber(ARGV[1]), tonumber(ARGV[2]))
|
||||
local numPoints = rcall("LLEN", dataKey)
|
||||
|
||||
return {metrics, data, numPoints}
|
||||
70
node_modules/bullmq/dist/esm/commands/getRanges-1.lua
generated
vendored
Normal file
70
node_modules/bullmq/dist/esm/commands/getRanges-1.lua
generated
vendored
Normal file
@@ -0,0 +1,70 @@
|
||||
--[[
|
||||
Get job ids per provided states
|
||||
|
||||
Input:
|
||||
KEYS[1] 'prefix'
|
||||
|
||||
ARGV[1] start
|
||||
ARGV[2] end
|
||||
ARGV[3] asc
|
||||
ARGV[4...] types
|
||||
]]
|
||||
local rcall = redis.call
|
||||
local prefix = KEYS[1]
|
||||
local rangeStart = tonumber(ARGV[1])
|
||||
local rangeEnd = tonumber(ARGV[2])
|
||||
local asc = ARGV[3]
|
||||
local results = {}
|
||||
|
||||
local function getRangeInList(listKey, asc, rangeStart, rangeEnd, results)
|
||||
if asc == "1" then
|
||||
local modifiedRangeStart
|
||||
local modifiedRangeEnd
|
||||
if rangeStart == -1 then
|
||||
modifiedRangeStart = 0
|
||||
else
|
||||
modifiedRangeStart = -(rangeStart + 1)
|
||||
end
|
||||
|
||||
if rangeEnd == -1 then
|
||||
modifiedRangeEnd = 0
|
||||
else
|
||||
modifiedRangeEnd = -(rangeEnd + 1)
|
||||
end
|
||||
|
||||
results[#results+1] = rcall("LRANGE", listKey,
|
||||
modifiedRangeEnd,
|
||||
modifiedRangeStart)
|
||||
else
|
||||
results[#results+1] = rcall("LRANGE", listKey, rangeStart, rangeEnd)
|
||||
end
|
||||
end
|
||||
|
||||
for i = 4, #ARGV do
|
||||
local stateKey = prefix .. ARGV[i]
|
||||
if ARGV[i] == "wait" or ARGV[i] == "paused" then
|
||||
-- Markers in waitlist DEPRECATED in v5: Remove in v6.
|
||||
local marker = rcall("LINDEX", stateKey, -1)
|
||||
if marker and string.sub(marker, 1, 2) == "0:" then
|
||||
local count = rcall("LLEN", stateKey)
|
||||
if count > 1 then
|
||||
rcall("RPOP", stateKey)
|
||||
getRangeInList(stateKey, asc, rangeStart, rangeEnd, results)
|
||||
else
|
||||
results[#results+1] = {}
|
||||
end
|
||||
else
|
||||
getRangeInList(stateKey, asc, rangeStart, rangeEnd, results)
|
||||
end
|
||||
elseif ARGV[i] == "active" then
|
||||
getRangeInList(stateKey, asc, rangeStart, rangeEnd, results)
|
||||
else
|
||||
if asc == "1" then
|
||||
results[#results+1] = rcall("ZRANGE", stateKey, rangeStart, rangeEnd)
|
||||
else
|
||||
results[#results+1] = rcall("ZREVRANGE", stateKey, rangeStart, rangeEnd)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
return results
|
||||
26
node_modules/bullmq/dist/esm/commands/getRateLimitTtl-2.lua
generated
vendored
Normal file
26
node_modules/bullmq/dist/esm/commands/getRateLimitTtl-2.lua
generated
vendored
Normal file
@@ -0,0 +1,26 @@
|
||||
--[[
|
||||
Get rate limit ttl
|
||||
|
||||
Input:
|
||||
KEYS[1] 'limiter'
|
||||
KEYS[2] 'meta'
|
||||
|
||||
ARGV[1] maxJobs
|
||||
]]
|
||||
|
||||
local rcall = redis.call
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/getRateLimitTTL"
|
||||
|
||||
local rateLimiterKey = KEYS[1]
|
||||
if ARGV[1] ~= "0" then
|
||||
return getRateLimitTTL(tonumber(ARGV[1]), rateLimiterKey)
|
||||
else
|
||||
local rateLimitMax = rcall("HGET", KEYS[2], "max")
|
||||
if rateLimitMax then
|
||||
return getRateLimitTTL(tonumber(rateLimitMax), rateLimiterKey)
|
||||
end
|
||||
|
||||
return rcall("PTTL", rateLimiterKey)
|
||||
end
|
||||
65
node_modules/bullmq/dist/esm/commands/getState-8.lua
generated
vendored
Normal file
65
node_modules/bullmq/dist/esm/commands/getState-8.lua
generated
vendored
Normal file
@@ -0,0 +1,65 @@
|
||||
--[[
|
||||
Get a job state
|
||||
|
||||
Input:
|
||||
KEYS[1] 'completed' key,
|
||||
KEYS[2] 'failed' key
|
||||
KEYS[3] 'delayed' key
|
||||
KEYS[4] 'active' key
|
||||
KEYS[5] 'wait' key
|
||||
KEYS[6] 'paused' key
|
||||
KEYS[7] 'waiting-children' key
|
||||
KEYS[8] 'prioritized' key
|
||||
|
||||
ARGV[1] job id
|
||||
Output:
|
||||
'completed'
|
||||
'failed'
|
||||
'delayed'
|
||||
'active'
|
||||
'prioritized'
|
||||
'waiting'
|
||||
'waiting-children'
|
||||
'unknown'
|
||||
]]
|
||||
local rcall = redis.call
|
||||
|
||||
if rcall("ZSCORE", KEYS[1], ARGV[1]) then
|
||||
return "completed"
|
||||
end
|
||||
|
||||
if rcall("ZSCORE", KEYS[2], ARGV[1]) then
|
||||
return "failed"
|
||||
end
|
||||
|
||||
if rcall("ZSCORE", KEYS[3], ARGV[1]) then
|
||||
return "delayed"
|
||||
end
|
||||
|
||||
if rcall("ZSCORE", KEYS[8], ARGV[1]) then
|
||||
return "prioritized"
|
||||
end
|
||||
|
||||
-- Includes
|
||||
--- @include "includes/checkItemInList"
|
||||
|
||||
local active_items = rcall("LRANGE", KEYS[4] , 0, -1)
|
||||
if checkItemInList(active_items, ARGV[1]) ~= nil then
|
||||
return "active"
|
||||
end
|
||||
|
||||
local wait_items = rcall("LRANGE", KEYS[5] , 0, -1)
|
||||
if checkItemInList(wait_items, ARGV[1]) ~= nil then
|
||||
return "waiting"
|
||||
end
|
||||
|
||||
local paused_items = rcall("LRANGE", KEYS[6] , 0, -1)
|
||||
if checkItemInList(paused_items, ARGV[1]) ~= nil then
|
||||
return "waiting"
|
||||
end
|
||||
|
||||
if rcall("ZSCORE", KEYS[7], ARGV[1]) then
|
||||
return "waiting-children"
|
||||
end
|
||||
|
||||
return "unknown"
|
||||
58
node_modules/bullmq/dist/esm/commands/getStateV2-8.lua
generated
vendored
Normal file
58
node_modules/bullmq/dist/esm/commands/getStateV2-8.lua
generated
vendored
Normal file
@@ -0,0 +1,58 @@
|
||||
--[[
|
||||
Get a job state
|
||||
|
||||
Input:
|
||||
KEYS[1] 'completed' key,
|
||||
KEYS[2] 'failed' key
|
||||
KEYS[3] 'delayed' key
|
||||
KEYS[4] 'active' key
|
||||
KEYS[5] 'wait' key
|
||||
KEYS[6] 'paused' key
|
||||
KEYS[7] 'waiting-children' key
|
||||
KEYS[8] 'prioritized' key
|
||||
|
||||
ARGV[1] job id
|
||||
Output:
|
||||
'completed'
|
||||
'failed'
|
||||
'delayed'
|
||||
'active'
|
||||
'waiting'
|
||||
'waiting-children'
|
||||
'unknown'
|
||||
]]
|
||||
local rcall = redis.call
|
||||
|
||||
if rcall("ZSCORE", KEYS[1], ARGV[1]) then
|
||||
return "completed"
|
||||
end
|
||||
|
||||
if rcall("ZSCORE", KEYS[2], ARGV[1]) then
|
||||
return "failed"
|
||||
end
|
||||
|
||||
if rcall("ZSCORE", KEYS[3], ARGV[1]) then
|
||||
return "delayed"
|
||||
end
|
||||
|
||||
if rcall("ZSCORE", KEYS[8], ARGV[1]) then
|
||||
return "prioritized"
|
||||
end
|
||||
|
||||
if rcall("LPOS", KEYS[4] , ARGV[1]) then
|
||||
return "active"
|
||||
end
|
||||
|
||||
if rcall("LPOS", KEYS[5] , ARGV[1]) then
|
||||
return "waiting"
|
||||
end
|
||||
|
||||
if rcall("LPOS", KEYS[6] , ARGV[1]) then
|
||||
return "waiting"
|
||||
end
|
||||
|
||||
if rcall("ZSCORE", KEYS[7] , ARGV[1]) then
|
||||
return "waiting-children"
|
||||
end
|
||||
|
||||
return "unknown"
|
||||
9
node_modules/bullmq/dist/esm/commands/includes/addBaseMarkerIfNeeded.lua
generated
vendored
Normal file
9
node_modules/bullmq/dist/esm/commands/includes/addBaseMarkerIfNeeded.lua
generated
vendored
Normal file
@@ -0,0 +1,9 @@
|
||||
--[[
|
||||
Add marker if needed when a job is available.
|
||||
]]
|
||||
|
||||
local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
||||
if not isPausedOrMaxed then
|
||||
rcall("ZADD", markerKey, 0, "0")
|
||||
end
|
||||
end
|
||||
15
node_modules/bullmq/dist/esm/commands/includes/addDelayMarkerIfNeeded.lua
generated
vendored
Normal file
15
node_modules/bullmq/dist/esm/commands/includes/addDelayMarkerIfNeeded.lua
generated
vendored
Normal file
@@ -0,0 +1,15 @@
|
||||
--[[
|
||||
Add delay marker if needed.
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "getNextDelayedTimestamp"
|
||||
|
||||
local function addDelayMarkerIfNeeded(markerKey, delayedKey)
|
||||
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
||||
if nextTimestamp ~= nil then
|
||||
-- Replace the score of the marker with the newest known
|
||||
-- next timestamp.
|
||||
rcall("ZADD", markerKey, nextTimestamp, "1")
|
||||
end
|
||||
end
|
||||
23
node_modules/bullmq/dist/esm/commands/includes/addDelayedJob.lua
generated
vendored
Normal file
23
node_modules/bullmq/dist/esm/commands/includes/addDelayedJob.lua
generated
vendored
Normal file
@@ -0,0 +1,23 @@
|
||||
--[[
|
||||
Adds a delayed job to the queue by doing the following:
|
||||
- Creates a new job key with the job data.
|
||||
- adds to delayed zset.
|
||||
- Emits a global event 'delayed' if the job is delayed.
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "addDelayMarkerIfNeeded"
|
||||
--- @include "getDelayedScore"
|
||||
|
||||
local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
|
||||
maxEvents, markerKey, delay)
|
||||
|
||||
local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
|
||||
|
||||
rcall("ZADD", delayedKey, score, jobId)
|
||||
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
|
||||
"jobId", jobId, "delay", delayedTimestamp)
|
||||
|
||||
-- mark that a delayed job is available
|
||||
addDelayMarkerIfNeeded(markerKey, delayedKey)
|
||||
end
|
||||
19
node_modules/bullmq/dist/esm/commands/includes/addJobFromScheduler.lua
generated
vendored
Normal file
19
node_modules/bullmq/dist/esm/commands/includes/addJobFromScheduler.lua
generated
vendored
Normal file
@@ -0,0 +1,19 @@
|
||||
--[[
|
||||
Add delay marker if needed.
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "storeAndEnqueueJob"
|
||||
|
||||
local function addJobFromScheduler(jobKey, jobId, opts, waitKey, pausedKey, activeKey, metaKey,
|
||||
prioritizedKey, priorityCounter, delayedKey, markerKey, eventsKey, name, maxEvents, timestamp,
|
||||
data, jobSchedulerId, repeatDelay)
|
||||
|
||||
opts['delay'] = repeatDelay
|
||||
opts['jobId'] = jobId
|
||||
|
||||
storeAndEnqueueJob(eventsKey, jobKey, jobId, name, data, opts,
|
||||
timestamp, nil, nil, jobSchedulerId, maxEvents,
|
||||
waitKey, pausedKey, activeKey, metaKey, prioritizedKey,
|
||||
priorityCounter, delayedKey, markerKey)
|
||||
end
|
||||
11
node_modules/bullmq/dist/esm/commands/includes/addJobInTargetList.lua
generated
vendored
Normal file
11
node_modules/bullmq/dist/esm/commands/includes/addJobInTargetList.lua
generated
vendored
Normal file
@@ -0,0 +1,11 @@
|
||||
--[[
|
||||
Function to add job in target list and add marker if needed.
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "addBaseMarkerIfNeeded"
|
||||
|
||||
local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
|
||||
rcall(pushCmd, targetKey, jobId)
|
||||
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
||||
end
|
||||
14
node_modules/bullmq/dist/esm/commands/includes/addJobWithPriority.lua
generated
vendored
Normal file
14
node_modules/bullmq/dist/esm/commands/includes/addJobWithPriority.lua
generated
vendored
Normal file
@@ -0,0 +1,14 @@
|
||||
--[[
|
||||
Function to add job considering priority.
|
||||
]]
|
||||
|
||||
-- Includes
|
||||
--- @include "addBaseMarkerIfNeeded"
|
||||
--- @include "getPriorityScore"
|
||||
|
||||
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
|
||||
isPausedOrMaxed)
|
||||
local score = getPriorityScore(priority, priorityCounterKey)
|
||||
rcall("ZADD", prioritizedKey, score, jobId)
|
||||
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
||||
end
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user