Review funkt

This commit is contained in:
2026-08-09 21:56:00 +00:00
parent f522ae9953
commit fe9dfa4519
11366 changed files with 2335077 additions and 97 deletions

View File

@@ -0,0 +1,3 @@
-- Insert a flow (tree) of jobs atomically. Param: $1 entries (jsonb array,
-- ordered roots-first). Returns the job ids in input order.
SELECT id FROM add_flow($1::jsonb) AS t(id);

View File

@@ -0,0 +1,5 @@
-- Insert a single job, routing it to waiting or delayed; returns its id.
SELECT add_job(
$1, $2, $3, $4::jsonb, $5::jsonb, $6, $7, $8, $9,
$10, $11, $12, $13, $14, $15
) AS id;

View File

@@ -0,0 +1,6 @@
-- Register/update a scheduler and enqueue its next iteration; returns the
-- delayed job id and its delay. Params: $1 queue, $2 scheduler_id,
-- $3 next_millis, $4 template_data (jsonb), $5 template_opts (jsonb),
-- $6 opts (jsonb), $7 delayed_opts (jsonb), $8 now, $9 producer_id.
SELECT job_id, delay
FROM add_job_scheduler($1, $2, $3, $4, $5, $6, $7, $8, $9);

View File

@@ -0,0 +1,3 @@
-- Fast set-based bulk insert of INDEPENDENT jobs (no parents, no dedup).
-- Param: $1 queue, $2 entries (jsonb array). Returns the job ids in input order.
SELECT id FROM add_jobs_bulk($1, $2::jsonb) AS t(id);

View File

@@ -0,0 +1,14 @@
-- Append a log line at the next per-job ordinal; returns its index.
-- Params: $1 queue, $2 job_id, $3 row.
-- The unqualified table name resolves in the configured BullMQ schema via
-- search_path (default schema: bullmq).
INSERT INTO job_log (queue, job_id, idx, row)
VALUES (
$1, $2,
COALESCE(
(SELECT MAX(idx) + 1 FROM job_log WHERE queue = $1 AND job_id = $2),
0
),
$3
)
RETURNING idx;

View File

@@ -0,0 +1,3 @@
-- Reschedule a delayed job. Params: $1 queue, $2 id, $3 delay, $4 now.
-- Returns 0 ok, -1 missing, -3 not delayed.
SELECT change_delay($1, $2, $3, $4) AS code;

View File

@@ -0,0 +1,3 @@
-- Change a job's priority (and reposition via lifo). Params: $1 queue, $2 id,
-- $3 priority, $4 lifo. Returns 0 ok, -1 missing.
SELECT change_priority($1, $2, $3, $4) AS code;

View File

@@ -0,0 +1,3 @@
-- Remove jobs of a state older than a timestamp, up to a limit (0 = all);
-- returns removed ids. Params: $1 queue, $2 type, $3 timestamp, $4 limit.
SELECT id FROM clean($1, $2, $3, $4) AS t(id);

View File

@@ -0,0 +1,9 @@
-- Clear a job's logs, keeping the latest $3 entries (NULL/0 → remove all).
-- Params: $1 queue, $2 job_id, $3 keepLogs.
DELETE FROM job_log
WHERE queue = $1 AND job_id = $2
AND idx < COALESCE(
(SELECT MAX(idx) FROM job_log WHERE queue = $1 AND job_id = $2)
- $3 + 1,
idx + 1
);

View File

@@ -0,0 +1,4 @@
-- Record one finished job into the queue/kind metrics (per-minute deltas).
-- Params: $1 queue, $2 kind ('completed' | 'failed'), $3 maxDataPoints,
-- $4 finish timestamp (epoch ms).
SELECT collect_metrics($1, $2, $3, $4);

View File

@@ -0,0 +1,6 @@
-- Unconditionally remove a deduplication key (Queue.removeDeduplicationKey /
-- the deprecated removeDebounceKey). Params: $1 queue, $2 dedup_id. Returns the
-- removed key (0 or 1 row).
DELETE FROM dedup
WHERE queue = $1 AND dedup_id = $2
RETURNING dedup_id;

View File

@@ -0,0 +1,2 @@
-- Remove waiting (and optionally delayed) jobs. Params: $1 queue, $2 delayed.
SELECT drain($1, $2);

View File

@@ -0,0 +1,3 @@
-- Refresh an active job's lock; returns 1 on success, 0 if the lock was lost.
-- Params: $1 queue, $2 id, $3 token, $4 lock_ms, $5 now_ms.
SELECT extend_lock($1, $2, $3, $4, $5) AS n;

View File

@@ -0,0 +1,25 @@
-- Refresh multiple active-job locks in one round-trip; returns the ids whose
-- locks could not be renewed. Params: $1 queue, $2 ids (text[]), $3 tokens
-- (text[]), $4 lock_ms, $5 now_ms.
WITH input AS (
SELECT ids.job_id, toks.token, ids.ord
FROM unnest($2::text[]) WITH ORDINALITY AS ids(job_id, ord)
JOIN unnest($3::text[]) WITH ORDINALITY AS toks(token, ord)
USING (ord)
),
updated AS (
UPDATE job
SET locked_until_ms = $5::bigint + $4::bigint
FROM input
WHERE queue = $1
AND id = input.job_id
AND state = 'active'
AND lock_token = input.token
RETURNING id
)
SELECT input.job_id AS id
FROM input
LEFT JOIN updated
ON updated.id = input.job_id
WHERE updated.id IS NULL
ORDER BY input.ord;

View File

@@ -0,0 +1,11 @@
-- Connected BullMQ clients, mirroring Redis `CLIENT LIST` for worker/queue
-- discovery. Each long-lived worker / QueueEvents listener names its dedicated
-- connection via `application_name` (the PostgreSQL analogue of
-- `CLIENT SETNAME`), so listing the named sessions on this database reproduces
-- the same discovery surface. Pooled query connections leave `application_name`
-- empty and are excluded. One row per connection; the backend formats them into
-- `name=<application_name>` lines for the shared client-list parser.
SELECT application_name
FROM pg_stat_activity
WHERE datname = current_database()
AND application_name <> '';

View File

@@ -0,0 +1,13 @@
-- Job counts by state for a queue. Param: $1 queue.
-- "prioritized" = waiting with priority > 0; "waiting" = waiting with priority 0.
SELECT
COUNT(*) FILTER (WHERE state = 'active') AS active,
COUNT(*) FILTER (WHERE state = 'completed') AS completed,
COUNT(*) FILTER (WHERE state = 'failed') AS failed,
COUNT(*) FILTER (WHERE state = 'delayed') AS delayed,
COUNT(*) FILTER (WHERE state = 'waiting' AND priority = 0) AS waiting,
COUNT(*) FILTER (WHERE state = 'waiting' AND priority > 0) AS prioritized,
COUNT(*) FILTER (WHERE state = 'waiting-children') AS "waiting-children",
(SELECT value FROM meta WHERE queue = $1 AND field = 'paused') AS paused
FROM job
WHERE queue = $1;

View File

@@ -0,0 +1,14 @@
-- Waiting-job counts per priority. Params: $1 queue, $2 priorities (bigint[]).
-- Returns one row per requested priority, in the input array order, with the
-- number of waiting jobs at that priority. Pausing is an O(1) meta flag that
-- leaves jobs in the 'waiting' state, so the counts are identical whether or
-- not the queue is paused (mirrors Redis, where pausing does not touch the
-- wait/prioritized sets).
SELECT COUNT(j.id) AS cnt
FROM unnest($2::bigint[]) WITH ORDINALITY AS pr(priority, ord)
LEFT JOIN job j
ON j.queue = $1
AND j.state = 'waiting'
AND j.priority = pr.priority
GROUP BY pr.ord
ORDER BY pr.ord;

View File

@@ -0,0 +1,7 @@
-- The current "winner" job id for a deduplication key, or none when absent or
-- expired. Params: $1 queue, $2 dedup_id, $3 now (ms).
SELECT job_id
FROM dedup
WHERE queue = $1
AND dedup_id = $2
AND (expire_at_ms IS NULL OR expire_at_ms > $3);

View File

@@ -0,0 +1,8 @@
-- All of a parent job's child dependencies with their status and stored value.
-- Params: $1 parent_queue, $2 parent_id. Status maps to the public categories:
-- processed → processed (value = return value), pending → unprocessed,
-- ignored → ignored (value = reason), failed → failed.
SELECT status::text AS status, child_key, value
FROM job_dependency
WHERE parent_queue = $1 AND parent_id = $2
ORDER BY child_id;

View File

@@ -0,0 +1,7 @@
-- One page of a parent's child dependencies in a single status category.
-- Params: $1 parent_queue, $2 parent_id, $3 status, $4 offset, $5 count.
SELECT child_key, value
FROM job_dependency
WHERE parent_queue = $1 AND parent_id = $2 AND status = $3::dep_status
ORDER BY child_id
OFFSET $4 LIMIT $5;

View File

@@ -0,0 +1,9 @@
-- Child-dependency counts per status for a parent job.
-- Params: $1 parent_queue, $2 parent_id. ("unprocessed" = pending children.)
SELECT
COUNT(*) FILTER (WHERE status = 'processed') AS processed,
COUNT(*) FILTER (WHERE status = 'pending') AS unprocessed,
COUNT(*) FILTER (WHERE status = 'ignored') AS ignored,
COUNT(*) FILTER (WHERE status = 'failed') AS failed
FROM job_dependency
WHERE parent_queue = $1 AND parent_id = $2;

View File

@@ -0,0 +1,7 @@
-- Ignored children of a parent as (child_key → failure reason) pairs. The reason
-- is stored as a JSON scalar string, so `#>> '{}'` extracts the raw text
-- (mirrors the Redis `<jobId>:failed` hash). Params: $1 parent_queue, $2 parent_id.
SELECT child_key, COALESCE(value #>> '{}', value::text) AS reason
FROM job_dependency
WHERE parent_queue = $1 AND parent_id = $2 AND status = 'ignored'
ORDER BY child_id;

View File

@@ -0,0 +1,2 @@
-- A job's full row. Params: $1 queue, $2 id.
SELECT * FROM job WHERE queue = $1 AND id = $2;

View File

@@ -0,0 +1,6 @@
-- A page of a job's logs, oldest first.
-- Params: $1 queue, $2 job_id, $3 offset, $4 limit.
SELECT row FROM job_log
WHERE queue = $1 AND job_id = $2
ORDER BY idx ASC
OFFSET $3 LIMIT $4;

View File

@@ -0,0 +1,2 @@
-- Total number of log lines for a job. Params: $1 queue, $2 job_id.
SELECT COUNT(*) AS count FROM job_log WHERE queue = $1 AND job_id = $2;

View File

@@ -0,0 +1,6 @@
-- A page of a job's logs, newest first.
-- Params: $1 queue, $2 job_id, $3 offset, $4 limit.
SELECT row FROM job_log
WHERE queue = $1 AND job_id = $2
ORDER BY idx DESC
OFFSET $3 LIMIT $4;

View File

@@ -0,0 +1,6 @@
-- Fetch a single scheduler's stored metadata and next-run score.
-- Params: $1 queue, $2 scheduler_id.
SELECT name, iteration_count, limit_count, start_date_ms, end_date_ms, tz,
pattern, every_ms, offset_ms, template_data, template_opts, next_run_ms
FROM scheduler
WHERE queue = $1 AND scheduler_id = $2;

View File

@@ -0,0 +1,2 @@
-- Number of registered schedulers in a queue. Param: $1 queue.
SELECT count(*)::int AS count FROM scheduler WHERE queue = $1;

View File

@@ -0,0 +1,10 @@
-- Fetch a page of schedulers ordered by next-run time. Params: $1 queue,
-- $2 asc (boolean), $3 offset, $4 count (NULL = all remaining).
SELECT scheduler_id, next_run_ms
FROM scheduler
WHERE queue = $1
ORDER BY CASE WHEN $2 THEN next_run_ms END ASC,
CASE WHEN NOT $2 THEN next_run_ms END DESC,
scheduler_id
OFFSET $3
LIMIT $4;

View File

@@ -0,0 +1,14 @@
-- Metrics for a queue/kind: the cumulative meta `count` and the per-minute data
-- points (newest first), sliced to [start, end] with Redis LRANGE semantics
-- (index 0 is the newest point; a negative end means "to the oldest").
-- Params: $1 queue, $2 kind ('completed' | 'failed'), $3 start, $4 end.
SELECT
COALESCE((SELECT count FROM metrics
WHERE queue = $1 AND kind = $2), 0)::bigint AS total,
COALESCE((
SELECT CASE
WHEN $4 < 0 THEN data[($3 + 1) : ]
ELSE data[($3 + 1) : ($4 + 1)]
END
FROM metrics WHERE queue = $1 AND kind = $2
), ARRAY[]::bigint[]) AS data;

View File

@@ -0,0 +1,8 @@
-- Processed children of a parent as (child_key → serialized value) pairs. The
-- value is returned as JSON text so the caller can JSON.parse it (mirrors the
-- Redis `<jobId>:processed` hash of stringified values). Params: $1 parent_queue,
-- $2 parent_id.
SELECT child_key, value::text AS value
FROM job_dependency
WHERE parent_queue = $1 AND parent_id = $2 AND status = 'processed'
ORDER BY child_id;

View File

@@ -0,0 +1,2 @@
-- The full queue metadata hash. Param: $1 queue.
SELECT field, value FROM meta WHERE queue = $1;

View File

@@ -0,0 +1,2 @@
-- A single queue metadata field's value. Params: $1 queue, $2 field.
SELECT value FROM meta WHERE queue = $1 AND field = $2;

View File

@@ -0,0 +1,3 @@
-- Several queue metadata fields. Params: $1 queue, $2 fields (text[]).
SELECT field, value FROM meta
WHERE queue = $1 AND field = ANY($2::text[]);

View File

@@ -0,0 +1,3 @@
-- Job ids in a state, sliced by [start, end]. Params: $1 queue, $2 type,
-- $3 start, $4 end, $5 asc.
SELECT id FROM get_range($1, $2, $3, $4, $5) AS t(id);

View File

@@ -0,0 +1,3 @@
-- Current rate-limit ttl in ms (Redis getRateLimitTtl semantics). Params:
-- $1 queue, $2 maxJobs (0 = unspecified → meta `max` / raw window), $3 now_ms.
SELECT rate_limit_ttl($1, $2, $3) AS ttl;

View File

@@ -0,0 +1,2 @@
-- A job's state and priority. Params: $1 queue, $2 id.
SELECT state, priority FROM job WHERE queue = $1 AND id = $2;

View File

@@ -0,0 +1,4 @@
-- Whether a queue metadata field exists. Params: $1 queue, $2 field.
SELECT EXISTS(
SELECT 1 FROM meta WHERE queue = $1 AND field = $2
) AS exists;

View File

@@ -0,0 +1,11 @@
-- Whether the queue has a claimable (waiting, non-paused) job right now. Used
-- by waitForJob to close the race where a NOTIFY fires before the listener is
-- established (mirrors the atomicity of Redis's blocking pop). Param: $1 queue.
SELECT
EXISTS(
SELECT 1 FROM job WHERE queue = $1 AND state = 'waiting'
)
AND NOT EXISTS(
SELECT 1 FROM meta
WHERE queue = $1 AND field = 'paused' AND value = '1'
) AS present;

View File

@@ -0,0 +1,4 @@
-- A job's finished state and result/reason (for waitUntilFinished / isFinished).
-- Params: $1 queue, $2 id.
SELECT state, return_value, failed_reason
FROM job WHERE queue = $1 AND id = $2;

View File

@@ -0,0 +1,6 @@
-- Whether a job is currently in the given state. Params: $1 queue, $2 id,
-- $3 state.
SELECT EXISTS(
SELECT 1 FROM job
WHERE queue = $1 AND id = $2 AND state = $3::job_state
) AS present;

View File

@@ -0,0 +1,15 @@
-- Whether a (non-prioritized) waiting job is in the wait or paused list. The
-- queue has no separate paused list: a waiting job belongs to the paused list
-- when the queue is paused, and to the wait list otherwise. $3 selects which
-- list to test (true = paused, false = wait). Params: $1 queue, $2 id, $3 paused.
SELECT EXISTS(
SELECT 1 FROM job j
WHERE j.queue = $1 AND j.id = $2
AND j.state = 'waiting' AND j.priority = 0
AND (
EXISTS(
SELECT 1 FROM meta m
WHERE m.queue = $1 AND m.field = 'paused' AND m.value = '1'
)
) = $3
) AS present;

View File

@@ -0,0 +1,6 @@
-- Whether a job is prioritized (waiting with a non-zero priority).
-- Params: $1 queue, $2 id.
SELECT EXISTS(
SELECT 1 FROM job
WHERE queue = $1 AND id = $2 AND state = 'waiting' AND priority > 0
) AS present;

View File

@@ -0,0 +1,4 @@
-- Whether an id corresponds to a registered scheduler. Params: $1 queue, $2 id.
SELECT EXISTS (
SELECT 1 FROM scheduler WHERE queue = $1 AND scheduler_id = $2
) AS exists;

View File

@@ -0,0 +1,10 @@
-- Whether the queue is "maxed": it has a global concurrency limit and the
-- number of active jobs has reached it (mirrors isQueueMaxed). Returns false
-- when no concurrency limit is configured.
-- Params: $1 queue.
SELECT COALESCE(
(SELECT count(*) FROM job WHERE queue = $1 AND state = 'active')
>= (SELECT value::integer FROM meta
WHERE queue = $1 AND field = 'concurrency'),
false
) AS maxed;

View File

@@ -0,0 +1,3 @@
-- Subscribe to the shared event-stream channel. Producers
-- `pg_notify('bullmq_events', <queue>)`; a consumer filters for its own queue.
LISTEN bullmq_events;

View File

@@ -0,0 +1,4 @@
-- Subscribe to the shared job-notification channel (fixed name keeps this
-- portable). Producers `pg_notify('bullmq_jobs', <queue>)`; a worker filters
-- notifications for its own queue.
LISTEN bullmq_jobs;

View File

@@ -0,0 +1,4 @@
-- Move an active job back to wait (Job.moveToWait / dynamic rate limit).
-- Params: $1 queue, $2 id, $3 token ('0' bypasses the lock check), $4 now_ms.
-- Returns the limiter window ms (>=0), or -1 when the job no longer exists.
SELECT move_active_to_wait($1, $2, $3, $4) AS n;

View File

@@ -0,0 +1,4 @@
-- Two-phase stalled-job recovery (mark on one pass, reclaim on the next);
-- returns the reclaimed ids. Params: $1 queue, $2 max_stalled_count,
-- $3 now_ms, $4 max_check_time_ms (stalledInterval).
SELECT id FROM move_stalled_jobs_to_wait($1, $2, $3, $4) AS t(id);

View File

@@ -0,0 +1,4 @@
-- Claim the next ready job for a worker (0 or 1 rows), honouring the limiter.
-- Params: $1 queue, $2 token, $3 lock_ms, $4 now_ms, $5 worker name,
-- $6 limiter max (worker option, NULL if none), $7 limiter duration ms.
SELECT * FROM move_to_active($1, $2, $3, $4, $5, $6, $7);

View File

@@ -0,0 +1,4 @@
-- Finish an active job successfully; returns the finished-at timestamp.
-- Params: $1 queue, $2 id, $3 token, $4 return_value (jsonb), $5 finished_on,
-- $6 remove_all, $7 keep_age (s), $8 keep_count.
SELECT move_to_completed($1, $2, $3, $4::jsonb, $5, $6, $7, $8) AS finished_on;

View File

@@ -0,0 +1,9 @@
-- Finish an active job successfully AND claim the next ready job in one
-- transaction (one commit) — the fused equivalent of move_to_completed followed
-- by move_to_active, mirroring Redis's moveToFinished. Returns 0 or 1 job rows.
-- Params: $1 queue, $2 id, $3 token, $4 return_value (jsonb), $5 finished_on,
-- $6 remove_all, $7 keep_age (s), $8 keep_count,
-- $9 lock_ms, $10 now_ms, $11 worker name,
-- $12 limiter max (worker option, NULL if none), $13 limiter duration ms.
SELECT * FROM move_to_completed_fetch(
$1, $2, $3, $4::jsonb, $5, $6, $7, $8, $9, $10, $11, $12, $13);

View File

@@ -0,0 +1,4 @@
-- Re-queue an active job to the delayed state (retry-with-delay / manual delay).
-- Params: $1 queue, $2 id, $3 token, $4 process_at, $5 delay,
-- $6 skip_attempt, $7 failed_reason, $8 stacktrace (jsonb).
SELECT move_to_delayed($1, $2, $3, $4, $5, $6, $7, $8::jsonb) AS n;

View File

@@ -0,0 +1,4 @@
-- Mark an active job failed; returns the finished-at timestamp.
-- Params: $1 queue, $2 id, $3 token, $4 failed_reason, $5 stacktrace (jsonb),
-- $6 finished_on, $7 remove_all, $8 keep_age (s), $9 keep_count.
SELECT move_to_failed($1, $2, $3, $4, $5::jsonb, $6, $7, $8, $9) AS finished_on;

View File

@@ -0,0 +1,9 @@
-- Fail (or retry) an active job AND claim the next ready job in one transaction
-- (one commit) — the fused equivalent of move_to_failed followed by
-- move_to_active, mirroring Redis's moveToFinished. Returns 0 or 1 job rows.
-- Params: $1 queue, $2 id, $3 token, $4 failed_reason, $5 stacktrace (jsonb),
-- $6 finished_on, $7 remove_all, $8 keep_age (s), $9 keep_count,
-- $10 lock_ms, $11 now_ms, $12 worker name,
-- $13 limiter max (worker option, NULL if none), $14 limiter duration ms.
SELECT * FROM move_to_failed_fetch(
$1, $2, $3, $4, $5::jsonb, $6, $7, $8, $9, $10, $11, $12, $13, $14);

View File

@@ -0,0 +1,3 @@
-- Move an active parent to waiting-children if it has pending children.
-- Params: $1 queue, $2 id, $3 token. Returns 1 (should wait), 0 (proceed).
SELECT move_to_waiting_children($1, $2, $3) AS code;

View File

@@ -0,0 +1,2 @@
-- The timestamp of the next delayed job, or NULL if there are none.
SELECT next_delay($1) AS next_delay;

View File

@@ -0,0 +1,3 @@
-- The worker "no job" signal: effective rate-limit ttl + next delayed-job time.
-- Params: $1 queue, $2 limiter max (worker option, NULL if none), $3 now_ms.
SELECT rate_limit_ttl, next_delay FROM next_signal($1, $2, $3);

View File

@@ -0,0 +1,4 @@
-- Obliterate a queue: delete up to $2 jobs; returns -1 (not paused), -2 (active
-- jobs without force), 1 (more to delete), or 0 (done). Params: $1 queue,
-- $2 count, $3 force.
SELECT obliterate($1, $2, $3) AS cursor;

View File

@@ -0,0 +1,19 @@
-- A parent job's child dependencies of a given status, sliced [offset, limit]
-- and joined to the child job rows (for fetchJobs). Params: $1 parent_queue,
-- $2 parent_id, $3 status ('pending' | 'processed'), $4 offset, $5 limit
-- (NULL = no limit). `total` is the full number of matching dependencies
-- (COUNT(*) OVER(), computed before the slice). `dep_value` is the stored
-- result for processed children. The remaining columns are the child
-- `job` row consumed by rowToJobJson.
SELECT
d.child_key,
d.value AS dep_value,
COUNT(*) OVER() AS total,
j.*
FROM job_dependency d
LEFT JOIN job j
ON j.queue = d.child_queue AND j.id = d.child_id
WHERE d.parent_queue = $1 AND d.parent_id = $2 AND d.status = $3::dep_status
ORDER BY d.child_key
OFFSET $4
LIMIT $5;

View File

@@ -0,0 +1,2 @@
-- Pause/resume the queue. Params: $1 queue, $2 paused.
SELECT pause($1, $2);

View File

@@ -0,0 +1,3 @@
-- Promote a delayed job to waiting. Params: $1 queue, $2 id.
-- Returns 0 ok, -1 missing, -3 not delayed.
SELECT promote($1, $2) AS code;

View File

@@ -0,0 +1,3 @@
-- Move up to `count` delayed jobs to waiting; returns the number moved.
-- Params: $1 queue, $2 count.
SELECT promote_jobs($1, $2) AS n;

View File

@@ -0,0 +1,3 @@
-- Append a custom event to the stream; returns the event id.
-- Params: $1 queue, $2 event, $3 data (jsonb).
SELECT publish_event($1, $2, $3::jsonb) AS id;

View File

@@ -0,0 +1,6 @@
-- A page of events newer than a cursor, oldest first.
-- Params: $1 queue, $2 cursor (exclusive), $3 limit.
SELECT id, event, data FROM event
WHERE queue = $1 AND id > $2
ORDER BY id ASC
LIMIT $3;

View File

@@ -0,0 +1,3 @@
-- The current max event id for a queue (used to resolve the '$' cursor).
-- Param: $1 queue.
SELECT COALESCE(MAX(id), 0)::bigint AS max FROM event WHERE queue = $1;

View File

@@ -0,0 +1,3 @@
-- Remove a job (and optionally its children). Returns 1 if removed, else 0.
-- Params: $1 queue, $2 id, $3 remove_children.
SELECT remove($1, $2, $3) AS n;

View File

@@ -0,0 +1,4 @@
-- Break a child's dependency link to its parent. Params: $1 queue, $2 job_id,
-- $3 parent_key, $4 now. Returns 0 (removed) or 1 (no relationship); raises on
-- missing job (-1) / missing parent (-5).
SELECT remove_child_dependency($1, $2, $3, $4) AS n;

View File

@@ -0,0 +1,9 @@
-- Conditionally remove a deduplication key, only when the given job is still
-- its (live) winner (mirrors removeDeduplicationKeyIfNeededOnRemoval and the
-- job-instance Job.removeDeduplicationKey, where Redis GET returns nil for an
-- expired key). Params: $1 queue, $2 dedup_id, $3 job_id, $4 now (ms). Returns
-- the removed key (0 or 1 row).
DELETE FROM dedup
WHERE queue = $1 AND dedup_id = $2 AND job_id = $3
AND (expire_at_ms IS NULL OR expire_at_ms > $4)
RETURNING dedup_id;

View File

@@ -0,0 +1,3 @@
-- Remove a scheduler and its still-pending job (emitting `removed` events);
-- returns 1 if the scheduler existed, 0 otherwise. Params: $1 queue, $2 id.
SELECT remove_job_scheduler($1, $2) AS removed;

View File

@@ -0,0 +1,2 @@
-- Remove queue metadata fields. Params: $1 queue, $2 fields (text[]).
DELETE FROM meta WHERE queue = $1 AND field = ANY($2::text[]);

View File

@@ -0,0 +1,6 @@
-- Clear the limiter window; returns the number of rows removed (0 or 1).
-- Param: $1 queue.
WITH d AS (
DELETE FROM rate_limit WHERE queue = $1 RETURNING 1
)
SELECT count(*)::int AS n FROM d;

View File

@@ -0,0 +1,3 @@
-- Recursively remove a parent's still-pending children. Params: $1 queue,
-- $2 job_id.
SELECT remove_unprocessed_children($1, $2);

View File

@@ -0,0 +1,4 @@
-- Re-queue a finished job (failed/completed) back to wait. Params: $1 queue,
-- $2 id, $3 state, $4 lifo, $5 reset_attempts_made, $6 reset_attempts_started.
-- Returns 1 ok, -1 missing, -3 not in the expected state.
SELECT reprocess_job($1, $2, $3, $4, $5, $6) AS code;

View File

@@ -0,0 +1,3 @@
-- Re-queue an active job back to waiting immediately (retry now).
-- Params: $1 queue, $2 id, $3 token, $4 lifo, $5 failed_reason, $6 stacktrace (jsonb).
SELECT retry_job($1, $2, $3, $4, $5, $6::jsonb) AS n;

View File

@@ -0,0 +1,3 @@
-- Move up to `count` finished jobs back to waiting; returns the number moved.
-- Params: $1 queue, $2 state, $3 count, $4 timestamp.
SELECT retry_jobs($1, $2, $3, $4) AS n;

View File

@@ -0,0 +1,4 @@
-- Upsert queue metadata fields. Params: $1 queue, $2 fields (text[]), $3 values (text[]).
INSERT INTO meta (queue, field, value)
SELECT $1, f, v FROM unnest($2::text[], $3::text[]) AS t(f, v)
ON CONFLICT (queue, field) DO UPDATE SET value = EXCLUDED.value;

View File

@@ -0,0 +1,3 @@
-- Force the limiter window (dynamic / manual rate limit).
-- Params: $1 queue, $2 expire ms, $3 now_ms.
SELECT set_rate_limit($1, $2, $3);

View File

@@ -0,0 +1,3 @@
-- Trim a job's oldest logs. Params: $1 queue, $2 job_id, $3 min_idx_to_keep.
DELETE FROM job_log
WHERE queue = $1 AND job_id = $2 AND idx < $3;

View File

@@ -0,0 +1,3 @@
-- Replace a job's data payload. Params: $1 queue, $2 id, $3 data (jsonb).
UPDATE job SET data = $3::jsonb WHERE queue = $1 AND id = $2
RETURNING id;

View File

@@ -0,0 +1,5 @@
-- Advance an existing scheduler to its next iteration (no template change);
-- returns the new delayed job id, or NULL if the scheduler is gone.
-- Params: $1 queue, $2 scheduler_id, $3 next_millis, $4 template_data (jsonb),
-- $5 delayed_opts (jsonb), $6 now, $7 producer_id.
SELECT update_job_scheduler_next_millis($1, $2, $3, $4, $5, $6, $7) AS job_id;

View File

@@ -0,0 +1,3 @@
-- Update a job's progress and emit a 'progress' event. Returns the number of
-- rows updated (0 = missing job). Params: $1 queue, $2 id, $3 progress (jsonb).
SELECT update_progress($1, $2, $3::jsonb) AS updated;

View File

@@ -0,0 +1,46 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.createPostgresBackend = void 0;
const postgres_connection_1 = require("./postgres-connection");
const postgres_queue_backend_1 = require("./postgres-queue-backend");
/**
* {@link BackendFactory} that builds a {@link PostgresQueueBackend}.
*
* The returned backend owns its {@link PostgresConnection} (a `pg.Pool` plus a
* dedicated `LISTEN` client); the high-level classes depend only on
* `IQueueBackend` and never touch a `pg` client directly.
*
* The `opts.connection` value is forwarded to {@link PostgresConnection} and may
* be a connection string, a node-postgres pool config (optionally carrying a
* `schema`), or an already-built `pg.Pool` instance. `pg` is lazily required
* only when a config/string is passed, so Redis-only users never need it
* installed.
*
* Inject this into the queue classes (or set it as the process-wide default via
* `setDefaultBackendFactory(createPostgresBackend)`) to back BullMQ with
* PostgreSQL.
*/
const createPostgresBackend = (name, opts, factoryOpts = {}) => {
const connection = new postgres_connection_1.PostgresConnection(opts.connection);
// Name a backend's dedicated, long-lived connection so it is discoverable via
// getWorkers / getQueueEvents (pg_stat_activity) — the PostgreSQL analogue of
// the Redis worker/queue-events named connection. Naming happens eagerly at
// waitUntilReady (see PostgresQueueBackend) so the name is set as soon as the
// backend is ready, exactly like Redis names its connection on creation.
// - Worker (withBlockingConnection): the bare queue name, or `:w:<name>`
// for a named worker — matching the getWorkers matcher.
// - QueueEvents (blocking): the queue name + `:qe` (QUEUE_EVENT_SUFFIX),
// matching the getQueueEvents matcher. (QueueEvents also re-applies this
// via setName when its consume loop starts.)
// Producers/FlowProducer get no name (they must not be counted as workers).
const workerName = opts.name;
let listenClientName;
if (factoryOpts.withBlockingConnection) {
listenClientName = `${name}${workerName ? `:w:${workerName}` : ''}`;
}
else if (factoryOpts.blocking) {
listenClientName = `${name}:qe`;
}
return new postgres_queue_backend_1.PostgresQueueBackend(connection, name, opts, true, listenClientName);
};
exports.createPostgresBackend = createPostgresBackend;

23
node_modules/bullmq/dist/cjs/postgres/index.js generated vendored Normal file
View File

@@ -0,0 +1,23 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.isPgPool = exports.LATEST_SCHEMA_VERSION = exports.quoteSchemaName = exports.DEFAULT_SCHEMA = exports.MIGRATION_ADVISORY_LOCK_KEY = exports.RECOMMENDED_POSTGRES_VERSION = exports.MINIMUM_POSTGRES_VERSION = exports.assertPostgresVersion = exports.UnsupportedPostgresVersionError = exports.SchemaVersionMismatchError = exports.runMigrations = exports.PostgresQueueBackend = exports.PostgresConnection = exports.createPostgresBackend = void 0;
var create_postgres_backend_1 = require("./create-postgres-backend");
Object.defineProperty(exports, "createPostgresBackend", { enumerable: true, get: function () { return create_postgres_backend_1.createPostgresBackend; } });
var postgres_connection_1 = require("./postgres-connection");
Object.defineProperty(exports, "PostgresConnection", { enumerable: true, get: function () { return postgres_connection_1.PostgresConnection; } });
var postgres_queue_backend_1 = require("./postgres-queue-backend");
Object.defineProperty(exports, "PostgresQueueBackend", { enumerable: true, get: function () { return postgres_queue_backend_1.PostgresQueueBackend; } });
var migrator_1 = require("./migrator");
Object.defineProperty(exports, "runMigrations", { enumerable: true, get: function () { return migrator_1.runMigrations; } });
Object.defineProperty(exports, "SchemaVersionMismatchError", { enumerable: true, get: function () { return migrator_1.SchemaVersionMismatchError; } });
Object.defineProperty(exports, "UnsupportedPostgresVersionError", { enumerable: true, get: function () { return migrator_1.UnsupportedPostgresVersionError; } });
Object.defineProperty(exports, "assertPostgresVersion", { enumerable: true, get: function () { return migrator_1.assertPostgresVersion; } });
Object.defineProperty(exports, "MINIMUM_POSTGRES_VERSION", { enumerable: true, get: function () { return migrator_1.MINIMUM_POSTGRES_VERSION; } });
Object.defineProperty(exports, "RECOMMENDED_POSTGRES_VERSION", { enumerable: true, get: function () { return migrator_1.RECOMMENDED_POSTGRES_VERSION; } });
Object.defineProperty(exports, "MIGRATION_ADVISORY_LOCK_KEY", { enumerable: true, get: function () { return migrator_1.MIGRATION_ADVISORY_LOCK_KEY; } });
Object.defineProperty(exports, "DEFAULT_SCHEMA", { enumerable: true, get: function () { return migrator_1.DEFAULT_SCHEMA; } });
Object.defineProperty(exports, "quoteSchemaName", { enumerable: true, get: function () { return migrator_1.quoteSchemaName; } });
var migrations_1 = require("./migrations");
Object.defineProperty(exports, "LATEST_SCHEMA_VERSION", { enumerable: true, get: function () { return migrations_1.LATEST_SCHEMA_VERSION; } });
var pg_types_1 = require("./pg-types");
Object.defineProperty(exports, "isPgPool", { enumerable: true, get: function () { return pg_types_1.isPgPool; } });

View File

@@ -0,0 +1,348 @@
-- BullMQ PostgreSQL backend — schema (types, sequences, tables, indexes).
--
-- Consolidated initial schema. All BullMQ objects live in a connection-level
-- schema (namespace); the operation functions live in 0002_functions.sql.
-- BullMQ PostgreSQL backend — initial schema (schema version 1).
--
-- This file is the *portable source of truth* for the schema. It uses only
-- standard SQL / PL-pgSQL so it can be shared verbatim with the future Elixir
-- and Python ports (which call the very same tables and functions).
--
-- All objects are created inside the backend's configured *schema* (the
-- connection-level namespace, default `bullmq`). The migration runner has
-- already created the schema and set `search_path` to it, so the unqualified
-- names below resolve into that schema. Unlike Redis — where a per-queue key
-- `prefix` namespaces every key — SQL uses the schema as the single namespace
-- for the whole connection, so there is no per-row/per-queue prefix.
--
-- The migration ledger table (`migration`) is bootstrapped by the
-- migration runner itself, so it is intentionally not created here.
--
-- v1 only establishes the foundation that is independent of the job-storage
-- model: per-queue metadata. Subsequent migrations add the job tables,
-- indexes and the atomic operation functions.
-- Per-queue metadata. Mirrors the Redis `<prefix>:<queue>:meta` hash: a small
-- key/value store keyed by (queue, field). Used for the queue version, global
-- concurrency, global rate limit, paused flag, etc.
CREATE TABLE IF NOT EXISTS meta (
queue text NOT NULL,
field text NOT NULL,
value text,
PRIMARY KEY (queue, field)
);
-- BullMQ PostgreSQL backend — core job schema (schema version 2).
--
-- Portable source of truth (standard SQL / PL-pgSQL) shared verbatim with the
-- future Elixir and Python ports. This migration establishes the *storage
-- model* (tables + indexes) for every BullMQ feature: FIFO/LIFO, priority,
-- delayed jobs, concurrency, locks/stalled detection, flows (parent/child
-- dependencies), deduplication, rate limiting, repeatable job schedulers, the
-- event stream and metrics. The atomic *operation* functions (addJob,
-- moveToActive, …) are layered on top in subsequent migrations.
--
-- ──────────────────────────────────────────────────────────────────────────
-- Design overview
-- ──────────────────────────────────────────────────────────────────────────
-- * Namespace = the connection's PostgreSQL *schema* (default `bullmq`), not a
-- per-queue key prefix. In Redis a `prefix` namespaces every key because the
-- keyspace is shared; in SQL the schema already isolates BullMQ from the
-- user's other tables, and a different schema (or database) gives a fully
-- independent BullMQ namespace. So there is no per-row/per-queue prefix — the
-- `queue` column alone discriminates within the schema.
--
-- * Single `job` table keyed by (queue, id). The Redis adapter spreads a
-- job's lifecycle across many keys (wait list, prioritized zset, delayed zset,
-- active list, completed/failed zsets, locks, …); here a single `state`
-- column plus a handful of promoted, indexable columns replaces all of them.
-- State transitions are plain `UPDATE`s; claiming uses `FOR UPDATE SKIP
-- LOCKED` so concurrent workers never block each other.
--
-- * Time is stored as `bigint` epoch-milliseconds (suffix `_ms`) to match
-- BullMQ's millisecond API exactly and avoid timezone math on hot paths.
--
-- * Partial indexes are state-scoped: each hot path (claim next ready job,
-- promote due delayed jobs, find stalled active jobs, clean finished jobs)
-- has a small index that only contains the rows in the relevant state.
--
-- * "paused" and "prioritized" are NOT physical states. Pausing is an O(1)
-- queue-level flag in `meta`; a prioritized job is simply a waiting job
-- with `priority > 0`. This avoids the bulk row rewrites Redis performs on
-- pause and removes the need for a separate prioritized structure — full
-- `ORDER BY (priority, seq)` does the job.
-- ──────────────────────────────────────────────────────────────────────────
-- Physical job states. (Pause = meta flag; prioritized = waiting + priority>0.)
CREATE TYPE job_state AS ENUM (
'waiting',
'active',
'completed',
'failed',
'delayed',
'waiting-children'
);
-- Status of a parent→child dependency (flows).
CREATE TYPE dep_status AS ENUM (
'pending', -- child not finished yet
'processed', -- child completed; `value` holds its return value
'ignored', -- child failed but parent was told to ignore it; `value` = reason
'failed' -- child failed and the failure is recorded against the parent
);
-- Global monotonic ordering for FIFO. A single sequence gives a total order
-- across all queues; FIFO within a queue is preserved because `seq` increases
-- with insertion. LIFO jobs are stored with a *negative* `seq` (from the same
-- sequence, negated) so they sort ahead of all FIFO jobs and, among themselves,
-- most-recently-added first — reproducing "last in, first out" with one column.
CREATE SEQUENCE job_seq;
-- Global monotonic id for the event stream (the Postgres analogue of a Redis
-- stream entry id). Consumers page forward with `id > cursor`.
CREATE SEQUENCE event_seq;
-- ──────────────────────────────────────────────────────────────────────────
-- Jobs
-- ──────────────────────────────────────────────────────────────────────────
CREATE TABLE job (
queue text NOT NULL,
id text NOT NULL, -- custom id, or numeric counter as text
seq bigint NOT NULL, -- FIFO order (negative = LIFO); see sequence above
name text NOT NULL,
state job_state NOT NULL,
-- Payload / options. BullMQ serializes these as JSON strings; stored as jsonb
-- (always valid JSON) so they are queryable and compact.
data jsonb NOT NULL DEFAULT '{}'::jsonb,
opts jsonb NOT NULL DEFAULT '{}'::jsonb,
-- Promoted, frequently-read scalars (also live implicitly in `opts`, but are
-- broken out so they can be indexed / updated cheaply).
priority integer NOT NULL DEFAULT 0, -- 0 = no priority (FIFO); >0 = prioritized
delay_ms bigint NOT NULL DEFAULT 0,
max_attempts integer NOT NULL DEFAULT 1,
attempts_made integer NOT NULL DEFAULT 0,
attempts_started integer NOT NULL DEFAULT 0,
-- Progress / results.
progress jsonb, -- number or object
return_value jsonb, -- BullMQ `returnvalue`
failed_reason text,
stacktrace jsonb, -- array of frames
deferred_failure text,
processed_by text, -- worker id that processed it
-- Timestamps (epoch ms).
added_at_ms bigint NOT NULL, -- BullMQ `timestamp`
process_at_ms bigint, -- when a delayed job becomes ready
processed_at_ms bigint, -- BullMQ `processedOn`
finished_at_ms bigint, -- BullMQ `finishedOn`
-- Locking / stalled detection. An active job is "locked" until
-- `locked_until_ms`; once it elapses the job is considered stalled.
lock_token text,
locked_until_ms bigint,
stalled_count integer NOT NULL DEFAULT 0,
-- Deduplication / scheduler linkage.
dedup_id text, -- BullMQ deduplicationId / debounceId
scheduler_id text, -- id of the scheduler that produced this job (BullMQ repeatJobKey)
-- Flow (parent/child) linkage. A child stores its parent's coordinates; a
-- parent tracks how many children are still unfinished.
parent_queue text,
parent_id text,
parent_key text, -- denormalized "<queue>:<id>" (Redis-compatible "<prefix>:<queue>:<id>")
pending_deps integer NOT NULL DEFAULT 0,
-- Two-phase stalled mark/sweep: set on the first stalled pass, reclaimed on
-- the next (see move_stalled_jobs_to_wait in 0002_functions.sql).
stalled_marked boolean NOT NULL DEFAULT false,
PRIMARY KEY (queue, id)
);
-- Claim next ready job: waiting jobs ordered by (priority, seq). Covers both the
-- "wait" (priority = 0) and "prioritized" (priority > 0) views, and the FIFO/LIFO
-- ordering via the signed `seq`. Used with FOR UPDATE SKIP LOCKED.
CREATE INDEX job_ready_idx
ON job (queue, priority, seq)
WHERE state = 'waiting';
-- Promote due delayed jobs (process_at_ms <= now) and find the next wake-up.
CREATE INDEX job_delayed_idx
ON job (queue, process_at_ms)
WHERE state = 'delayed';
-- Active jobs: stalled scan by lock expiry, and O(1)-ish concurrency counting
-- (COUNT over the partial index for a queue).
CREATE INDEX job_active_idx
ON job (queue, locked_until_ms)
WHERE state = 'active';
-- Finished jobs: range listing and age/count-based retention + cleaning.
CREATE INDEX job_finished_idx
ON job (queue, state, finished_at_ms)
WHERE state IN ('completed', 'failed');
-- Parents blocked on children.
CREATE INDEX job_waiting_children_idx
ON job (queue, seq)
WHERE state = 'waiting-children';
-- Reverse lookup from a child to its parent (e.g. orphan/repair scans).
CREATE INDEX job_parent_idx
ON job (parent_queue, parent_id)
WHERE parent_id IS NOT NULL;
-- Locate the job(s) produced by a given scheduler.
CREATE INDEX job_scheduler_idx
ON job (queue, scheduler_id)
WHERE scheduler_id IS NOT NULL;
-- ──────────────────────────────────────────────────────────────────────────
-- Job logs (append-only, per-job ordered lines)
-- ──────────────────────────────────────────────────────────────────────────
CREATE TABLE job_log (
queue text NOT NULL,
job_id text NOT NULL,
idx bigint NOT NULL, -- per-job ordinal (0-based); range reads + trimming
row text NOT NULL,
PRIMARY KEY (queue, job_id, idx),
FOREIGN KEY (queue, job_id)
REFERENCES job (queue, id) ON DELETE CASCADE
);
-- ──────────────────────────────────────────────────────────────────────────
-- Flow dependencies (parent → child)
-- ──────────────────────────────────────────────────────────────────────────
-- Owned by the parent. Reproduces BullMQ's processed/ignored/failed/unprocessed
-- child sets with a single `status` column, and stores each processed child's
-- return value (or failure reason) in `value`. `pending_deps` on the parent job
-- is the count of rows here still in 'pending'.
CREATE TABLE job_dependency (
parent_queue text NOT NULL,
parent_id text NOT NULL,
child_queue text NOT NULL,
child_id text NOT NULL,
child_key text NOT NULL, -- denormalized "<queue>:<id>" (Redis-compatible "<prefix>:<queue>:<id>")
status dep_status NOT NULL DEFAULT 'pending',
value jsonb,
PRIMARY KEY (parent_queue, parent_id, child_key),
FOREIGN KEY (parent_queue, parent_id)
REFERENCES job (queue, id) ON DELETE CASCADE
);
-- Paginate a parent's children by category (processed / pending / …).
CREATE INDEX dep_parent_status_idx
ON job_dependency (parent_queue, parent_id, status);
-- Resolve dependencies from the child side (when a child finishes, or when a
-- child dependency is explicitly removed).
CREATE INDEX dep_child_idx
ON job_dependency (child_queue, child_id);
-- ──────────────────────────────────────────────────────────────────────────
-- Event stream (QueueEvents)
-- ──────────────────────────────────────────────────────────────────────────
-- The Postgres analogue of the Redis events stream. `id` is globally monotonic;
-- consumers page forward with `id > cursor`. Blocking reads are delivered via
-- LISTEN/NOTIFY (the operation layer NOTIFYs on insert); trimming deletes the
-- lowest ids beyond the configured max length.
CREATE TABLE event (
queue text NOT NULL,
id bigint NOT NULL DEFAULT nextval('event_seq'),
event text NOT NULL,
data jsonb NOT NULL DEFAULT '{}'::jsonb,
created_at_ms bigint NOT NULL,
PRIMARY KEY (queue, id)
);
CREATE TABLE metrics (
queue text NOT NULL,
kind text NOT NULL, -- 'completed' | 'failed'
count bigint NOT NULL DEFAULT 0, -- cumulative finished jobs
prev_ts bigint, -- ts of the last data point
prev_count bigint NOT NULL DEFAULT 0, -- count at the last data point
data bigint[] NOT NULL DEFAULT '{}',-- per-minute deltas, newest first
PRIMARY KEY (queue, kind)
);
-- ──────────────────────────────────────────────────────────────────────────
-- Rate limiting (token window per queue)
-- ──────────────────────────────────────────────────────────────────────────
-- One row per queue holding the current limiter window: `points` counts the
-- jobs consumed in the window, which ends at `expire_at_ms`. Mirrors the single
-- Redis `<prefix>:<queue>:limiter` counter key (with its PTTL). The global
-- limiter is the only mode the open-source backend supports (per-group rate
-- limiting was removed in BullMQ 3.0).
CREATE TABLE rate_limit (
queue text NOT NULL,
points bigint NOT NULL DEFAULT 0,
expire_at_ms bigint NOT NULL,
PRIMARY KEY (queue)
);
-- ──────────────────────────────────────────────────────────────────────────
-- Deduplication keys
-- ──────────────────────────────────────────────────────────────────────────
CREATE TABLE dedup (
queue text NOT NULL,
dedup_id text NOT NULL,
job_id text NOT NULL,
expire_at_ms bigint, -- NULL = no expiry; otherwise lazily reclaimed
PRIMARY KEY (queue, dedup_id)
);
-- Expired dedup keys are reclaimed lazily by operation functions in
-- 0002_functions.sql (for example deduplicate_job and
-- dedup_finalize). This index keeps those lookups/deletes efficient.
CREATE INDEX dedup_expire_idx
ON dedup (queue, expire_at_ms)
WHERE expire_at_ms IS NOT NULL;
-- ──────────────────────────────────────────────────────────────────────────
-- Job schedulers (repeatable jobs)
--
-- A scheduler is a job factory: it stores a template (data/opts) plus a repeat
-- spec (cron `pattern` or fixed `every` ms) and, on each upsert, produces the
-- next delayed job `repeat:<schedulerId>:<nextMillis>`. For cron the caller
-- computes nextMillis (JS cron-parser); for `every` the backend computes it,
-- honouring `offset_ms` (the phase offset from `start_date`).
-- ──────────────────────────────────────────────────────────────────────────
CREATE TABLE scheduler (
queue text NOT NULL,
scheduler_id text NOT NULL,
name text,
next_run_ms bigint, -- next iteration's due time
pattern text, -- cron expression (mutually exclusive with every_ms)
every_ms bigint, -- fixed interval in ms
tz text, -- timezone for cron evaluation
start_date_ms bigint,
end_date_ms bigint,
limit_count integer, -- max number of iterations (NULL = unlimited)
iteration_count integer NOT NULL DEFAULT 0,
template_data jsonb, -- payload template reused for every iteration
template_opts jsonb, -- options template reused for every iteration
producer_id text,
offset_ms bigint, -- 'every' phase offset from start_date
PRIMARY KEY (queue, scheduler_id)
);
-- Find schedulers whose next iteration is due, and order schedulers by next run.
CREATE INDEX scheduler_next_idx
ON scheduler (queue, next_run_ms);
-- ── keepLastIfActive proto-next storage ──────────────────────────────────
-- When a job is deduplicated while its winner is *active* and keepLastIfActive
-- is set, the new job's payload is stashed here (Redis `dn:<id>` hash) and the
-- dedup key is persisted (no expiry). When the active winner finishes, the
-- stashed payload is turned into a real job (the new winner). At most one
-- proto-next exists per id; a later add while active overwrites it.
CREATE TABLE dedup_next (
queue text NOT NULL,
dedup_id text NOT NULL,
payload jsonb NOT NULL, -- { name, data, opts, jobId }
PRIMARY KEY (queue, dedup_id)
);

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,26 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.LATEST_SCHEMA_VERSION = exports.MIGRATIONS = void 0;
const sql_loader_1 = require("../sql-loader");
/**
* The ordered list of migrations bundled with this version of BullMQ. Append a
* new entry (never edit or reorder existing ones) whenever the schema changes.
*/
exports.MIGRATIONS = [
{
version: 1,
name: '0001_schema',
load: () => (0, sql_loader_1.loadMigrationSql)('0001_schema.sql'),
},
{
version: 2,
name: '0002_functions',
load: () => (0, sql_loader_1.loadMigrationSql)('0002_functions.sql'),
},
];
/**
* The highest schema version this BullMQ build knows how to produce. Compared
* against the version recorded in the database to decide whether to migrate
* (database older), no-op (equal), or refuse to run (database newer).
*/
exports.LATEST_SCHEMA_VERSION = exports.MIGRATIONS.length > 0 ? exports.MIGRATIONS[exports.MIGRATIONS.length - 1].version : 0;

232
node_modules/bullmq/dist/cjs/postgres/migrator.js generated vendored Normal file
View File

@@ -0,0 +1,232 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.SchemaVersionMismatchError = exports.UnsupportedPostgresVersionError = exports.RECOMMENDED_POSTGRES_VERSION = exports.MINIMUM_POSTGRES_VERSION = exports.MIGRATION_ADVISORY_LOCK_KEY = exports.DEFAULT_SCHEMA = void 0;
exports.assertPostgresVersion = assertPostgresVersion;
exports.quoteSchemaName = quoteSchemaName;
exports.runMigrations = runMigrations;
exports.getCurrentSchemaVersion = getCurrentSchemaVersion;
const migrations_1 = require("./migrations");
/**
* Default PostgreSQL schema (namespace) the backend lives in. The schema is the
* connection-level namespace for *all* queues — the SQL-native replacement for
* Redis's per-queue key `prefix`.
*/
exports.DEFAULT_SCHEMA = 'bullmq';
/**
* Stable key for the transaction-scoped advisory lock that serializes
* migrations across processes. The integer spells `BULL` (0x42554c4c) and is
* documented here so other runtimes (the Elixir/Python ports) use the exact
* same lock. The lock is namespaced per schema (via `hashtext(schema)`) so
* migrating one namespace never blocks another.
*/
exports.MIGRATION_ADVISORY_LOCK_KEY = 0x42554c4c; // 1112493644
/**
* Lowest PostgreSQL *major* version the backend supports. Below this the schema
* and operation functions rely on features (or fixes) that are absent or
* unreliable, so we refuse to run rather than fail later in a surprising way.
*
* Rationale: the operation functions use `INSERT … ON CONFLICT`, transaction
* advisory locks, `to_regclass`, multi-array `unnest(…) WITH ORDINALITY`, and
* lean on the read-write "expanded array" representation for cheap in-loop
* accumulation — all comfortably available here, on a version that is still
* within the PostgreSQL support window.
*/
exports.MINIMUM_POSTGRES_VERSION = 13;
/**
* Recommended lowest PostgreSQL *major* version. Between {@link
* MINIMUM_POSTGRES_VERSION} and this we still run, but emit a one-time warning
* (mirrors the Redis backend's `recommendedMinimumVersion`).
*/
exports.RECOMMENDED_POSTGRES_VERSION = 14;
/**
* Thrown when the connected PostgreSQL server is older than {@link
* MINIMUM_POSTGRES_VERSION}. Pass `skipVersionCheck: true` on the connection to
* bypass the check (at your own risk).
*/
class UnsupportedPostgresVersionError extends Error {
constructor(serverVersion, minimumVersion) {
super(`BullMQ: the PostgreSQL backend requires server version ` +
`${minimumVersion} or newer, but the server reports ${serverVersion}. ` +
`Upgrade PostgreSQL, or pass \`skipVersionCheck: true\` on the ` +
`connection to bypass this check at your own risk.`);
this.serverVersion = serverVersion;
this.minimumVersion = minimumVersion;
this.name = 'UnsupportedPostgresVersionError';
}
}
exports.UnsupportedPostgresVersionError = UnsupportedPostgresVersionError;
/**
* Verifies the connected server meets {@link MINIMUM_POSTGRES_VERSION} (throws
* an {@link UnsupportedPostgresVersionError} otherwise) and warns once when it
* is below {@link RECOMMENDED_POSTGRES_VERSION}. No-op when `skipVersionCheck`
* is set. Uses `server_version_num` (e.g. `160002` for 16.2), whose integer
* major component is `num / 10000` for every supported release.
*/
async function assertPostgresVersion(client, skipVersionCheck = false) {
var _a, _b, _c, _d;
if (skipVersionCheck) {
return;
}
const { rows } = await client.query(`SELECT current_setting('server_version_num') AS num, ` +
`current_setting('server_version') AS ver`);
const major = Math.floor(parseInt((_b = (_a = rows[0]) === null || _a === void 0 ? void 0 : _a.num) !== null && _b !== void 0 ? _b : '0', 10) / 10000);
const reported = (_d = (_c = rows[0]) === null || _c === void 0 ? void 0 : _c.ver) !== null && _d !== void 0 ? _d : 'unknown';
if (major < exports.MINIMUM_POSTGRES_VERSION) {
throw new UnsupportedPostgresVersionError(reported, exports.MINIMUM_POSTGRES_VERSION);
}
if (major < exports.RECOMMENDED_POSTGRES_VERSION) {
const warned = assertPostgresVersion._warnedRecommendedVersion;
if (!warned) {
assertPostgresVersion._warnedRecommendedVersion = true;
console.warn(`BullMQ: PostgreSQL ${exports.RECOMMENDED_POSTGRES_VERSION} or newer is ` +
`recommended for the PostgreSQL backend (detected ${reported}).`);
}
}
}
/**
* Validates a PostgreSQL schema name and returns it double-quoted for safe
* interpolation into DDL (schema names cannot be passed as bind parameters).
*
* Only simple identifiers are allowed (letter/underscore start, then
* letters/digits/underscores/`$`, max 63 bytes), which both keeps the value
* injection-safe and avoids surprising case-folding / quoting edge cases.
*/
function quoteSchemaName(schema) {
if (!/^[A-Za-z_][A-Za-z0-9_$]*$/.test(schema) || schema.length > 63) {
throw new Error(`BullMQ: invalid PostgreSQL schema name ${JSON.stringify(schema)}. ` +
'Use a simple identifier (letters, digits, underscores; max 63 chars).');
}
return `"${schema}"`;
}
/**
* Thrown when the database schema is newer than this BullMQ build supports.
*
* This happens when the schema was migrated by a newer BullMQ release and an
* older instance then connects: the older code may not understand the newer
* structures, so we refuse to operate rather than risk corruption. The fix is
* to upgrade BullMQ — schema downgrades are not supported.
*/
class SchemaVersionMismatchError extends Error {
constructor(databaseVersion, supportedVersion) {
super(`BullMQ: the PostgreSQL schema is at version ${databaseVersion}, but this ` +
`version of BullMQ only supports schema versions up to ${supportedVersion}. ` +
`The database was migrated by a newer BullMQ release; upgrade BullMQ to ` +
`continue (schema downgrades are not supported).`);
this.databaseVersion = databaseVersion;
this.supportedVersion = supportedVersion;
this.name = 'SchemaVersionMismatchError';
}
}
exports.SchemaVersionMismatchError = SchemaVersionMismatchError;
/**
* Brings the database schema up to {@link LATEST_SCHEMA_VERSION}.
*
* Run on the backend's first `waitUntilReady()` (a constructor cannot perform
* async I/O). Behaviour by current database version:
*
* - **older** than supported → applies the pending migrations in order.
* - **equal** to supported → no-op.
* - **newer** than supported → throws {@link SchemaVersionMismatchError}.
*
* ## Atomicity
*
* The whole operation runs inside a **single transaction**: every pending
* migration's SQL and its ledger row are committed together, or nothing is. If
* any migration fails the transaction is rolled back and the database is left
* exactly at its previous schema version — there are no partially-applied
* upgrades.
*
* For this guarantee to hold, migration `.sql` files must contain only
* transaction-safe statements. PostgreSQL DDL (`CREATE TABLE`/`FUNCTION`/`INDEX`,
* `ALTER …`, …) is transactional, but a few commands are not and must never be
* used in a migration: `CREATE INDEX CONCURRENTLY`, `VACUUM`, `CREATE DATABASE`,
* etc.
*
* ## Isolation
*
* A transaction-scoped `pg_advisory_xact_lock` serializes concurrent starters
* (many Queue/Worker instances booting at once across processes), so the
* migrations run exactly once. The lock is acquired as the first statement and
* released automatically when the transaction commits or rolls back (or if the
* connection dies), so it can never leak. Late starters block until the winner
* commits, then observe the up-to-date version and no-op.
*
* ## Namespace
*
* All objects are created in `schema` (default {@link DEFAULT_SCHEMA}), the
* connection-level namespace that replaces Redis's per-queue key prefix. The
* schema is created if missing and `search_path` is set (transaction-locally)
* so the migrations' unqualified table names resolve into it.
*
* The caller MUST provide a single dedicated session (e.g. a checked-out
* `pg.PoolClient` or a standalone `pg.Client`), never the pool itself, so the
* lock and the transaction share one connection.
*
* @returns the schema version the database is at after the call.
*/
async function runMigrations(client, schema = exports.DEFAULT_SCHEMA, options = {}) {
const quotedSchema = quoteSchemaName(schema);
// Fail fast on an unsupported server *before* opening the migration
// transaction, so an old server surfaces a clear error rather than a cryptic
// syntax/feature failure partway through applying DDL.
await assertPostgresVersion(client, options.skipVersionCheck);
await client.query('BEGIN');
try {
// Isolation: serialize concurrent migrators for THIS schema. The lock is
// held for the lifetime of this transaction and released automatically on
// COMMIT/ROLLBACK. Namespaced by schema so different namespaces don't block.
await client.query('SELECT pg_advisory_xact_lock($1, hashtext($2))', [
exports.MIGRATION_ADVISORY_LOCK_KEY,
schema,
]);
// Create the namespace and point unqualified names at it for the rest of
// the transaction (SET LOCAL is reverted on COMMIT/ROLLBACK).
await client.query(`CREATE SCHEMA IF NOT EXISTS ${quotedSchema}`);
await client.query(`SET LOCAL search_path TO ${quotedSchema}`);
await ensureLedgerTable(client);
const currentVersion = await getCurrentSchemaVersion(client);
if (currentVersion > migrations_1.LATEST_SCHEMA_VERSION) {
// Rolled back by the catch below; nothing has been written anyway.
throw new SchemaVersionMismatchError(currentVersion, migrations_1.LATEST_SCHEMA_VERSION);
}
if (currentVersion < migrations_1.LATEST_SCHEMA_VERSION) {
for (const migration of migrations_1.MIGRATIONS) {
if (migration.version > currentVersion) {
await applyMigration(client, migration);
}
}
}
// Atomicity: every migration applied above and its ledger row commit
// together as one unit.
await client.query('COMMIT');
return Math.max(currentVersion, migrations_1.LATEST_SCHEMA_VERSION);
}
catch (err) {
await client.query('ROLLBACK');
throw err;
}
}
/**
* Reads the schema version currently recorded in the database, or 0 for a fresh
* database. Assumes the ledger table already exists (see
* {@link ensureLedgerTable}).
*/
async function getCurrentSchemaVersion(client) {
var _a, _b;
const { rows } = await client.query('SELECT COALESCE(MAX(version), 0)::int AS version FROM migration');
return (_b = (_a = rows[0]) === null || _a === void 0 ? void 0 : _a.version) !== null && _b !== void 0 ? _b : 0;
}
async function ensureLedgerTable(client) {
await client.query(`CREATE TABLE IF NOT EXISTS migration (
version integer PRIMARY KEY,
name text NOT NULL,
applied_at timestamptz NOT NULL DEFAULT now()
)`);
}
async function applyMigration(client, migration) {
await client.query(migration.load());
await client.query('INSERT INTO migration (version, name) VALUES ($1, $2)', [
migration.version,
migration.name,
]);
}

14
node_modules/bullmq/dist/cjs/postgres/pg-types.js generated vendored Normal file
View File

@@ -0,0 +1,14 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.isPgPool = isPgPool;
/**
* Narrows whether a user-provided connection value is already an instantiated
* `pg.Pool` (as opposed to a config object / connection string that we must use
* to construct one).
*/
function isPgPool(value) {
return (!!value &&
typeof value.connect === 'function' &&
typeof value.query === 'function' &&
typeof value.end === 'function');
}

View File

@@ -0,0 +1,211 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.PostgresConnection = void 0;
const tslib_1 = require("tslib");
const events_1 = require("events");
const pg_types_1 = require("./pg-types");
const migrator_1 = require("./migrator");
/**
* Lazily loads the optional `pg` (node-postgres) driver. Redis-only users never
* hit this path, so they never need `pg` installed.
*
* Only reached when the caller passes a config/connection string (not an
* already-constructed `pg.Pool`). In native ESM environments where `require` is
* unavailable, callers should pass a `pg.Pool` instance instead.
*/
function loadPgModule() {
try {
if (typeof require === 'function') {
return require('pg');
}
}
catch (_a) {
// Fall through to the friendly error below.
}
throw new Error("The PostgreSQL backend could not load the optional 'pg' package. " +
'Install it with `npm install pg`. In a native ESM environment, pass an ' +
'already-constructed `pg.Pool` instance as the connection instead of a ' +
'config object or connection string.');
}
/**
* Owns the PostgreSQL connection resources for a single backend:
*
* - a `pg.Pool` for regular, short-lived queries, and
* - a dedicated, long-lived `LISTEN` client used by the blocking
* "wait for job" primitive (lazily established).
*
* Lifecycle mirrors {@link RedisConnection}: it is an {@link EventEmitter} that
* surfaces normalized `'ready' | 'error' | 'close'` events, exposes
* {@link PostgresConnection.waitUntilReady} (which also runs the schema
* migrations exactly once, on a dedicated checked-out client) and
* {@link PostgresConnection.close}.
*/
class PostgresConnection extends events_1.EventEmitter {
constructor(connection) {
super();
/**
* `true` when {@link listenClient} is a standalone `pg.Client` we must `end()`
* (owned pool); `false` when it is a pooled client we must `release()`.
*/
this.listenClientIsStandalone = false;
if ((0, pg_types_1.isPgPool)(connection)) {
this.pool = connection;
this.ownsPool = false;
this.schema = migrator_1.DEFAULT_SCHEMA;
this.skipVersionCheck = false;
this.pgModule = undefined;
this.listenClientConfig = undefined;
}
else {
const pg = loadPgModule();
const _a = typeof connection === 'string'
? {
schema: undefined,
skipVersionCheck: undefined,
connectionString: connection,
}
: connection, { schema, skipVersionCheck } = _a, poolConfig = tslib_1.__rest(_a, ["schema", "skipVersionCheck"]);
this.schema = schema !== null && schema !== void 0 ? schema : migrator_1.DEFAULT_SCHEMA;
this.skipVersionCheck = skipVersionCheck !== null && skipVersionCheck !== void 0 ? skipVersionCheck : false;
// Validate early so a bad schema name fails fast (and before any DDL).
const quotedSchema = (0, migrator_1.quoteSchemaName)(this.schema);
// Pin every pooled connection's search_path to the schema so the `.sql`
// command files use unqualified, portable names. Quoted to match the
// migration's quoted CREATE SCHEMA (case-preserving).
const searchPathOption = `-c search_path=${quotedSchema}`;
const existingOptions = poolConfig.options;
const resolvedConfig = Object.assign(Object.assign({}, poolConfig), { options: existingOptions
? `${existingOptions} ${searchPathOption}`
: searchPathOption });
this.pool = new pg.Pool(resolvedConfig);
this.ownsPool = true;
// Keep the means to build a dedicated LISTEN connection on demand.
this.pgModule = pg;
this.listenClientConfig = resolvedConfig;
}
// The pool emits 'error' for idle clients that drop; surface it but never
// let it crash the process — hence the guarded {@link emitError} (a bare
// `emit('error')` with no listeners throws).
this.pool.on('error', err => this.emitError(err));
}
/**
* Forwards an underlying pool / LISTEN-client error as this connection's
* `'error'` event, but only when a listener is attached. `EventEmitter.emit`
* throws when emitting `'error'` with no listeners, so an unguarded forward
* would turn an idle-client error into a hard process crash. Mirrors the
* guard in {@link RedisConnection}.
*/
emitError(err) {
if (this.listenerCount('error') > 0) {
this.emit('error', err);
}
}
/**
* Resolves once the pool is reachable and the schema is up to date.
*
* Idempotent and memoized: the migration runs exactly once per connection,
* on a single dedicated client checked out of the pool (so the migration's
* advisory lock and transaction share one session — see {@link runMigrations}).
*/
async waitUntilReady() {
if (!this.readyPromise) {
this.readyPromise = this.bootstrap();
}
return this.readyPromise;
}
async bootstrap() {
const client = await this.pool.connect();
try {
await (0, migrator_1.runMigrations)(client, this.schema, {
skipVersionCheck: this.skipVersionCheck,
});
}
finally {
client.release();
}
// Defer so listeners attached synchronously after construction still fire.
setTimeout(() => this.emit('ready'), 0);
}
/**
* Returns the dedicated client used for `LISTEN`/`NOTIFY`, establishing it on
* first use.
*
* When this connection owns the pool we use a *standalone* `pg.Client` (its
* own dedicated TCP connection) so the long-lived `LISTEN` never consumes a
* pool slot — this is what lets the query pool run at `max: 1` without
* deadlocking. When the user supplied their own `pg.Pool` we check a client
* out of it (so such pools should be sized `>= 2`).
*/
async getListenClient() {
// Memoize the establishment promise (not just the resolved client) so two
// callers racing on the first use don't each open a connection — the
// `await` below is exactly where a second caller would otherwise slip in.
if (!this.listenClientPromise) {
this.listenClientPromise = (async () => {
if (this.pgModule && this.listenClientConfig) {
const client = new this.pgModule.Client(this.listenClientConfig);
await client.connect();
client.on('error', err => this.emitError(err));
this.listenClientIsStandalone = true;
this.listenClient = client;
return client;
}
else {
const client = await this.pool.connect();
client.on('error', err => this.emitError(err));
this.listenClientIsStandalone = false;
this.listenClient = client;
return client;
}
})();
}
return this.listenClientPromise;
}
/**
* Truthy once {@link PostgresConnection.close} has begun.
*/
get isClosing() {
return this.closing;
}
/**
* Closes the connection: releases the `LISTEN` client and (if owned) ends the
* pool. Safe to call multiple times.
*/
async close() {
if (!this.closing) {
this.closing = (async () => {
var _a, _b;
// Await any in-flight establishment so we never leak a LISTEN client
// whose `getListenClient` promise was still pending when close() ran.
const client = (_a = this.listenClient) !== null && _a !== void 0 ? _a : (await ((_b = this.listenClientPromise) === null || _b === void 0 ? void 0 : _b.catch(() => undefined)));
this.listenClientPromise = undefined;
if (client) {
client.removeAllListeners();
if (this.listenClientIsStandalone) {
// Standalone dedicated connection: end it outright.
await client.end();
}
else {
// Pooled client: return it to the pool.
client.release();
}
this.listenClient = undefined;
}
if (this.ownsPool) {
await this.pool.end();
}
this.emit('close');
})();
}
return this.closing;
}
/**
* Forcibly tears down the connection. For PostgreSQL there is no distinct
* "disconnect without waiting" semantics beyond closing, so this delegates to
* {@link PostgresConnection.close}.
*/
async disconnect() {
return this.close();
}
}
exports.PostgresConnection = PostgresConnection;

File diff suppressed because it is too large Load Diff

67
node_modules/bullmq/dist/cjs/postgres/sql-loader.js generated vendored Normal file
View File

@@ -0,0 +1,67 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.loadMigrationSql = loadMigrationSql;
exports.loadCommandSql = loadCommandSql;
const fs_1 = require("fs");
const path_1 = require("path");
const url_1 = require("url");
function getDirname() {
if (typeof __dirname !== 'undefined' && __dirname) {
return __dirname;
}
const stack = new Error().stack || '';
for (const line of stack.split('\n')) {
const match = line.match(/(file:\/\/\/[^\s)]+)/);
if (match) {
try {
return (0, path_1.dirname)((0, url_1.fileURLToPath)(match[1]));
}
catch (_a) {
// continue searching
}
}
}
throw new Error('Could not determine sql-loader directory path');
}
const currentDir = getDirname();
/**
* Loads a migration's SQL from its `.sql` file — the portable source of truth
* shared with the Elixir/Python ports. Results are cached after the first read.
*
* The `.sql` files live next to this module under `migrations/`. The published
* build copies them alongside the compiled output (a `copy:sql` build step,
* analogous to how the Lua scripts are bundled), so the same relative lookup
* works at runtime.
*/
const MIGRATIONS_DIR = (0, path_1.join)(currentDir, 'migrations');
/**
* Runtime queries live under `commands/`. Each `.sql` file is one parameterized
* statement (a `SELECT fn(...)` for the PL/pgSQL operations, or a direct
* query). They contain NO schema/namespace references — the connection's
* `search_path` selects the schema — so they are portable verbatim to the
* Python/Elixir/PHP/Rust ports (mirroring how the Redis backend's `.lua`
* scripts never hardcode the key prefix).
*/
const COMMANDS_DIR = (0, path_1.join)(currentDir, 'commands');
const migrationCache = new Map();
const commandCache = new Map();
function loadMigrationSql(file) {
let sql = migrationCache.get(file);
if (sql === undefined) {
sql = (0, fs_1.readFileSync)((0, path_1.join)(MIGRATIONS_DIR, file), 'utf8');
migrationCache.set(file, sql);
}
return sql;
}
/**
* Loads a runtime command's SQL by name (without the `.sql` extension), cached
* after the first read.
*/
function loadCommandSql(name) {
let sql = commandCache.get(name);
if (sql === undefined) {
sql = (0, fs_1.readFileSync)((0, path_1.join)(COMMANDS_DIR, `${name}.sql`), 'utf8');
commandCache.set(name, sql);
}
return sql;
}