Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,51 @@ jobs:
REDIS_URL: redis://127.0.0.1:6379
run: npm run test:redis

# Redis Cluster (3 masters, useHashTags) × redis driver 4|5
test-redis-cluster:
name: Redis Cluster / redis@${{ matrix.redis-driver }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
redis-driver: ['4', '5']

steps:
- uses: actions/checkout@v4

- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: '22'
cache: npm

- name: Install dependencies
run: npm ci || npm install

- name: Pin redis@${{ matrix.redis-driver }}
run: |
npm install --no-save redis@${{ matrix.redis-driver }}
node -e "console.log('redis driver:', require('redis/package.json').version)"

- name: Start Redis Cluster
run: |
sudo apt-get update -qq
sudo apt-get install -y -qq redis-server
for port in 7001 7002 7003; do
mkdir -p /tmp/redis-cluster/$port
redis-server --port $port --cluster-enabled yes \
--cluster-config-file /tmp/redis-cluster/$port/nodes.conf \
--dir /tmp/redis-cluster/$port --save '' --appendonly no --daemonize yes
done
sleep 1
redis-cli --cluster create 127.0.0.1:7001 127.0.0.1:7002 127.0.0.1:7003 --cluster-replicas 0 --cluster-yes
timeout 30 sh -c "until redis-cli -p 7001 cluster info | grep -q 'cluster_state:ok'; do sleep 1; done"

- name: Jest live tests (Redis Cluster)
env:
REDIS_CLUSTER_URLS: redis://127.0.0.1:7001,redis://127.0.0.1:7002,redis://127.0.0.1:7003
run: npm run test:jest:adapters -- -t Cluster

# Node 20/22/24 × mongodb driver 5|6|7
test-mongo:
name: Node ${{ matrix.node }} / mongo@${{ matrix.mongodb-driver }}
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,5 @@ npm-debug.log*
*.tgz
/.claude
/docs/superpowers
/.superpowers
.versions
2 changes: 1 addition & 1 deletion .meteorignore
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
.cursor
.cursorignore
.superpowers
.claude
.github
.gitignore
Expand All @@ -22,7 +23,6 @@ HISTORY.md
index.cjs
index.cts
index.d.cts
LICENSE
docs
test
package.json
Expand Down
1 change: 1 addition & 0 deletions .npmignore
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ CLAUDE.md
.claude/
.cursor/
.cursorignore
.superpowers/

# Agent Skills source — distributed cross-tool via `npx skills add veliovgroup/josk` from GitHub, not in the npm tarball
skills/
Expand Down
612 changes: 197 additions & 415 deletions README.md

Large diffs are not rendered by default.

40 changes: 25 additions & 15 deletions adapters/mongo.d.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
export type Collection = import("mongodb").Collection;
export type Db = import("mongodb").Db;
export type MongoDbLike = {
collection: (name: string) => object;
command: (command: {
ping: number;
}) => Promise<unknown>;
};
export type JoSk = import("../index.js").JoSk;
export type JoSkExecuteMode = import("../index.js").JoSkExecuteMode;
export type JoSkLock = import("../index.js").JoSkLock;
Expand All @@ -9,8 +13,8 @@ export type AdapterPingResult = {
statusCode: number;
error?: unknown;
};
export type MongoAdapterOption = {
db: Db;
export type MongoAdapterOption<D extends MongoDbLike = MongoDbLike> = {
db: D;
lockCollectionName?: string | undefined;
prefix?: string | undefined;
resetOnInit?: boolean | undefined;
Expand All @@ -22,25 +26,29 @@ export type MongoTask = {
executeAt?: Date | undefined;
isInterval: boolean;
isDeleted: boolean;
claimLeaseId?: string | undefined;
};
/** Class representing MongoDB adapter for JoSk */
export class MongoAdapter {
/**
* Class representing MongoDB adapter for JoSk
* @template {MongoDbLike} [D=MongoDbLike]
*/
export class MongoAdapter<D extends MongoDbLike = MongoDbLike> {
/**
* Create a MongoAdapter instance
* @param {MongoAdapterOption} opts - configuration object
* @param {MongoAdapterOption<D>} opts - configuration object
*/
constructor(opts?: MongoAdapterOption);
constructor(opts?: MongoAdapterOption<D>);
name: string;
prefix: string;
lockCollectionName: string;
resetOnInit: boolean;
/** @type {Db} */
db: Db;
/** @type {D} */
db: D;
uniqueName: string;
/** @type {Collection} */
collection: Collection;
/** @type {Collection} */
lockCollection: Collection;
/** @type {ReturnType<D['collection']>} */
collection: ReturnType<D["collection"]>;
/** @type {ReturnType<D['collection']>} */
lockCollection: ReturnType<D["collection"]>;
/**
* @returns {Promise<void>}
*/
Expand Down Expand Up @@ -76,12 +84,14 @@ export class MongoAdapter {
*/
add(uid: string, isInterval: boolean, delay: number): Promise<boolean>;
/**
* @param {{ uid: string }} task
* Skips the write when `task.claimLeaseId` no longer matches storage.
* @param {{ uid: string, claimLeaseId?: string }} task
* @param {Date} nextExecuteAt
* @returns {Promise<boolean>}
*/
update(task: {
uid: string;
claimLeaseId?: string;
}, nextExecuteAt: Date): Promise<boolean>;
/**
* @param {Date} nextExecuteAt
Expand Down
112 changes: 69 additions & 43 deletions adapters/mongo.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,9 @@
// `Db.command()`, and `Collection` APIs, but are not officially supported.

/**
* @typedef {import('mongodb').Collection} Collection
* @typedef {import('mongodb').Db} Db
* @typedef {object} MongoDbLike
* @property {(name: string) => object} collection
* @property {(command: { ping: number }) => Promise<unknown>} command
* @typedef {import('../index.js').JoSk} JoSk
* @typedef {import('../index.js').JoSkExecuteMode} JoSkExecuteMode
* @typedef {import('../index.js').JoSkLock} JoSkLock
Expand All @@ -22,8 +23,9 @@
*/

/**
* @template {MongoDbLike} [D=MongoDbLike]
* @typedef {object} MongoAdapterOption
* @property {Db} db
* @property {D} db
* @property {string} [lockCollectionName]
* @property {string} [prefix]
* @property {boolean} [resetOnInit]
Expand All @@ -37,6 +39,7 @@
* @property {Date} [executeAt]
* @property {boolean} isInterval
* @property {boolean} isDeleted
* @property {string} [claimLeaseId]
*/

const logError = (error, ...args) => {
Expand All @@ -46,7 +49,7 @@ const logError = (error, ...args) => {
};

/**
* @param {Collection} collection
* @param {object} collection
* @param {object} keys
* @param {object} opts
* @returns {Promise<void>}
Expand Down Expand Up @@ -86,40 +89,58 @@ const ensureIndex = async (collection, keys, opts) => {
};

/**
* Update pipeline for re-registering an interval. Keeps the stored `executeAt`
* when the task is an existing, non-deleted interval with the same `delay` and
* the stored time is earlier than `executeAt`; otherwise schedules `executeAt`.
* Update pipeline for re-registering an interval. Preserve a claimed task's
* recovery deadline; otherwise keep an unchanged interval's earlier schedule.
* Field references inside one `$set` stage read the pre-update document.
* @param {string} uid
* @param {number} delay
* @param {Date} executeAt
* @returns {object[]}
*/
const intervalUpsertPipeline = (uid, delay, executeAt) => [{
$set: {
executeAt: {
$cond: [{
$and: [
{ $eq: ['$isInterval', true] },
{ $eq: ['$isDeleted', false] },
{ $eq: ['$delay', { $literal: delay }] },
{ $eq: [{ $type: '$executeAt' }, 'date'] },
{ $lt: ['$executeAt', { $literal: executeAt }] }
]
}, '$executeAt', { $literal: executeAt }]
},
uid: { $literal: uid },
delay: { $literal: delay },
isInterval: true,
isDeleted: false
}
}];
const intervalUpsertPipeline = (uid, delay, executeAt) => {
const existingInterval = [
{ $eq: ['$isInterval', true] },
{ $eq: ['$isDeleted', false] },
{ $eq: [{ $type: '$executeAt' }, 'date'] }
];
const activeClaim = {
$and: [
...existingInterval,
{ $eq: [{ $type: '$claimLeaseId' }, 'string'] },
{ $ne: ['$claimLeaseId', ''] }
]
};

return [{
$set: {
executeAt: {
$cond: [activeClaim, '$executeAt', {
$cond: [{
$and: [
...existingInterval,
{ $eq: ['$delay', { $literal: delay }] },
{ $lt: ['$executeAt', { $literal: executeAt }] }
]
}, '$executeAt', { $literal: executeAt }]
}]
},
claimLeaseId: { $cond: [activeClaim, '$claimLeaseId', '$$REMOVE'] },
uid: { $literal: uid },
delay: { $literal: delay },
isInterval: true,
isDeleted: false
}
}];
};

/** Class representing MongoDB adapter for JoSk */
/**
* Class representing MongoDB adapter for JoSk
* @template {MongoDbLike} [D=MongoDbLike]
*/
class MongoAdapter {
/**
* Create a MongoAdapter instance
* @param {MongoAdapterOption} opts - configuration object
* @param {MongoAdapterOption<D>} opts - configuration object
*/
constructor(opts = {}) {
this.name = 'mongo';
Expand All @@ -133,12 +154,12 @@ class MongoAdapter {
});
}

/** @type {Db} */
/** @type {D} */
this.db = opts.db;
this.uniqueName = `__JobTasks__${this.prefix}`;
/** @type {Collection} */
/** @type {ReturnType<D['collection']>} */
this.collection = opts.db.collection(this.uniqueName);
/** @type {Collection} */
/** @type {ReturnType<D['collection']>} */
this.lockCollection = opts.db.collection(this.lockCollectionName);
/** @type {JoSk | undefined} */
this.joskInstance = void 0;
Expand Down Expand Up @@ -326,7 +347,8 @@ class MongoAdapter {
executeAt,
isInterval,
isDeleted: false
}
},
$unset: { claimLeaseId: '' }
}, {
upsert: true
});
Expand All @@ -338,7 +360,8 @@ class MongoAdapter {
}

/**
* @param {{ uid: string }} task
* Skips the write when `task.claimLeaseId` no longer matches storage.
* @param {{ uid: string, claimLeaseId?: string }} task
* @param {Date} nextExecuteAt
* @returns {Promise<boolean>}
*/
Expand All @@ -356,13 +379,14 @@ class MongoAdapter {
await this.ready();

try {
const updateResult = await this.collection.updateOne({
uid: task.uid,
isDeleted: false
}, {
$set: {
executeAt: nextExecuteAt
}
const filter = { uid: task.uid, isDeleted: false };
if (typeof task.claimLeaseId === 'string' && task.claimLeaseId !== '') {
filter.claimLeaseId = task.claimLeaseId;
}

const updateResult = await this.collection.updateOne(filter, {
$set: { executeAt: nextExecuteAt },
$unset: { claimLeaseId: '' }
});
return (updateResult?.matchedCount || 0) >= 1;
} catch (opError) {
Expand Down Expand Up @@ -446,7 +470,7 @@ class MongoAdapter {
});

const task = result?._id ? result : result?.value;
return task || null;
return task ? { ...task, claimLeaseId: lock.leaseId } : null;
} catch (mongoError) {
this.joskInstance.__errorHandler(mongoError, '[MongoAdapter] [iterate] [claim]', 'Exception inside MongoAdapter#__claimNextTask() method', null);
return null;
Expand Down Expand Up @@ -511,7 +535,7 @@ class MongoAdapter {
});

if ((result.modifiedCount || 0) === tasks.length) {
return tasks;
return tasks.map((task) => ({ ...task, claimLeaseId: lock.leaseId }));
}

const claimed = await this.collection.find({
Expand All @@ -528,7 +552,9 @@ class MongoAdapter {
}).toArray();
const claimedIds = new Set(claimed.map((task) => String(task._id)));

return tasks.filter((task) => claimedIds.has(String(task._id)));
return tasks
.filter((task) => claimedIds.has(String(task._id)))
.map((task) => ({ ...task, claimLeaseId: lock.leaseId }));
} catch (mongoError) {
this.joskInstance.__errorHandler(mongoError, '[MongoAdapter] [iterate] [batchClaim]', 'Exception inside MongoAdapter#__claimNextTasks() method', null);
return [];
Expand Down
4 changes: 3 additions & 1 deletion adapters/postgres.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -82,12 +82,14 @@ export class PostgresAdapter {
*/
add(uid: string, isInterval: boolean, delay: number): Promise<boolean>;
/**
* @param {{ uid: string }} task
* Skips the write when `task.claimLeaseId` no longer matches storage.
* @param {{ uid: string, claimLeaseId?: string }} task
* @param {Date} nextExecuteAt
* @returns {Promise<boolean>}
*/
update(task: {
uid: string;
claimLeaseId?: string;
}, nextExecuteAt: Date): Promise<boolean>;
/**
* @param {Date} nextExecuteAt
Expand Down
Loading
Loading