Paginator utility
There are a handful of tasks that I find myself reimplementing fairly often, and I've built up a small library of snippets for dealing with these tasks in a generic way. I find that writing these utilities in a completely generic manner means I can reuse them between projects, and also forces me to be disciplined about how I design my codebase.
I'm working on a batch job right now, and needed to iterate over all results from a large database. Here's the utility I use for this:
export interface Paginator<T, Cursor> {
hasMore: boolean;
next: () => Promise<Paginator<T, Cursor>>;
results: T[];
iterate: (
fn: (
results: T[],
meta: { pageNumber: number; resultCount: number },
) => Promise<void>,
) => Promise<void>;
}
export function createPaginator<T, Cursor>(
fetcher: (opts: { pageSize: number; cursor: Cursor }) => Promise<T[]>,
{
pageSize,
startCursor,
getCursorFromResult,
}: {
pageSize: number;
startCursor: Cursor;
getCursorFromResult: (data: T) => Cursor;
},
): Paginator<T, Cursor> {
async function advance(
cursor: Cursor,
{ pageNumber, resultCount }: { pageNumber: number; resultCount: number },
): Promise<Paginator<T, Cursor>> {
const results = await fetcher({ pageSize, cursor });
// Opportunity for refinement: allow this check to be configured
if (results.length === 0) {
return {
hasMore: false,
next: () => {
throw new Error();
},
results,
iterate: () => {
throw new Error();
},
};
}
const finalResult = results.slice(-1)[0];
const nextCursor = getCursorFromResult(finalResult);
const nextPaginator: Paginator<T, Cursor> = {
hasMore: true,
results,
next: () =>
advance(nextCursor, {
pageNumber: pageNumber + 1,
resultCount: resultCount + results.length,
}),
async iterate(fn) {
let p = nextPaginator;
while (p.hasMore) {
await fn(p.results, { pageNumber, resultCount });
p = await p.next();
}
},
};
return nextPaginator;
}
return {
hasMore: true,
results: [],
next: () => advance(startCursor, { pageNumber: 0, resultCount: 0 }),
async iterate(fn) {
const p = await advance(startCursor, { pageNumber: 0, resultCount: 0 });
return p.iterate(fn);
},
};
}
Note that there's nothing here that's specific to my actual use case.
Here's a usage example, very closely patterned after my actual use case:
const paginator = createPaginator(
async ({ pageSize, cursor }): Promise<Array<{ id: number }>> => {
return this.sql.query(
`
select u.id from users as u
where id > $1
order by id
limit $2;
`,
[cursor, pageSize]
);
},
{
pageSize: 50,
startCursor: 0,
getCursorFromResult: (result) => result.id,
}
);
await paginator.iterate(async (results, { resultCount }) => {
resultCount > 0 && console.log(`Completed ${resultCount} results so far`);
await Promise.all(
results.map(async ({ id: userId }) => {
// In the real implementation, I cross-reference two systems here.
await this.updateUser(userId);
})
);
// Wait a minute between pages so we don't hit rate limits or bring down our server.
await new Promise((ok) => setTimeout(ok, 60e3));
});