Page through the product list with a cursor

1 check passed

GET /products returned the first 100 products and stopped. It now takes ?limit= (default 24, at most 100) and ?after=, and returns next, an opaque cursor holding the last product's name and id. Keyset paging keeps every page one index range scan and does not shift when products are added.

Justin Torre committed 3 days ago

48ffd57cd0a33f134ab45c6d504220a97107542d

Parent c6a00ae Browse files History

Changed files

5 files changed98 additions10 deletionsagainst c6a00ae

Changed.rebasely/kb/concepts/catalog.md+15−3
ChangedREADME.md+1−1
Changedsrc/app.ts+5−2
Changedsrc/catalog.ts+42−4
Changedtest/catalog.test.ts+35−0

Changed.rebasely/kb/concepts/catalog.md+15−3

View file

ChangedREADME.md+1−1

View file

Changedsrc/app.ts+5−2

View file

Changedsrc/catalog.ts+42−4

View file

Changedtest/catalog.test.ts+35−0

View file

Checks

1 check passed

Status	Check	Reported by	Description	Details
Success	duragit/ci/test	Rebasely Actions	succeeded	Run

Runs

Status	Workflow	Event	Started
Success	ci	Push	2 h ago

		@@ -6,7 +6,7 @@
		describes:
		- src/catalog.ts
		- migrations/0001_products.sql
		-reviewed: 2026-10-08
		+reviewed: 2026-10-13
		---
		# Catalog

		@@ -14,9 +14,21 @@

		## Reading

		-Shoppers see only active products. `GET /products` lists them by name, then id, so the order is stable when two products share a name. `GET /products/:id` answers 404 for a product that is inactive, the same as for one that does not exist, so a retired product disappears from the shop without breaking old links into an error page.
		+Shoppers see only active products. `GET /products/:id` answers 404 for a product that is inactive, the same as for one that does not exist, so a retired product disappears from the shop without breaking old links into an error page.

		-The list returns the first 100 active products. That covers the current catalog; a larger one needs paging.
		+## Paging
		+
		+`GET /products` returns active products by name, then id, so the order is stable when two products share a name. It returns a page at a time:
		+
		+```
		+GET /products?limit=24
		+{ "products": [...], "next": "WyJMaW5lbiB3b3JrIGFwcm9uIiwibGluZW4tYXByb24iXQ" }
		+GET /products?limit=24&after=WyJMaW5lbiB3b3JrIGFwcm9uIiwibGluZW4tYXByb24iXQ
		+```
		+
		+`limit` is 1 to 100 and defaults to 24, one catalog page. `next` is null on the last page. The cursor is the name and id of the last product on the page, base64url encoded; clients should pass it back unchanged and not build their own.
		+
		+A cursor rather than `?page=3` because the query starts right after the last row (`(name, id) > (?, ?)`) on the `products_by_name` index, so page 40 costs the same as page 1, and a product added or retired between two requests does not shift the next page by one.

		## Writing

		@@ -44,7 +44,7 @@

		\| Method and path \| What it does \|
		\| --- \| --- \|
		-\| `GET /products` \| Active products, by name \|
		+\| `GET /products` \| Active products by name, a page at a time: `?limit=` (1 to 100, default 24) and `?after=` the previous page's `next` \|
		\| `GET /products/:id` \| One active product \|
		\| `PUT /admin/products/:id` \| Create or replace a product (admin token) \|
		\| `PUT /admin/products/:id/image` \| Upload the product's image (admin token) \|

		@@ -2,7 +2,7 @@
		// (the clock and id generation) and returns a fetch handler.

		import { createCart, getCart, setQuantity } from "./carts.ts";
		-import { getProduct, listProducts, parseProductInput, upsertProduct } from "./catalog.ts";
		+import { getProduct, listProducts, parseLimit, parseProductInput, upsertProduct } from "./catalog.ts";
		import { checkout } from "./checkout.ts";
		import type { Env } from "./env.ts";
		import { errorResponse, HttpError, json, readJson, requireAdmin } from "./http.ts";
		@@ -27,7 +27,10 @@
		const r = new Router();
		const images = imageUrls(env.IMAGES_URL);

		- r.on("GET", "/products", async () => json({ products: await listProducts(env.DB, images) }));
		+ r.on("GET", "/products", async (req) => {
		+ const q = new URL(req.url).searchParams;
		+ return json(await listProducts(env.DB, images, parseLimit(q.get("limit")), q.get("after")));
		+ });

		r.on("GET", "/products/:id", async (_req, p) => {
		const product = await getProduct(env.DB, p.id!, images);

		@@ -38,12 +38,50 @@
		};
		}

		-// Active products by name. The first 100 for now; a catalog bigger than that needs pages.
		-export async function listProducts(db: D1Database, images: ImageUrls): Promise<Product[]> {
		+export const DEFAULT_PAGE = 24;
		+export const MAX_PAGE = 100;
		+
		+export type Page = { products: Product[]; next: string \| null };
		+
		+// Active products by name, a page at a time. The cursor is the (name, id) of the last product on
		+// the previous page, so pages stay correct while products are added or retired, and the query
		+// walks the products_by_name index instead of skipping rows with OFFSET.
		+export async function listProducts(db: D1Database, images: ImageUrls, limit = DEFAULT_PAGE, after: string \| null = null): Promise<Page> {
		+ const from = after === null ? null : decodeCursor(after);
		const { results } = await db
		- .prepare(`SELECT ${COLUMNS} FROM products WHERE active = 1 ORDER BY name, id LIMIT 100`)
		+ .prepare(
		+ `SELECT ${COLUMNS} FROM products
		+ WHERE active = 1 AND (?1 IS NULL OR (name, id) > (?1, ?2))
		+ ORDER BY name, id LIMIT ?3`,
		+ )
		+ .bind(from?.[0] ?? null, from?.[1] ?? null, limit + 1)
		.all<Row>();
		- return results.map((r) => toProduct(r, images));
		+ const page = results.slice(0, limit);
		+ const last = page[page.length - 1];
		+ return { products: page.map((r) => toProduct(r, images)), next: results.length > limit && last ? encodeCursor(last.name, last.id) : null };
		+}
		+
		+export function parseLimit(value: string \| null): number {
		+ if (value === null) return DEFAULT_PAGE;
		+ const n = Number(value);
		+ if (!Number.isInteger(n) \|\| n < 1 \|\| n > MAX_PAGE) throw new HttpError(400, "bad_limit", `limit is a whole number from 1 to ${MAX_PAGE}.`);
		+ return n;
		+}
		+
		+function encodeCursor(name: string, id: string): string {
		+ const bytes = new TextEncoder().encode(JSON.stringify([name, id]));
		+ return btoa(String.fromCharCode(...bytes)).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
		+}
		+
		+function decodeCursor(cursor: string): [string, string] {
		+ try {
		+ const bin = atob(cursor.replace(/-/g, "+").replace(/_/g, "/"));
		+ const v: unknown = JSON.parse(new TextDecoder().decode(Uint8Array.from(bin, (c) => c.charCodeAt(0))));
		+ if (Array.isArray(v) && v.length === 2 && typeof v[0] === "string" && typeof v[1] === "string") return [v[0], v[1]];
		+ } catch {
		+ // fall through to the error below
		+ }
		+ throw new HttpError(400, "bad_cursor", "after is a cursor from a previous page's next.");
		}

		// Any product, active or not. Shopper routes check `active` themselves.

		@@ -54,3 +54,38 @@
		assert.equal((await s.call("DELETE", "/products")).status, 405);
		assert.equal((await s.call("GET", "/products/%E0%A4%A")).status, 400);
		});
		+
		+test("pages through the catalog with a cursor", async () => {
		+ const s = setup();
		+ for (const id of ["a-mug", "b-tote", "c-socks", "d-apron", "e-candle"]) await s.addProduct(id, { name: id });
		+ const first = (await s.call("GET", "/products?limit=2")).body;
		+ assert.deepEqual(first.products.map((p: { id: string }) => p.id), ["a-mug", "b-tote"]);
		+ const second = (await s.call("GET", `/products?limit=2&after=${first.next}`)).body;
		+ assert.deepEqual(second.products.map((p: { id: string }) => p.id), ["c-socks", "d-apron"]);
		+ const last = (await s.call("GET", `/products?limit=2&after=${second.next}`)).body;
		+ assert.deepEqual(last.products.map((p: { id: string }) => p.id), ["e-candle"]);
		+ assert.equal(last.next, null);
		+});
		+
		+test("products that share a name are ordered by id and none is skipped", async () => {
		+ const s = setup();
		+ for (const id of ["mug-blue", "mug-green", "mug-red"]) await s.addProduct(id, { name: "Enamel mug" });
		+ const first = (await s.call("GET", "/products?limit=2")).body;
		+ const rest = (await s.call("GET", `/products?limit=2&after=${first.next}`)).body;
		+ assert.deepEqual([...first.products, ...rest.products].map((p: { id: string }) => p.id), ["mug-blue", "mug-green", "mug-red"]);
		+});
		+
		+test("a page boundary survives a product being added before it", async () => {
		+ const s = setup();
		+ for (const id of ["b-tote", "c-socks", "d-apron"]) await s.addProduct(id, { name: id });
		+ const first = (await s.call("GET", "/products?limit=2")).body;
		+ await s.addProduct("a-mug", { name: "a-mug" });
		+ const second = (await s.call("GET", `/products?limit=2&after=${first.next}`)).body;
		+ assert.deepEqual(second.products.map((p: { id: string }) => p.id), ["d-apron"]);
		+});
		+
		+test("refuses a bad limit or cursor", async () => {
		+ const s = setup();
		+ for (const q of ["limit=0", "limit=101", "limit=ten"]) assert.equal((await s.call("GET", `/products?${q}`)).body.error, "bad_limit", q);
		+ assert.equal((await s.call("GET", "/products?after=not-a-cursor")).body.error, "bad_cursor");
		+});