examples/storefront Public
Search the catalog with D1 full text search
GET /search?q= matches product names and descriptions in an FTS5 index kept in a second D1 database, storefront-search, bound as SEARCH. Hits are read back from the catalog, so results always have the current price and stock and leave out inactive products. PUT /admin/products/:id updates the product's row in the index after saving it, and POST /admin/search/rebuild refills the index from the catalog.
Changed files
10 files changed262 additions10 deletionsagainst 94db762
Changedpackage.json+1−1
| @@ -12,7 +12,7 @@ | ||
| "test": "node --disable-warning=ExperimentalWarning --test \"test/**/*.test.ts\"", | ||
| "typecheck": "tsc", | ||
| "dev": "npx wrangler dev", | ||
| - "db:migrate": "npx wrangler d1 migrations apply storefront --local", | ||
| + "db:migrate": "npx wrangler d1 migrations apply storefront --local && npx wrangler d1 migrations apply storefront-search --local", | ||
| "db:seed": "npx wrangler d1 execute storefront --local --file scripts/seed.sql" | ||
| }, | ||
| "devDependencies": { | ||
Addedsearch/migrations/0001_products_fts.sql+13−0
| @@ -0,0 +1,13 @@ | ||
| +-- The search index, in its own D1 database (storefront-search). One row per active product with | ||
| +-- the text shoppers search: its name and description. Everything else about a product is read | ||
| +-- from the storefront database, which stays the source of truth. The index can be emptied and | ||
| +-- rebuilt from the catalog at any time with POST /admin/search/rebuild. | ||
| +-- | ||
| +-- porter stems English words (aprons matches apron); unicode61 with remove_diacritics 2 lets | ||
| +-- creme match Crème. | ||
| +CREATE VIRTUAL TABLE products_fts USING fts5( | ||
| + product_id UNINDEXED, | ||
| + name, | ||
| + description, | ||
| + tokenize = 'porter unicode61 remove_diacritics 2' | ||
| +); | ||
Changedsrc/app.ts+20−1
| @@ -10,6 +10,7 @@ | ||
| import { getOrder } from "./orders.ts"; | ||
| import { stripePayments, type Payments } from "./payments.ts"; | ||
| import { Router } from "./router.ts"; | ||
| +import { indexProduct, parseQuery, rebuildIndex, searchProducts } from "./search.ts"; | ||
| export type Deps = { | ||
| now: () => number; | ||
| @@ -32,6 +33,12 @@ | ||
| return json(await listProducts(env.DB, images, parseLimit(q.get("limit")), q.get("after"))); | ||
| }); | ||
| + r.on("GET", "/search", async (req) => { | ||
| + const params = new URL(req.url).searchParams; | ||
| + const q = parseQuery(params.get("q")); | ||
| + return json({ query: q, products: await searchProducts(env.DB, env.SEARCH, images, q, parseLimit(params.get("limit"))) }); | ||
| + }); | ||
| + | ||
| r.on("GET", "/products/:id", async (_req, p) => { | ||
| const product = await getProduct(env.DB, p.id!, images); | ||
| if (!product || !product.active) throw new HttpError(404, "not_found", "No such product."); | ||
| @@ -41,7 +48,19 @@ | ||
| r.on("PUT", "/admin/products/:id", async (req, p) => { | ||
| requireAdmin(req, env.ADMIN_TOKEN); | ||
| const input = parseProductInput(p.id!, await readJson(req)); | ||
| - return json({ product: await upsertProduct(env.DB, input, deps.now(), images) }); | ||
| + const product = await upsertProduct(env.DB, input, deps.now(), images); | ||
| + try { | ||
| + await indexProduct(env.SEARCH, product); | ||
| + } catch (e) { | ||
| + console.error(JSON.stringify({ msg: "indexing the product failed", product: product.id, error: String(e) })); | ||
| + throw new HttpError(503, "search_index_failed", "The product was saved, but the search index was not updated. Send the same request again."); | ||
| + } | ||
| + return json({ product }); | ||
| + }); | ||
| + | ||
| + r.on("POST", "/admin/search/rebuild", async (req) => { | ||
| + requireAdmin(req, env.ADMIN_TOKEN); | ||
| + return json({ indexed: await rebuildIndex(env.DB, env.SEARCH) }); | ||
| }); | ||
| r.on("PUT", "/admin/products/:id/image", async (req, p) => { | ||
Changedsrc/catalog.ts+15−0
| @@ -90,6 +90,21 @@ | ||
| return row ? toProduct(row, images) : null; | ||
| } | ||
| +// The active products among `ids`, in the order given. Ids that are unknown or inactive are left | ||
| +// out. Search uses it to turn index hits into products with their current price and stock. | ||
| +export async function activeProducts(db: D1Database, ids: string[], images: ImageUrls): Promise<Product[]> { | ||
| + if (!ids.length) return []; | ||
| + const { results } = await db | ||
| + .prepare(`SELECT ${COLUMNS} FROM products WHERE active = 1 AND id IN (SELECT value FROM json_each(?))`) | ||
| + .bind(JSON.stringify(ids)) | ||
| + .all<Row>(); | ||
| + const byId = new Map(results.map((r) => [r.id, r])); | ||
| + return ids.flatMap((id) => { | ||
| + const row = byId.get(id); | ||
| + return row ? [toProduct(row, images)] : []; | ||
| + }); | ||
| +} | ||
| + | ||
| export type ProductInput = { | ||
| id: string; | ||
| name: string; | ||
Changedsrc/env.ts+1−0
| @@ -3,6 +3,7 @@ | ||
| export interface Env { | ||
| DB: D1Database; | ||
| + SEARCH: D1Database; // the catalog's search index (src/search.ts) | ||
| IMAGES: R2Bucket; | ||
| ORDER_EVENTS: Queue<OrderEvent>; | ||
| IMAGES_URL: string; // the bucket's custom domain, where browsers load images | ||
Addedsrc/search.ts+87−0
| @@ -0,0 +1,87 @@ | ||
| +// Catalog search: a full text index of product names and descriptions in its own D1 database | ||
| +// (SEARCH, search/migrations/). The storefront database stays the source of truth. The index | ||
| +// holds only the text search matches on, and every hit is read back from the catalog, so a | ||
| +// result always has the current price and stock and never includes an inactive product. | ||
| +// See .rebasely/kb/concepts/search.md. | ||
| + | ||
| +import { activeProducts, type Product } from "./catalog.ts"; | ||
| +import { HttpError } from "./http.ts"; | ||
| +import type { ImageUrls } from "./images.ts"; | ||
| + | ||
| +export const MAX_QUERY = 100; | ||
| +const MAX_TERMS = 8; | ||
| +const REBUILD_PAGE = 50; | ||
| + | ||
| +// Name matches count ten times as much as description matches. The first weight is for | ||
| +// product_id, which is not indexed. | ||
| +const RANK = "bm25(products_fts, 0.0, 10.0, 1.0)"; | ||
| + | ||
| +type Indexed = { id: string; name: string; description: string; active: boolean }; | ||
| + | ||
| +// Turns what a shopper typed into an FTS5 query. Each word becomes a quoted string, so FTS5 | ||
| +// syntax in the input (AND, NEAR, column filters, a stray quote) is just text, and the last word | ||
| +// matches as a prefix, so results keep up while someone is still typing. Null when nothing | ||
| +// searchable is left. | ||
| +export function toMatch(q: string): string | null { | ||
| + const words = q.normalize("NFKC").match(/[\p{L}\p{N}]+/gu)?.slice(0, MAX_TERMS) ?? []; | ||
| + if (!words.length) return null; | ||
| + return words.map((w, i) => (i === words.length - 1 ? `"${w}"*` : `"${w}"`)).join(" "); | ||
| +} | ||
| + | ||
| +export function parseQuery(value: string | null): string { | ||
| + const q = (value ?? "").trim(); | ||
| + if (!q || q.length > MAX_QUERY) throw new HttpError(400, "bad_query", `q is the text to search for, 1 to ${MAX_QUERY} characters.`); | ||
| + return q; | ||
| +} | ||
| + | ||
| +// The products matching `q`, best match first. | ||
| +export async function searchProducts(db: D1Database, search: D1Database, images: ImageUrls, q: string, limit: number): Promise<Product[]> { | ||
| + const match = toMatch(q); | ||
| + if (match === null) return []; | ||
| + const { results } = await search | ||
| + .prepare(`SELECT product_id FROM products_fts WHERE products_fts MATCH ? ORDER BY ${RANK} LIMIT ?`) | ||
| + .bind(match, limit) | ||
| + .all<{ product_id: string }>(); | ||
| + return activeProducts( | ||
| + db, | ||
| + results.map((r) => r.product_id), | ||
| + images, | ||
| + ); | ||
| +} | ||
| + | ||
| +// Brings one product's row in the index in line with the catalog: replaced when the product is | ||
| +// active, removed when it is not. Called after every write to the product. | ||
| +export async function indexProduct(search: D1Database, p: Indexed): Promise<void> { | ||
| + const remove = search.prepare("DELETE FROM products_fts WHERE product_id = ?").bind(p.id); | ||
| + if (!p.active) { | ||
| + await remove.run(); | ||
| + return; | ||
| + } | ||
| + await search.batch([ | ||
| + remove, | ||
| + search.prepare("INSERT INTO products_fts (product_id, name, description) VALUES (?, ?, ?)").bind(p.id, p.name, p.description), | ||
| + ]); | ||
| +} | ||
| + | ||
| +// Empties the index and fills it again from every active product in the catalog, a page at a | ||
| +// time. For a new search database, and for repairing one that has drifted. While it runs, | ||
| +// searches can miss products that have not been added back yet. | ||
| +export async function rebuildIndex(db: D1Database, search: D1Database): Promise<number> { | ||
| + await search.prepare("DELETE FROM products_fts").run(); | ||
| + let after = ""; | ||
| + let indexed = 0; | ||
| + for (;;) { | ||
| + const { results } = await db | ||
| + .prepare("SELECT id, name, description FROM products WHERE active = 1 AND id > ? ORDER BY id LIMIT ?") | ||
| + .bind(after, REBUILD_PAGE) | ||
| + .all<{ id: string; name: string; description: string }>(); | ||
| + if (!results.length) break; | ||
| + await search.batch( | ||
| + results.map((r) => search.prepare("INSERT INTO products_fts (product_id, name, description) VALUES (?, ?, ?)").bind(r.id, r.name, r.description)), | ||
| + ); | ||
| + indexed += results.length; | ||
| + after = results.at(-1)!.id; | ||
| + if (results.length < REBUILD_PAGE) break; | ||
| + } | ||
| + return indexed; | ||
| +} | ||
Changedtest/helpers/app.ts+2−2
| @@ -3,7 +3,7 @@ | ||
| import { createApp } from "../../src/app.ts"; | ||
| import type { Env } from "../../src/env.ts"; | ||
| import type { OrderEvent } from "../../src/events.ts"; | ||
| -import { testDb } from "./d1.ts"; | ||
| +import { testDb, testSearchDb } from "./d1.ts"; | ||
| import { FakePayments } from "./payments.ts"; | ||
| import { testQueue } from "./queue.ts"; | ||
| import { testBucket } from "./r2.ts"; | ||
| @@ -13,7 +13,7 @@ | ||
| export function setup() { | ||
| const events = testQueue<OrderEvent>(); | ||
| - const env: Env = { DB: testDb(), IMAGES: testBucket(), ORDER_EVENTS: events, IMAGES_URL: "https://images.shop.test", ADMIN_TOKEN, STRIPE_SECRET_KEY: "sk_test_unused" }; | ||
| + const env: Env = { DB: testDb(), SEARCH: testSearchDb(), IMAGES: testBucket(), ORDER_EVENTS: events, IMAGES_URL: "https://images.shop.test", ADMIN_TOKEN, STRIPE_SECRET_KEY: "sk_test_unused" }; | ||
| const payments = new FakePayments(); | ||
| let n = 0; | ||
| const app = createApp(env, { now: () => NOW, newId: () => `id-${++n}`, payments: () => payments }); | ||
Changedtest/helpers/d1.ts+14−6
| @@ -1,5 +1,5 @@ | ||
| -// A D1 stand-in over node:sqlite with every file in migrations/ applied, so tests run the | ||
| -// storefront's real SQL. It covers what the code calls: prepare, bind, first, all, run and batch. | ||
| +// A D1 stand-in over node:sqlite with every file in a migrations folder applied (migrations/ | ||
| +// unless told otherwise), so tests run the storefront's real SQL. It covers what the code calls: prepare, bind, first, all, run and batch. | ||
| // Foreign keys are on, as they are in D1. | ||
| import { readdirSync, readFileSync } from "node:fs"; | ||
| @@ -7,7 +7,9 @@ | ||
| import { DatabaseSync, type SQLInputValue } from "node:sqlite"; | ||
| import { fileURLToPath } from "node:url"; | ||
| -const MIGRATIONS = join(dirname(fileURLToPath(import.meta.url)), "../../migrations"); | ||
| +const ROOT = join(dirname(fileURLToPath(import.meta.url)), "../.."); | ||
| +const MIGRATIONS = join(ROOT, "migrations"); | ||
| +const SEARCH_MIGRATIONS = join(ROOT, "search/migrations"); | ||
| class Statement { | ||
| readonly db: DatabaseSync; | ||
| @@ -44,10 +46,10 @@ | ||
| export class TestD1 { | ||
| readonly sqlite = new DatabaseSync(":memory:"); | ||
| - constructor() { | ||
| + constructor(migrations = MIGRATIONS) { | ||
| this.sqlite.exec("PRAGMA foreign_keys = ON"); | ||
| - const files = readdirSync(MIGRATIONS).filter((f) => f.endsWith(".sql")).sort(); | ||
| - for (const f of files) this.sqlite.exec(readFileSync(join(MIGRATIONS, f), "utf8")); | ||
| + const files = readdirSync(migrations).filter((f) => f.endsWith(".sql")).sort(); | ||
| + for (const f of files) this.sqlite.exec(readFileSync(join(migrations, f), "utf8")); | ||
| } | ||
| prepare(sql: string): Statement { | ||
| @@ -71,4 +73,10 @@ | ||
| export function testDb(): D1Database { | ||
| return new TestD1() as unknown as D1Database; | ||
| +} | ||
| + | ||
| +// The search index database, with search/migrations/ applied. node:sqlite is built with FTS5, | ||
| +// as D1 is. | ||
| +export function testSearchDb(): D1Database { | ||
| + return new TestD1(SEARCH_MIGRATIONS) as unknown as D1Database; | ||
| } | ||
Addedtest/search.test.ts+101−0
| @@ -0,0 +1,101 @@ | ||
| +import assert from "node:assert/strict"; | ||
| +import { test } from "node:test"; | ||
| +import { toMatch } from "../src/search.ts"; | ||
| +import { ADMIN_TOKEN, setup } from "./helpers/app.ts"; | ||
| + | ||
| +const ids = (res: { body: { products: { id: string }[] } }) => res.body.products.map((p) => p.id); | ||
| + | ||
| +async function shop() { | ||
| + const s = setup(); | ||
| + await s.addProduct("linen-apron", { name: "Linen work apron", description: "Heavy linen with two front pockets and cross-back straps.", price_cents: 5200 }); | ||
| + await s.addProduct("canvas-tote", { name: "Waxed canvas tote", description: "A 20 litre tote with leather handles. Pairs well with the linen apron.", price_cents: 4800 }); | ||
| + await s.addProduct("enamel-mug", { name: "Enamel camp mug", description: "A 350 ml steel mug with a rolled rim.", price_cents: 1600 }); | ||
| + await s.addProduct("creme-soap", { name: "Crème fraîche soap", description: "Cold process, unscented.", price_cents: 900 }); | ||
| + return s; | ||
| +} | ||
| + | ||
| +test("toMatch quotes every word and matches the last one as a prefix", () => { | ||
| + assert.equal(toMatch("linen apr"), '"linen" "apr"*'); | ||
| + assert.equal(toMatch(" Mug! "), '"Mug"*'); | ||
| +}); | ||
| + | ||
| +test("toMatch turns FTS5 syntax into plain words", () => { | ||
| + assert.equal(toMatch('apron" OR name:mug NEAR(x'), '"apron" "OR" "name" "mug" "NEAR" "x"*'); | ||
| + assert.equal(toMatch("*** --- ()"), null); | ||
| +}); | ||
| + | ||
| +test("finds products by name and description, name matches first", async () => { | ||
| + const s = await shop(); | ||
| + const res = await s.call("GET", "/search?q=linen"); | ||
| + assert.equal(res.status, 200); | ||
| + assert.equal(res.body.query, "linen"); | ||
| + assert.deepEqual(ids(res), ["linen-apron", "canvas-tote"]); | ||
| +}); | ||
| + | ||
| +test("matches while the shopper is still typing, across word forms and accents", async () => { | ||
| + const s = await shop(); | ||
| + assert.deepEqual(ids(await s.call("GET", "/search?q=enam")), ["enamel-mug"]); | ||
| + assert.deepEqual(ids(await s.call("GET", "/search?q=mugs")), ["enamel-mug"]); | ||
| + assert.deepEqual(ids(await s.call("GET", "/search?q=creme")), ["creme-soap"]); | ||
| +}); | ||
| + | ||
| +test("results carry the catalog's current price and stock", async () => { | ||
| + const s = await shop(); | ||
| + const cartId = (await s.call("POST", "/carts")).body.cart.id; | ||
| + await s.call("PUT", `/carts/${cartId}/items/enamel-mug`, { quantity: 2 }); | ||
| + await s.call("POST", `/carts/${cartId}/checkout`, { | ||
| + email: "ada@park.test", | ||
| + ship_to: { name: "Ada Park", line1: "12 Alder St", city: "Portland", postal_code: "97205", country: "US" }, | ||
| + }); | ||
| + const [mug] = (await s.call("GET", "/search?q=mug")).body.products; | ||
| + assert.equal(mug.stock, 3); | ||
| + assert.equal(mug.price_cents, 1600); | ||
| +}); | ||
| + | ||
| +test("an edited product is found by its new text, a retired one not at all", async () => { | ||
| + const s = await shop(); | ||
| + await s.addProduct("enamel-mug", { name: "Enamel camp cup", description: "A 350 ml steel cup." }); | ||
| + assert.deepEqual(ids(await s.call("GET", "/search?q=mug")), []); | ||
| + assert.deepEqual(ids(await s.call("GET", "/search?q=cup")), ["enamel-mug"]); | ||
| + await s.addProduct("linen-apron", { name: "Linen work apron", active: false }); | ||
| + assert.deepEqual(ids(await s.call("GET", "/search?q=apron")), ["canvas-tote"]); | ||
| +}); | ||
| + | ||
| +test("limit caps the results and q is required", async () => { | ||
| + const s = await shop(); | ||
| + assert.deepEqual(ids(await s.call("GET", "/search?q=linen&limit=1")), ["linen-apron"]); | ||
| + assert.equal((await s.call("GET", "/search?q=%20")).body.error, "bad_query"); | ||
| + assert.equal((await s.call("GET", `/search?q=${"a".repeat(101)}`)).body.error, "bad_query"); | ||
| + assert.equal((await s.call("GET", "/search?q=linen&limit=0")).body.error, "bad_limit"); | ||
| + assert.deepEqual(ids(await s.call("GET", "/search?q=%2A%2A")), []); | ||
| +}); | ||
| + | ||
| +test("a rebuild puts every active product back in the index", async () => { | ||
| + const s = await shop(); | ||
| + // A product written straight to the catalog, as a seed script does, is not in the index yet. | ||
| + await s.env.DB.prepare( | ||
| + "INSERT INTO products (id, name, description, price_cents, stock, active, created_at, updated_at) VALUES ('wool-socks', 'Merino wool socks', '', 1800, 40, 1, 0, 0)", | ||
| + ).run(); | ||
| + assert.deepEqual(ids(await s.call("GET", "/search?q=socks")), []); | ||
| + | ||
| + assert.equal((await s.call("POST", "/admin/search/rebuild")).status, 401); | ||
| + const res = await s.call("POST", "/admin/search/rebuild", undefined, { authorization: `Bearer ${ADMIN_TOKEN}` }); | ||
| + assert.deepEqual(res.body, { indexed: 5 }); | ||
| + assert.deepEqual(ids(await s.call("GET", "/search?q=socks")), ["wool-socks"]); | ||
| + assert.deepEqual(ids(await s.call("GET", "/search?q=linen")), ["linen-apron", "canvas-tote"]); | ||
| +}); | ||
| + | ||
| +test("a failed index write is reported, and the product is still saved", async () => { | ||
| + const s = await shop(); | ||
| + const search = s.env.SEARCH; | ||
| + s.env.SEARCH = { | ||
| + prepare: (sql: string) => search.prepare(sql), | ||
| + batch: async () => { | ||
| + throw new Error("D1_ERROR: storefront-search is unavailable"); | ||
| + }, | ||
| + } as unknown as D1Database; | ||
| + const res = await s.call("PUT", "/admin/products/enamel-mug", { name: "Enamel camp cup", price_cents: 1600 }, s.admin); | ||
| + assert.equal(res.status, 503); | ||
| + assert.equal(res.body.error, "search_index_failed"); | ||
| + assert.equal((await s.call("GET", "/products/enamel-mug")).body.product.name, "Enamel camp cup"); | ||
| +}); | ||
Changedwrangler.jsonc+8−0
| @@ -12,6 +12,14 @@ | ||
| "database_name": "storefront", | ||
| "database_id": "", | ||
| "migrations_dir": "migrations" | ||
| + }, | ||
| + // The catalog's search index, apart from the shop's data so search reads never queue behind | ||
| + // checkout's writes. Create it with `npx wrangler d1 create storefront-search`. | ||
| + { | ||
| + "binding": "SEARCH", | ||
| + "database_name": "storefront-search", | ||
| + "database_id": "", | ||
| + "migrations_dir": "search/migrations" | ||
| } | ||
| ], | ||
| "r2_buckets": [{ "binding": "IMAGES", "bucket_name": "storefront-images" }], | ||
Checks
No checks reported for this commit.
Runs
No runs for this commit.