From 37b67644310b7fa2c4c8095afab2a8bbd2cea31a Mon Sep 17 00:00:00 2001 From: code Date: Tue, 14 Jul 2026 17:35:30 +0000 Subject: [PATCH] Add content index: keywords, tags/categories, search, and XML sitemap Ships Blog tags/categories, Internal search, and XML sitemap together as one shared mechanism, plus a new meta keywords request, rather than three separate ones - the "worth deciding together" call ISSUES.md made when XML sitemap was first written. Content stays in files. Per-page metadata is four Twig blocks in the root layout, the same override mechanism already used for title/description/og_* - keywords (rendered), tags/changefreq/priority (not rendered, harvested only). App\ContentIndexer crawls every routable page (Overlay::listPageDirs(), new) and pulls each block via Renderer::renderForIndex() (new) calling Twig's own renderBlock() API, not regex-parsing .twig source, so overrides and layout inheritance resolve exactly like a real render. Rendered HTML is stripped and indexed into a SQLite FTS5 table for search. Off by default (content_index_enabled) - same posture as Matomo/admin auth, since this is a real SQLite dependency plenty of sites won't want. Verified true zero footprint when disabled: no data/novaconium.sqlite gets created, all three consumer routes 404 like they don't exist. content_index_auto (default true) reindexes lazily on first stale touch of a consumer route, never on a normal page view; both that path and the explicit `php novaconium/bin/index-content.php` share one reindex(). Two real bugs caught by testing, not review: a reentrancy bug where /search's own sidecar calling ensureFresh() during the crawl triggered a nested reindex() mid-transaction (fixed with a static re-entrancy guard), and a wrong PDO constant in the search sidecar. Also fixed two unrelated pre-existing bugs found while building this: an unescaped {{ }} in admin/docs/sidecars that fataled Twig on any real render of that page, and a stale "no database layer yet" claim there and in Lib\Input's docblock, left over from before Lib\Db shipped. Lib\Db's migrations_dir now accepts an ordered list of roots, not just one path, so the content index's schema could ship as a framework migration (novaconium/migrations/) without colliding with project migrations in App/migrations/ - the two-root extension point flagged in AGENTS.md when SQLite groundwork shipped. Migrations are tracked by path relative to the repo root rather than bare filename so two roots with a same-named file can't shadow each other. New consumer routes: novaconium/pages/sitemap.xml/ and novaconium/pages/search/ (framework defaults), App/pages/blog/tag/[tag]/ (project-owned, since blog/ is project content - the existing hand -written post array in App/pages/blog/index.php is untouched). Added tags to the 4 existing blog posts as a real demonstration. Closes the Blog tags/categories, Internal search, and XML sitemap backlog items in novaconium/ISSUES.md. --- AGENTS.md | 84 +++++-- App/config.php | 9 + App/pages/blog/hello-world/index.twig | 1 + App/pages/blog/second-post/index.twig | 1 + App/pages/blog/style-guide/index.twig | 1 + App/pages/blog/tag/[tag]/index.php | 65 +++++ App/pages/blog/tag/[tag]/index.twig | 24 ++ App/pages/blog/twig-syntax-guide/index.twig | 1 + README.md | 6 +- novaconium/ISSUES.md | 224 ++++++++++------- novaconium/bin/create-static-page.php | 4 + novaconium/bin/index-content.php | 23 ++ novaconium/config.php | 31 ++- novaconium/lib/Db.php | 89 ++++--- novaconium/lib/Input.php | 8 +- .../migrations/0001_create_content_index.sql | 25 ++ novaconium/pages/_layout/layout.twig | 10 + .../pages/admin/docs/_layout/layout.twig | 1 + .../pages/admin/docs/content-index/index.twig | 84 +++++++ .../pages/admin/docs/database/index.twig | 13 +- novaconium/pages/admin/docs/index.twig | 3 +- novaconium/pages/admin/docs/seo/index.twig | 10 +- .../pages/admin/docs/sidecars/index.twig | 2 +- novaconium/pages/search/index.php | 103 ++++++++ novaconium/pages/search/index.twig | 31 +++ novaconium/pages/sitemap.xml/index.php | 73 ++++++ novaconium/src/ContentIndexer.php | 234 ++++++++++++++++++ novaconium/src/IndexedPage.php | 26 ++ novaconium/src/Overlay.php | 53 ++++ novaconium/src/Renderer.php | 41 +++ 30 files changed, 1123 insertions(+), 157 deletions(-) create mode 100644 App/pages/blog/tag/[tag]/index.php create mode 100644 App/pages/blog/tag/[tag]/index.twig create mode 100644 novaconium/bin/index-content.php create mode 100644 novaconium/migrations/0001_create_content_index.sql create mode 100644 novaconium/pages/admin/docs/content-index/index.twig create mode 100644 novaconium/pages/search/index.php create mode 100644 novaconium/pages/search/index.twig create mode 100644 novaconium/pages/sitemap.xml/index.php create mode 100644 novaconium/src/ContentIndexer.php create mode 100644 novaconium/src/IndexedPage.php diff --git a/AGENTS.md b/AGENTS.md index 2fe5413..30a5cbd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -146,15 +146,27 @@ parameterized queries as the sole SQL-injection defense. Each connection is lazy and independent (opened on first `Db::query()`/`Db::connection()` call naming it, same shape as `Lib\Csrf`'s lazy session start), and migrates automatically at that point: plain `.sql` files under that connection's own -`migrations_dir`, filename-ordered, tracked in that connection's own -auto-created `schema_migrations` table (each connection's applied -migrations are independent of any other connection's), run once each. -`novaconium/bin/migrate.php` loops every configured connection and runs the -same migration step explicitly (e.g. from a deploy script) without serving -a request first. Only `'sqlite'` and `'mysql'` drivers are implemented; a -connection's `migrations_dir` is optional — omit it to never run migrations -against that connection (e.g. a legacy database this project shouldn't -manage schema for). +`migrations_dir` — a single path, or (since the content index shipped) an +**ordered list of roots**, each root's own files filename-ordered and +roots processed fully in the order given, not interleaved by filename +across roots (so a framework root always finishes before a project root on +the same connection). Tracked by path **relative to the repo root** (e.g. +`novaconium/migrations/0001_x.sql`), not bare filename — two roots can +each contain a same-named file, and tracking by bare filename would make +the second one seen look "already applied" and silently skip it; a +repo-relative path is also portable across environments, unlike a full +absolute path, which would make every migration look new again after a +clone/deploy to a different directory. `realpath()` normalizes any `..` a +`migrations_dir` like `__DIR__ . '/../App/migrations'` would otherwise +leave in the tracked name. Each connection's own auto-created +`schema_migrations` table tracks its migrations independently of any other +connection's; each is only ever run once. `novaconium/bin/migrate.php` +loops every configured connection and runs the same migration step +explicitly (e.g. from a deploy script) without serving a request first. +Only `'sqlite'` and `'mysql'` drivers are implemented; a connection's +`migrations_dir` is optional — omit it to never run migrations against +that connection (e.g. a legacy database this project shouldn't manage +schema for). **`db_connections` is the one config key in the project that isn't plain shallow-merge** — `bootstrap.php`/`bin/*.php`'s usual `array_merge($config, @@ -185,11 +197,12 @@ instead — project-owned like `App/`, gitignored per-file (`*.sqlite`/`-journal`/`-wal`/`-shm`, with a tracked `.gitkeep` so the directory exists in a fresh clone) rather than wholesale like `public/cache/`, since a project might reasonably want other non-DB files -there later. Only `App/migrations/` (the framework default connection's -`migrations_dir`) is scanned by default — the framework ships no core -tables of its own yet — if a future framework feature needs a shipped -migration, extend this to the same App-over-novaconium two-root scan -`Overlay.php` already does for pages/lib, don't invent a second mechanism. +there later. The default connection's `migrations_dir` scans +`novaconium/migrations/` (framework-shipped schema, e.g. the content +index below) before `App/migrations/` (project schema) — see the +`migrations_dir` array-form paragraph above. Any *other* connection a +project adds still defaults to no `migrations_dir` at all unless it sets +one; the two-root default is specific to `default`. `Lib\Session` (`novaconium/lib/Session.php`) is a thin wrapper around native PHP sessions (`session_start()`/`$_SESSION`, not a custom store), @@ -257,6 +270,49 @@ something is gated there, which defeats the point of hiding it. Returns consistently with the rest of `/admin/*`: wide open until a password is configured, gated once one is. +`App\ContentIndexer` (`novaconium/src/ContentIndexer.php`) is the shared +crawler behind `/sitemap.xml`, `/search`, and blog tag browsing (see +`/admin/docs/content-index`) — `App\`, not `Lib\`, since it's rendering +infrastructure akin to `Renderer`/`Router`, not project-overridable +content. Content stays in files; only metadata is indexed. Per-page +metadata is four Twig blocks declared in the root layout next to the SEO +blocks (`keywords`, `tags`, `changefreq`, `priority` — the last three +never rendered into the page, only harvested) and pulled via +`Renderer::renderForIndex()`, which calls Twig's own +`TemplateWrapper::renderBlock()` per block rather than parsing `.twig` +source — this is deliberate: it gets App-over-novaconium override and +layout-inheritance resolution for free, the same way a real render does. +**`config['content_index_enabled']` defaults to `false`** — same posture +as `matomo_url`/`admin_password_hash`, since this is a real SQLite +dependency plenty of sites won't want. Every consumer route checks the +flag *before* touching `Lib\Db` and 404s if it's off, so the feature has +zero filesystem footprint (no `data/novaconium.sqlite`) when disabled — +verified end-to-end, not assumed, since "off" silently still creating a +database file would defeat the point. + +**Reentrancy hazard, already hit once:** `ContentIndexer::reindex()` +renders every routable page as part of the crawl — including `/search` +itself, which is also a real page and also calls +`ContentIndexer::ensureFresh()` from its own sidecar. Without a guard, +crawling `/search` would trigger a nested `reindex()` call mid-transaction +and fatal on a second `PDO::beginTransaction()`. `ContentIndexer` guards +this with a `private static bool $indexing` flag, checked at the top of +both `ensureFresh()` and `reindex()` — either no-ops while a reindex is +already running on the call stack. Any future consumer route added under +this mechanism inherits the same hazard for free (it'll also get crawled, +and if its sidecar also calls `ensureFresh()`, the guard already covers +it) — don't remove the flag thinking it's unnecessary. + +`reindex()` also forces `$_SERVER['REQUEST_METHOD']` to `'GET'` for the +duration of the crawl (restoring whatever it was before, in a `finally`) +— sidecars are expected to be side-effect-free for non-POST requests +anyway (ordinary HTTP-safe-method hygiene), but this guarantees a lazy +reindex triggered from within a POST request can never leak that POST +into an unrelated page's sidecar purely because the crawler happened to +render it. A crawl is a full truncate-and-rebuild inside one transaction, +not incremental — simple and correct at this site's scale; don't add +incremental/diffing logic without a real need for it. + ## Running it ``` diff --git a/App/config.php b/App/config.php index e314a18..d6c7113 100644 --- a/App/config.php +++ b/App/config.php @@ -42,4 +42,13 @@ return [ // set to actually gate anything; open access otherwise, same as the // rest of /admin/*. // 'draft_routes' => ['blog/upcoming-post'], + + // Docs: /admin/docs/content-index — powers /sitemap.xml, /search, and + // blog tag browsing. Off by default (depends on SQLite); enable with: + // 'content_index_enabled' => true, + // + // Or enable but skip the automatic lazy reindex, relying only on + // `php novaconium/bin/index-content.php` (e.g. from a deploy step): + // 'content_index_enabled' => true, + // 'content_index_auto' => false, ]; diff --git a/App/pages/blog/hello-world/index.twig b/App/pages/blog/hello-world/index.twig index 8705157..c1a2da0 100644 --- a/App/pages/blog/hello-world/index.twig +++ b/App/pages/blog/hello-world/index.twig @@ -6,6 +6,7 @@ {% block description %}The first post on this blog — a plain sidecar-less page, like every other post here now.{% endblock %} {% block robots %}index, follow{% endblock %} +{% block tags %}welcome, meta{% endblock %} {% block canonical %}{{ request_path|default('/') }}{% endblock %} {% block og_type %}article{% endblock %} diff --git a/App/pages/blog/second-post/index.twig b/App/pages/blog/second-post/index.twig index f9c8e57..5dd9223 100644 --- a/App/pages/blog/second-post/index.twig +++ b/App/pages/blog/second-post/index.twig @@ -6,6 +6,7 @@ {% block description %}A second post at its own URL, showing that adding a new page under App/pages/blog/ needs nothing but a new directory.{% endblock %} {% block robots %}index, follow{% endblock %} +{% block tags %}meta{% endblock %} {% block canonical %}{{ request_path|default('/') }}{% endblock %} {% block og_type %}article{% endblock %} diff --git a/App/pages/blog/style-guide/index.twig b/App/pages/blog/style-guide/index.twig index 5e2d0fc..b46becc 100644 --- a/App/pages/blog/style-guide/index.twig +++ b/App/pages/blog/style-guide/index.twig @@ -6,6 +6,7 @@ {% block description %}A showcase of this theme's default styling for headings, lists, tables, code, and other common HTML elements.{% endblock %} {% block robots %}index, follow{% endblock %} +{% block tags %}css, reference{% endblock %} {% block canonical %}{{ request_path|default('/') }}{% endblock %} {% block og_type %}article{% endblock %} diff --git a/App/pages/blog/tag/[tag]/index.php b/App/pages/blog/tag/[tag]/index.php new file mode 100644 index 0000000..b6912a6 --- /dev/null +++ b/App/pages/blog/tag/[tag]/index.php @@ -0,0 +1,65 @@ +fetchAll(PDO::FETCH_ASSOC); + +// $posts is [] (not an error) when nothing matches — the twig template +// renders a plain "no posts tagged ..." message for that case, same as +// /search does for a query with no results. +return [ + 'tag' => $tag, + 'posts' => $posts, +]; diff --git a/App/pages/blog/tag/[tag]/index.twig b/App/pages/blog/tag/[tag]/index.twig new file mode 100644 index 0000000..0f255b1 --- /dev/null +++ b/App/pages/blog/tag/[tag]/index.twig @@ -0,0 +1,24 @@ +{% extends layout %} + +{% import '_layout/icons.twig' as icons %} + +{% block title %}Posts tagged “{{ tag }}”{% endblock %} +{% block description %}Blog posts tagged {{ tag }}.{% endblock %} +{% block robots %}noindex, follow{% endblock %} + +{% block content %} +

{{ icons.tag() }}Posts tagged “{{ tag }}”

+ + {% if posts|length > 0 %} + + {% else %} +

No posts tagged “{{ tag }}”.

+ {% endif %} +{% endblock %} diff --git a/App/pages/blog/twig-syntax-guide/index.twig b/App/pages/blog/twig-syntax-guide/index.twig index a99f152..21de78a 100644 --- a/App/pages/blog/twig-syntax-guide/index.twig +++ b/App/pages/blog/twig-syntax-guide/index.twig @@ -6,6 +6,7 @@ {% block description %}A tour of the Twig syntax used throughout this site — output, filters, control structures, inheritance, and a few gotchas.{% endblock %} {% block robots %}index, follow{% endblock %} +{% block tags %}twig, reference{% endblock %} {% block canonical %}{{ request_path|default('/') }}{% endblock %} {% block og_type %}article{% endblock %} diff --git a/README.md b/README.md index 7d8829c..0f50363 100644 --- a/README.md +++ b/README.md @@ -19,11 +19,12 @@ A tiny, Hugo-flavored PHP micro-framework. Routes are directories on disk, pages - **Form security by default** — `Lib\Input`, a cleaning accessor for `$_POST`/`$_GET` (defense-in-depth against HTML/script injection, not a substitute for parameterized queries), and `Lib\Csrf`, standalone session-token CSRF protection called directly from a sidecar. Both ship in `novaconium/lib/`, wired into the contact form, `/admin/clear-cache`, and `/admin/password-hash`. - **SQLite/MySQL database, zero setup** — `Lib\Db`, a thin PDO wrapper (no ORM) supporting multiple named connections open at once — e.g. this site's own SQLite data plus a MySQL connection to a legacy database, usable in the same sidecar — each with its own plain-SQL migration convention, applied automatically on first use or via `php novaconium/bin/migrate.php`. SQLite data lives in a project-owned top-level `data/` directory, outside both `public/` and `novaconium/`. See `/admin/docs/database`. - **Sessions with flash data** — `Lib\Session`, a thin wrapper around native PHP sessions with CodeIgniter-style flash values (set now, readable on exactly the next request) for post/redirect/GET flows without a query-string flag. Lazy-start, same mechanism `Lib\Csrf` already uses. See `/admin/docs/session`. +- **Content index: sitemap, search, tags** — `/sitemap.xml`, full-text `/search` (SQLite FTS5), and blog tag browsing all share one crawler that renders every page and harvests `keywords`/`tags`/`changefreq`/`priority` Twig blocks via Twig's own `renderBlock()` — no front-matter, no separate metadata files. Off by default (depends on SQLite); reindexes lazily on demand or via `php novaconium/bin/index-content.php`. See `/admin/docs/content-index`. - **No build step, no Composer** — clone it, point Apache (or `php -S`) at `public/`, and it runs. Twig is vendored as source; see `/admin/docs/upgrading-twig` for upgrading it. ## Getting started -**Requirements:** PHP 8.1+ (uses `readonly` constructor-promoted properties) with the `pdo_sqlite` extension (bundled with PHP, just needs to be enabled — no separate install; add `pdo_mysql` too if using a MySQL connection), and, for production, Apache with `mod_rewrite` and `AllowOverride All`. +**Requirements:** PHP 8.1+ (uses `readonly` constructor-promoted properties) with the `pdo_sqlite` extension (bundled with PHP, just needs to be enabled — no separate install; add `pdo_mysql` too if using a MySQL connection), and, for production, Apache with `mod_rewrite` and `AllowOverride All`. The content index's search (`/admin/docs/content-index`) additionally needs SQLite's FTS5 extension, bundled with `pdo_sqlite` on virtually every modern PHP build — only relevant if `content_index_enabled` is turned on. ### Run it locally (no Apache needed) @@ -79,7 +80,7 @@ php novaconium/bin/create-static-page.php blog/my-new-post ## Documentation -The full framework documentation — routing, sidecars, libraries, database, session, layouts, static caching, SEO, Matomo analytics, admin authentication, draft pages, styling, project layout, and third-party notices — lives at `/admin/docs` on any running instance (so it travels with the code, no internet connection needed). Highlights: +The full framework documentation — routing, sidecars, libraries, database, session, content index, layouts, static caching, SEO, Matomo analytics, admin authentication, draft pages, styling, project layout, and third-party notices — lives at `/admin/docs` on any running instance (so it travels with the code, no internet connection needed). Highlights: - [Getting started](http://127.0.0.1:8000/admin/docs/getting-started) - [Routing](http://127.0.0.1:8000/admin/docs/routing) @@ -87,6 +88,7 @@ The full framework documentation — routing, sidecars, libraries, database, ses - [Libraries](http://127.0.0.1:8000/admin/docs/libraries) - [Database](http://127.0.0.1:8000/admin/docs/database) - [Session](http://127.0.0.1:8000/admin/docs/session) +- [Content index](http://127.0.0.1:8000/admin/docs/content-index) - [Layouts](http://127.0.0.1:8000/admin/docs/layouts) - [Static caching](http://127.0.0.1:8000/admin/docs/caching) - [SEO](http://127.0.0.1:8000/admin/docs/seo) diff --git a/novaconium/ISSUES.md b/novaconium/ISSUES.md index e39da35..481c1d6 100644 --- a/novaconium/ISSUES.md +++ b/novaconium/ISSUES.md @@ -54,27 +54,19 @@ expected vs. actual behavior. For features, include the motivating use case.> Suggested build order (foundations first, since admin login builds on two of the others): -1. **Blog tags/categories** — no dependencies, but now needs a metadata - source design decision first (`PostRepository` was removed when - `hello-world`/`second-post` became plain Twig pages — see the entry - below), so worth doing first or together with Blog RSS feed. -2. **Blog RSS feed** — no hard dependency, but a per-tag feed is easiest - once tags/categories exist. -3. **Internal search** — SQLite groundwork it needed is done; ready to - build. -4. **XML sitemap** — no hard dependency, but shares crawling logic with - Internal search, so easiest right after (or alongside) it. -5. **Media/file manager** — no hard dependency; usable standalone, though +1. **Blog RSS feed** — no hard dependency; a per-tag feed is now easy to + add too, since tag browsing/the content index shipped (see Done). +2. **Media/file manager** — no hard dependency; usable standalone, though best gated behind admin login once that exists. -6. **Syntax highlighting on code blocks** — no hard dependency on the +3. **Syntax highlighting on code blocks** — no hard dependency on the copy button (see Done — shipped 2026-07-14), but touches the same `
` markup it did.
-7. **Admin login & user management** — SQLite groundwork and session
+4. **Admin login & user management** — SQLite groundwork and session
    handling it needed are both done; ready to build.
-8. **Ecommerce functionality** — needs admin login & user management
+5. **Ecommerce functionality** — needs admin login & user management
    (order/product admin, and customer accounts); SQLite groundwork and
    session handling (cart) it needed are both done.
-9. **Paywall functionality** — needs everything Ecommerce needs, plus
+6. **Paywall functionality** — needs everything Ecommerce needs, plus
    Ecommerce itself for the recurring-billing/payment-gateway plumbing;
    build after it rather than in parallel — also now has a concrete
    precedent to follow for the "gated content must skip the static cache"
@@ -82,34 +74,13 @@ of the others):
    the caching/auth standing rule in `AGENTS.md`), which was still an open
    question when this entry was originally written.
 
-MySQL support, Session handling (with flash sessions), and Draft pages
-(admin-only preview) all shipped 2026-07-14 — see Done.
+MySQL support, Session handling (with flash sessions), Draft pages
+(admin-only preview), and Blog tags/categories + Internal search + XML
+sitemap (shipped together as one content index — see Done) all shipped
+2026-07-14.
 
 See **Won't Do** below for 404 tracking, dropped in favor of Matomo.
 
-### Blog tags/categories
-
-- **Type:** Feature
-- **Status:** Backlog
-- **Priority:** Medium
-- **Added:** 2026-07-12
-
-Tag (or category) each blog post so posts can be browsed/filtered by
-topic — e.g. `App/pages/blog/tag/[tag]/index.php` listing matching posts,
-using the `[param]` route-capture mechanism (see `/admin/docs/routing`;
-this project's own `App/pages/blog/` doesn't currently use `[param]` for
-posts, every post is its own plain directory named after its slug).
-`Lib\PostRepository` (which used to back `hello-world`/`second-post`) was
-removed when those two posts became plain Twig pages, like
-`twig-syntax-guide` and `style-guide` — so there's no shared metadata
-store anymore. `App/pages/blog/index.php` now just hand-lists each
-post's slug/title/excerpt in a plain array; adding tags means adding a
-`tags` field to each entry there and a lookup by tag over that same
-array. Fine at current scale (4 posts); if the array grows unwieldy or
-tags need real querying, that's a SQLite-groundwork question — no need
-to block on it now. Once a feed exists, consider a per-tag feed too
-(e.g. `App/pages/blog/tag/[tag]/feed/index.php`).
-
 ### Blog RSS feed
 
 - **Type:** Feature
@@ -119,61 +90,21 @@ to block on it now. Once a feed exists, consider a per-tag feed too
 
 An RSS (or Atom) feed for the blog, e.g. `App/pages/blog/feed/index.php`
 returning `Response::xml(...)` — the sidecar contract already supports
-this, no new mechanism needed. `App/pages/blog/index.php` already
-hand-lists every post's slug/title/excerpt in a plain array (no
-`PostRepository` anymore — see the Blog tags/categories entry above for
-why); the remaining gap is a published-date field per entry so a feed
-can sort them, regardless of whether that array stays hardcoded or moves
-onto SQLite later. Should link from `` in the blog layout (or the root layout) for
-feed auto-discovery, and probably wants its own `App/pages/blog/_layout/`
-or a sidecar-only page — no `index.twig` needed since the sidecar returns
-XML directly (see `/admin/docs/sidecars`'s JSON-only example for the same
-pattern, just with `Response::json()` instead of `Response::xml()`).
-
-### Internal search
-
-- **Type:** Feature
-- **Status:** Backlog
-- **Priority:** Medium
-- **Depends on:** SQLite groundwork (Done)
-- **Added:** 2026-07-12
-
-Crawl the site's own pages (likely via `App/pages/` + rendered output,
-rather than an external HTTP crawl, to avoid hitting the static cache/
-`.htaccess` layer) and index the content into SQLite once the groundwork
-above exists, so a search box can query it — a `search` sidecar (e.g.
-`App/pages/search/index.php`) that reads the index and returns matching
-pages, no external search service. Needs a decision on crawl trigger
-(on-demand via an admin action vs. a cron-like re-crawl) and on indexing
-granularity (whole-page vs. per-section). Consider reusing the SQLite
-FTS5 extension if available, rather than hand-rolling text search.
-
-### XML sitemap
-
-- **Type:** Feature
-- **Status:** Backlog
-- **Priority:** Medium
-- **Added:** 2026-07-12
-
-Generate a `/sitemap.xml` listing every routable page (`App/pages/` +
-`novaconium/pages/` overlay, same resolution `Router`/`Overlay` already do)
-for search engine discovery — one more piece of the SEO groundwork already
-laid (`/admin/docs/seo`, canonical links, etc.). No hard dependency on
-SQLite: a first version can be built by walking `pages_dirs` directly
-(skipping reserved `_`/`404` segments, resolving `[param]` routes only if
-their concrete values are knowable from some data source — moot for
-`App/pages/blog/` today, since every post there is now a plain directory
-rather than a `[param]` route) the same way `Router::resolve()` walks it,
-with no separate crawl step.
-That said, it may be worth sharing a crawler with Internal search rather
-than building two separate page-enumeration mechanisms — if Internal
-search's crawler already walks and records every real URL (including
-resolved `[param]` values), the sitemap can just be a different
-serialization of that same data instead of its own logic. Worth deciding
-together when either is picked up. Should also exclude `/admin/docs/*` if
-it isn't publicly reachable (see admin authentication) and the `noindex`
-pages already marked via the `robots` block.
+this, no new mechanism needed. `App/pages/blog/index.php` still
+hand-lists every post's slug/title/excerpt in a plain array (unaffected by
+the content index shipped alongside Blog tags/categories/Internal
+search/XML sitemap below — that index is a derived, metadata-only layer,
+not a replacement for this array); the remaining gap is a published-date
+field per entry so a feed can sort them. Should link from `` in the blog layout (or the
+root layout) for feed auto-discovery, and probably wants its own
+`App/pages/blog/_layout/` or a sidecar-only page — no `index.twig` needed
+since the sidecar returns XML directly (see `/admin/docs/sidecars`'s
+JSON-only example for the same pattern, just with `Response::json()`
+instead of `Response::xml()`). Now that tag browsing exists
+(`/admin/docs/content-index`), consider a per-tag feed too (e.g.
+`App/pages/blog/tag/[tag]/feed/index.php`, querying `content_tags` the
+same way `App/pages/blog/tag/[tag]/index.php` already does).
 
 ### Media/file manager
 
@@ -301,6 +232,109 @@ _Nothing yet._
 
 ## Done
 
+### Content index: keywords, tags/categories, search, XML sitemap
+
+- **Type:** Feature
+- **Status:** Done
+- **Priority:** Medium
+- **Depends on:** SQLite groundwork (Done)
+- **Added:** 2026-07-12 (Blog tags/categories, Internal search, XML
+  sitemap entries) / 2026-07-14 (keywords, combined)
+- **Shipped:** 2026-07-14
+
+Shipped Blog tags/categories, Internal search, and XML sitemap together,
+plus a new meta-keywords request, as one feature rather than three —
+exactly the "worth deciding together when either is picked up" call this
+file made when XML sitemap was first written. All three (plus a fourth,
+new: ``) turned out to be one shared crawler with
+thin consumers, not separate mechanisms.
+
+Design: content stays in files (Twig pages, no CMS-style body-in-database
+— keeps the Hugo-style file-based-routing pitch intact). Per-page metadata
+is four Twig blocks in `novaconium/pages/_layout/layout.twig`, the same
+override mechanism already used for `title`/`description`/`og_*`:
+`keywords` (rendered, new `` tag), `tags` (comma-separated, not
+rendered), `changefreq`/`priority` (sitemap hints, not rendered). No
+front-matter, no separate metadata file convention. `App\ContentIndexer`
+(`novaconium/src/ContentIndexer.php`) crawls every routable page
+(`Overlay::listPageDirs()`, a new method — skips `_`/`404`/`[param]`
+directories, matching `Router::resolve()`'s reserved-segment rule and the
+original XML sitemap entry's stated V1 limitation that wildcard routes
+aren't crawled without a data source to resolve concrete values) and pulls
+each block's value via `Renderer::renderForIndex()` (new method) calling
+Twig's own `TemplateWrapper::renderBlock()` — not regex-parsing `.twig`
+source — so overrides and layout inheritance resolve exactly like a real
+render. Rendered HTML is `strip_tags()`-stripped into a SQLite FTS5 table
+for search. A page listed in `draft_routes` or whose `robots` block
+resolves to `noindex` is skipped entirely (never indexed), same convention
+`/admin/docs/seo` already documents for admin/internal pages.
+
+**Off by default** (`content_index_enabled`, default `false`) — all three
+consumers depend on SQLite, a real dependency plenty of sites built on
+this framework won't want, same reasoning that already keeps Matomo/admin
+auth off by default. Verified end-to-end that disabling it is a true
+zero-footprint no-op: no `data/novaconium.sqlite` gets created just
+because the feature exists in the codebase, and all three consumer routes
+404 exactly as if they didn't exist.
+
+Two trigger paths sharing one `reindex()`: lazy (`content_index_auto`,
+default `true` — a cheap mtime-staleness check on first touch of a
+consumer route, never on a normal page view) and explicit
+(`php novaconium/bin/index-content.php`, same shape as `bin/migrate.php`,
+ignores `content_index_auto`).
+
+**Two real bugs caught by testing, not review:**
+1. **Reentrancy** — the crawl renders every page, including `/search`
+   itself, whose own sidecar calls `ContentIndexer::ensureFresh()`;
+   without a guard this triggered a nested `reindex()` mid-transaction and
+   fataled on a second `PDO::beginTransaction()`. Fixed with a
+   `private static bool $indexing` guard checked at the top of both
+   `ensureFresh()` and `reindex()`.
+2. **Wrong PDO constant** (`PDO::KEY_PAIR` instead of
+   `PDO::FETCH_KEY_PAIR`) in the search sidecar, caught immediately by
+   actually hitting `/search` with a real query rather than trusting the
+   code read correctly.
+
+Also fixed two unrelated pre-existing bugs discovered while building this
+(both blocked/were adjacent to the crawler rendering every page for real):
+`novaconium/pages/admin/docs/sidecars/index.twig` had a literal
+un-escaped `{{ }}` in prose text that fataled Twig with a syntax error on
+any real render of that page (it had apparently never actually been
+visited before); and both that page and `Lib\Input`'s doc-comment still
+said "there's no database layer in this framework yet" despite `Lib\Db`
+having shipped weeks earlier.
+
+**`migrations_dir` (`Lib\Db`) now accepts an ordered list of roots, not
+just one path** — needed so the content index's schema
+(`novaconium/migrations/0001_create_content_index.sql`) could ship as a
+framework migration without colliding with project migrations in
+`App/migrations/`. This is the first framework-shipped migration, and the
+two-root extension point `AGENTS.md` flagged as a future need when SQLite
+groundwork shipped. Migrations are now tracked by path relative to the
+repo root (not bare filename) specifically to prevent two roots each
+containing a same-named file from shadowing one another —
+`realpath()`-normalized so a `migrations_dir` containing `..` (like the
+default connection's own `__DIR__ . '/../App/migrations'`) doesn't produce
+an ugly, unstable tracked name.
+
+New consumer routes: `novaconium/pages/sitemap.xml/index.php` (framework
+-default — confirmed a directory literally named `sitemap.xml` resolves
+correctly, since `Router` only splits on `/`), `novaconium/pages/search/`
+(framework-default, FTS5-backed, the search term wrapped as an escaped
+quoted phrase before binding — parameter binding stops SQL injection but
+not FTS5's own query-language parsing of the bound value, verified against
+a literal `"` and several FTS operator characters, not just assumed safe),
+and `App/pages/blog/tag/[tag]/` (project-owned, since `blog/` is project
+content — `App/pages/blog/index.php`'s hand-written post array is
+untouched, `content_tags` is a derived index on top of it, not a
+replacement). Added `{% block tags %}` to the 4 existing blog posts so tag
+browsing has real content to demonstrate against — an exception to the
+"ship mechanism only, no demo content" pattern of prior sessions, since
+here the target content already existed and tag browsing is meaningless
+to verify without it. Documented at `/admin/docs/content-index`, with
+supporting updates to `/admin/docs/seo`, `/admin/docs/database`, and
+`novaconium/bin/create-static-page.php`'s scaffolded template.
+
 ### Draft pages (admin-only preview)
 
 - **Type:** Feature
diff --git a/novaconium/bin/create-static-page.php b/novaconium/bin/create-static-page.php
index 1567b9c..5f826e9 100644
--- a/novaconium/bin/create-static-page.php
+++ b/novaconium/bin/create-static-page.php
@@ -72,6 +72,10 @@ $template = << [
             'driver'         => 'sqlite',
             'path'           => __DIR__ . '/../data/novaconium.sqlite',
-            'migrations_dir' => __DIR__ . '/../App/migrations',
+            'migrations_dir' => [
+                __DIR__ . '/migrations',
+                __DIR__ . '/../App/migrations',
+            ],
         ],
     ],
 
@@ -73,4 +80,20 @@ return [
     // since a world-readable cached copy would otherwise permanently leak
     // the draft the first time an admin previewed it.
     'draft_routes' => [],
+
+    // Content index (see /admin/docs/content-index) — backs /sitemap.xml,
+    // /search, and blog tag browsing. Off by default: all three depend on
+    // SQLite (Lib\Db), a real dependency plenty of sites built on this
+    // framework won't want at all, the same reasoning that keeps Matomo
+    // and admin auth off by default above. When false, all three routes
+    // 404 exactly as if they didn't exist, and nothing ever touches
+    // Lib\Db because of this feature — no data/novaconium.sqlite gets
+    // created just because the code exists. content_index_auto only
+    // matters once enabled: true (the default) reindexes lazily,
+    // on-demand, the first time a stale index is actually needed (never on
+    // a normal page view); false disables that and leaves indexing
+    // entirely to `php novaconium/bin/index-content.php`, e.g. from a
+    // deploy step.
+    'content_index_enabled' => false,
+    'content_index_auto'    => true,
 ];
diff --git a/novaconium/lib/Db.php b/novaconium/lib/Db.php
index 9322948..3032489 100644
--- a/novaconium/lib/Db.php
+++ b/novaconium/lib/Db.php
@@ -119,27 +119,44 @@ final class Db
     }
 
     /**
-     * Applies any *.sql file under $migrationsDir not yet recorded in this
-     * connection's own schema_migrations table, in filename order — so
-     * migrations are named with a numeric prefix (0001_create_x.sql,
-     * 0002_add_y.sql, ...) to control apply order. Each file is tracked by
-     * filename once applied and never re-run. Each connection tracks its
-     * own schema_migrations table in its own database, independent of any
-     * other configured connection. A connection with no migrations_dir set
-     * skips this entirely — e.g. a connection to a legacy database this
-     * project shouldn't manage schema for.
+     * Applies any *.sql file under $migrationsDirs not yet recorded in this
+     * connection's own schema_migrations table. $migrationsDirs is an
+     * ordered list of roots (a single string is accepted too, wrapped
+     * internally) — each root's own files are applied in filename order
+     * within that root (numeric prefixes, e.g. 0001_create_x.sql,
+     * 0002_add_y.sql, control order within a root), and roots are fully
+     * processed one at a time in the order given, not interleaved by
+     * filename across roots. This lets framework-shipped migrations (e.g.
+     * novaconium/migrations/) always apply before a project's own
+     * (App/migrations/) on the same connection — see
+     * novaconium/config.php's db_connections.default.migrations_dir and
+     * AGENTS.md.
      *
-     * Runs automatically on every first connection() call per process, per
-     * connection name — cheap (one query plus a directory glob), so no
-     * separate "migrate" step is required, matching the framework's
-     * zero-config philosophy elsewhere (e.g. static caching).
-     * novaconium/bin/migrate.php exists to run it explicitly for every
-     * configured connection (e.g. from a deploy script) without serving a
-     * request first.
+     * Each file is tracked once applied and never re-run, keyed by its path
+     * relative to the repo root (e.g. novaconium/migrations/0001_x.sql) —
+     * not bare filename, because two roots can each contain a same-named
+     * file (a framework migration and an unrelated project migration both
+     * numbered 0001_...); tracking by bare filename would make the second
+     * one seen look "already applied" and silently skip it. A relative
+     * path is also portable across environments, unlike a full absolute
+     * path, which would make every migration look "new" again after a
+     * clone/deploy to a different directory.
+     *
+     * A connection with no migrations_dir set skips this entirely — e.g. a
+     * connection to a legacy database this project shouldn't manage schema
+     * for. Runs automatically on every first connection() call per
+     * process, per connection name — cheap (one query plus a directory
+     * glob per root), so no separate "migrate" step is required, matching
+     * the framework's zero-config philosophy elsewhere (e.g. static
+     * caching). novaconium/bin/migrate.php exists to run it explicitly for
+     * every configured connection (e.g. from a deploy script) without
+     * serving a request first.
+     *
+     * @param string|string[]|null $migrationsDirs
      */
-    private static function migrate(PDO $pdo, ?string $migrationsDir): void
+    private static function migrate(PDO $pdo, string|array|null $migrationsDirs): void
     {
-        if ($migrationsDir === null) {
+        if ($migrationsDirs === null) {
             return;
         }
 
@@ -150,26 +167,38 @@ final class Db
             ')'
         );
 
-        if (!is_dir($migrationsDir)) {
-            return;
-        }
-
         $applied = $pdo->query('SELECT filename FROM schema_migrations')->fetchAll(PDO::FETCH_COLUMN);
         $applied = array_flip($applied);
 
-        $files = glob(rtrim($migrationsDir, '/') . '/*.sql') ?: [];
-        sort($files);
+        // novaconium/lib/ -> novaconium/ -> repo root, two levels up.
+        $repoRoot = rtrim(realpath(dirname(__DIR__, 2)) ?: dirname(__DIR__, 2), '/') . '/';
 
-        foreach ($files as $file) {
-            $filename = basename($file);
-            if (isset($applied[$filename])) {
+        foreach ((array) $migrationsDirs as $dir) {
+            if (!is_dir($dir)) {
                 continue;
             }
 
-            $pdo->exec((string) file_get_contents($file));
+            $files = glob(rtrim($dir, '/') . '/*.sql') ?: [];
+            sort($files);
 
-            $insert = $pdo->prepare('INSERT INTO schema_migrations (filename, applied_at) VALUES (?, ?)');
-            $insert->execute([$filename, gmdate('Y-m-d\TH:i:s\Z')]);
+            foreach ($files as $file) {
+                // realpath() resolves any ".." left over from a
+                // migrations_dir like __DIR__ . '/../App/migrations' (glob()
+                // doesn't normalize the paths it returns), so the tracked
+                // name is clean, e.g. "App/migrations/0001_x.sql" rather
+                // than "novaconium/../App/migrations/0001_x.sql".
+                $resolved = realpath($file) ?: $file;
+                $trackedName = str_starts_with($resolved, $repoRoot) ? substr($resolved, strlen($repoRoot)) : basename($file);
+
+                if (isset($applied[$trackedName])) {
+                    continue;
+                }
+
+                $pdo->exec((string) file_get_contents($file));
+
+                $insert = $pdo->prepare('INSERT INTO schema_migrations (filename, applied_at) VALUES (?, ?)');
+                $insert->execute([$trackedName, gmdate('Y-m-d\TH:i:s\Z')]);
+            }
         }
     }
 
diff --git a/novaconium/lib/Input.php b/novaconium/lib/Input.php
index 320dc37..af7f287 100644
--- a/novaconium/lib/Input.php
+++ b/novaconium/lib/Input.php
@@ -13,10 +13,10 @@ namespace Lib;
  * novaconium/src/Renderer.php), so this cleaning is a second layer, not the
  * only one. The real defense against SQL injection is parameterized queries
  * (PDO prepared statements) — never string concatenation or
- * sanitize-then-interpolate, however "cleaned" the input looks. There's no
- * database layer in this framework yet (SQLite groundwork is tracked as
- * Backlog in novaconium/ISSUES.md); when one lands, use PDO prepared statements
- * exclusively. This class deliberately does not (and will not) expose an
+ * sanitize-then-interpolate, however "cleaned" the input looks. Lib\Db (see
+ * /admin/docs/database) is the database layer — its query() method uses
+ * PDO prepared statements exclusively for this reason. This class
+ * deliberately does not (and will not) expose an
  * "sqlSafe()"-style method — no string transform makes arbitrary input safe
  * to concatenate into SQL, and a method implying otherwise would be actively
  * dangerous.
diff --git a/novaconium/migrations/0001_create_content_index.sql b/novaconium/migrations/0001_create_content_index.sql
new file mode 100644
index 0000000..d556e04
--- /dev/null
+++ b/novaconium/migrations/0001_create_content_index.sql
@@ -0,0 +1,25 @@
+CREATE TABLE IF NOT EXISTS content_pages (
+    route TEXT PRIMARY KEY,
+    title TEXT NOT NULL,
+    description TEXT NOT NULL,
+    keywords TEXT NOT NULL DEFAULT '',
+    changefreq TEXT NOT NULL DEFAULT '',
+    priority TEXT NOT NULL DEFAULT '',
+    source_mtime INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS content_tags (
+    route TEXT NOT NULL,
+    tag TEXT NOT NULL,
+    PRIMARY KEY (route, tag)
+);
+
+CREATE INDEX IF NOT EXISTS idx_content_tags_tag ON content_tags(tag);
+
+CREATE VIRTUAL TABLE IF NOT EXISTS content_search USING fts5(route UNINDEXED, title, body);
+
+CREATE TABLE IF NOT EXISTS content_index_meta (
+    id INTEGER PRIMARY KEY CHECK (id = 1),
+    newest_source_mtime INTEGER NOT NULL,
+    indexed_at TEXT NOT NULL
+);
diff --git a/novaconium/pages/_layout/layout.twig b/novaconium/pages/_layout/layout.twig
index 497bae2..a726312 100644
--- a/novaconium/pages/_layout/layout.twig
+++ b/novaconium/pages/_layout/layout.twig
@@ -7,9 +7,19 @@
     {% block title %}{{ site_name }}{% endblock %}
     
     
+    
     
     
 
+    {# tags/changefreq/priority are never output as HTML — declared here
+       only so they're overridable per-page (same mechanism as every SEO
+       block above) and harvestable by ContentIndexer via Twig's
+       renderBlock() API, not rendered into the page itself. See
+       /admin/docs/content-index. #}
+    {% block tags %}{% endblock %}
+    {% block changefreq %}monthly{% endblock %}
+    {% block priority %}0.5{% endblock %}
+
     {# Open Graph / Facebook #}
     
     
diff --git a/novaconium/pages/admin/docs/_layout/layout.twig b/novaconium/pages/admin/docs/_layout/layout.twig
index d47470f..7ead60b 100644
--- a/novaconium/pages/admin/docs/_layout/layout.twig
+++ b/novaconium/pages/admin/docs/_layout/layout.twig
@@ -15,6 +15,7 @@
                 
  • {{ icons.book() }}Configuration
    • 
                 
  • {{ icons.book() }}Database
    • 
                 
  • {{ icons.book() }}Session
    • 
+                
  • {{ icons.search() }}Content index
    • 
                 
  • {{ icons.lock() }}Admin authentication
    • 
                 
  • {{ icons.lock() }}Draft pages
    • 
                 
  • {{ icons.book() }}Layouts
    • 
diff --git a/novaconium/pages/admin/docs/content-index/index.twig b/novaconium/pages/admin/docs/content-index/index.twig
new file mode 100644
index 0000000..edfbf06
--- /dev/null
+++ b/novaconium/pages/admin/docs/content-index/index.twig
@@ -0,0 +1,84 @@
+{% extends 'admin/docs/_layout/layout.twig' %}
+
+{% import '_layout/icons.twig' as icons %}
+
+{% block title %}Content index{% endblock %}
+
+{% block description %}The shared crawler behind /sitemap.xml, /search, and blog tag browsing.{% endblock %}
+
+{% block robots %}noindex, nofollow{% endblock %}
+
+{% block docs_content %}
+    
    Content index
    
+
+    
    Three features — {{ icons.sitemap() }}/sitemap.xml, {{ icons.search() }}/search, and blog {{ icons.tag() }}tag browsing — share one underlying mechanism rather than three separate ones: a crawler (App\ContentIndexer, novaconium/src/ContentIndexer.php) that renders every routable page, pulls its metadata, and stores it in SQLite.
    
+
+    
    Content itself stays in files — plain Twig pages, same as everywhere else in this framework. Nothing here moves a page's body into a database; only metadata is extracted and indexed.
    
+
+    
    Off by default
    
+
+    
    All three features depend on {{ icons.book() }}SQLite — a real dependency plenty of sites built on this framework won't want at all, the same reasoning that keeps Matomo and admin authentication off until a project opts in. content_index_enabled (default false) gates the whole subsystem:
    
+
+    
    <?php
+// App/config.php
+return [
+    'content_index_enabled' => true,
+];
    
+
+    
    When it's off, /sitemap.xml, /search, and every /blog/tag/<tag> route return a plain 404 — exactly as if they didn't exist — and nothing ever touches Lib\Db because of this feature. data/novaconium.sqlite isn't created just because the code is present in the codebase; each consumer checks the flag before constructing anything that would open a connection.
    
+
+    
    Declaring metadata on a page
    
+
+    
    Four Twig blocks, declared in novaconium/pages/_layout/layout.twig next to the rest of the SEO blocks — same override mechanism, just harvested rather than always rendered:
    
+
+    
+        
+            
+        
+        
+            
+            
+            
+            
+        
+    
    BlockDefaultUsed for
    keywordsemptyrendered as <meta name="keywords"> on every page (see SEO)
    tagsemptycomma-separated; indexed into content_tags for tag browsing
    changefreqmonthly/sitemap.xml's <changefreq>
    priority0.5/sitemap.xml's <priority>
    
+
+    
    {% verbatim %}{% block tags %}yellow, the best, summer, hot{% endblock %}
+{% block changefreq %}weekly{% endblock %}
+{% block priority %}1.0{% endblock %}{% endverbatim %}
    
+
+    
    See any post under App/pages/blog/ for a working tags example.
    
+
+    
    How the crawl works
    
+
+    
    ContentIndexer::reindex() walks every page under both page roots (Overlay::listPageDirs(), skipping _-prefixed, 404, and [param]-wildcard directories — a wildcard route's concrete values aren't knowable without a data source, so dynamic routes aren't crawled yet), renders each one via Renderer::renderForIndex() (the same sidecar-then-Twig path a real request takes, minus HTTP output and the static cache write), and pulls each metadata block via Twig's own renderBlock() API — not by parsing .twig source — so App-over-novaconium overrides and layout inheritance resolve exactly the way they do for a real visit.
    
+
+    
    A page is skipped entirely (not stored) if it's listed in {{ icons.lock() }}draft_routes, or if its resolved robots block contains noindex — the same convention SEO already documents for admin/internal pages. The rendered HTML is stripped with strip_tags() for a plain-text copy stored in a SQLite FTS5 virtual table for search.
    
+
+    
    Every reindex is a full rebuild, not incremental — all three tables are truncated and repopulated inside one transaction, simple and correct at this scale rather than trying to diff what changed. Because the crawl renders every page's sidecar as a plain GET (forcing $_SERVER['REQUEST_METHOD'] to 'GET' for the duration, regardless of what triggered the reindex, and restoring it afterward), a POST-guarded sidecar action is never accidentally triggered by indexing — the same HTTP-safe-method hygiene any GET handler is already expected to have.
    
+
+    
    When it runs
    
+
+    
    Two paths, both calling the same reindex():
    
+
+    
      
+        
    • Lazy (default): ContentIndexer::ensureFresh(), called from the /sitemap.xml, /search, and tag-browsing sidecars themselves — never from a normal page view, so browsing the rest of the site never pays any indexing cost. It compares the newest source-file mtime across every page (a cheap stat pass, no rendering) against the last indexed time, and only reindexes if something actually changed. Set content_index_auto to false to disable this and rely only on the CLI below.
      • 
+        
    • Explicit: php novaconium/bin/index-content.php — same shape as bin/migrate.php, for a deploy step. Always reindexes (ignores content_index_auto); exits immediately if content_index_enabled is false.
      • 
+    
    
+
+    
    Search
    
+
+    
    /search?q=... runs an FTS5 MATCH query. The search term is wrapped as a quoted phrase (embedded " doubled) before binding — parameter binding stops SQL injection, but the bound value is still parsed as its own FTS5 query-language expression, so an unescaped " or FTS operator in user input could otherwise throw a syntax error or search for something unintended.
    
+
+    
    Sitemap
    
+
+    
    /sitemap.xml lists every indexed page's <loc>, <lastmod> (the page's own source file mtime), <changefreq>, and <priority>.
    
+
+    
    Blog tag browsing
    
+
+    
    App/pages/blog/tag/[tag]/index.php — project-owned, since blog/ itself is project content — uses the {{ icons.link() }}[param] capture to read the tag from the URL and queries content_tags joined to content_pages. App/pages/blog/index.php's own hand-written post list is untouched by any of this — it stays the source of truth for the main blog listing; content_tags is a derived index built from each post's own tags block, not a replacement for it.
    
+
+    
    Migrations, and the two-root scan
    
+
+    
    The schema (content_pages, content_tags, content_search, content_index_meta) ships as a framework migration, novaconium/migrations/0001_create_content_index.sql — the first framework-owned migration, and the reason {{ icons.book() }}migrations_dir now accepts an ordered list of roots instead of a single path: the default connection's migrations_dir is [novaconium/migrations, App/migrations], so framework migrations always apply before a project's own on the same connection.
    
+{% endblock %}
diff --git a/novaconium/pages/admin/docs/database/index.twig b/novaconium/pages/admin/docs/database/index.twig
index 4ac2121..6206d7d 100644
--- a/novaconium/pages/admin/docs/database/index.twig
+++ b/novaconium/pages/admin/docs/database/index.twig
@@ -41,7 +41,10 @@ $legacyRows = Db::query('SELECT * FROM widgets', [], 'legacy')->fetchAll();
    @@ -66,13 +69,13 @@ return [

    This is the one config key in the project that doesn't follow the usual shallow-merge rule. Every other App/config.php key replaces the framework default outright (see {{ icons.book() }}Configuration) — but a plain shallow merge on db_connections would let the snippet above silently delete the framework's default connection just by adding legacy. So Lib\Db merges db_connections one level deeper, by connection name: the example above ends up with both default (SQLite, from the framework) and legacy (MySQL, from App/config.php) configured at once. To actually replace default, redeclare a default key yourself.

    -

    Only 'sqlite' and 'mysql' are implemented as driver values. migrations_dir is optional per connection — omit it to never run migrations against that connection (e.g. a legacy database this project shouldn't manage schema for).

    +

    Only 'sqlite' and 'mysql' are implemented as driver values. migrations_dir is optional per connection — omit it to never run migrations against that connection (e.g. a legacy database this project shouldn't manage schema for) — and accepts either a single path (legacy's example above) or an ordered list of roots (the default connection's example above), each scanned for its own *.sql files.

    The default connection's path lives in a top-level data/ directory — a sibling of App/, novaconium/, and public/, not nested inside any of them. This is deliberate: it can't live under public/ (would be directly web-accessible), and it can't live under novaconium/ either, since {{ icons.book() }}updating the framework means overwriting that whole directory — anything persisted there would be destroyed by the next update. data/ is project-owned, like App/, and untouched by a framework update. Its contents (*.sqlite and the SQLite journal/WAL/SHM sidecar files) are gitignored; only a .gitkeep is tracked so the directory exists in a fresh clone.

    Migrations

    -

    Plain .sql files under each connection's own migrations_dir, applied in filename order — name them with a numeric prefix to control ordering:

    +

    Plain .sql files under each connection's own migrations_dir root(s), applied in filename order within each root — name them with a numeric prefix to control ordering:

    -- App/migrations/0001_create_posts.sql
 CREATE TABLE posts (
@@ -81,9 +84,9 @@ CREATE TABLE posts (
     body TEXT NOT NULL
 );
    -

    Each file is tracked by filename in that connection's own schema_migrations table (created automatically in that connection's database) and only ever run once — default and legacy each track their own applied migrations independently. Migrations for a given connection apply automatically the first time it's used in a process — zero-config, the same "just works" philosophy as static caching — or explicitly for every configured connection at once, without serving a request first:

    +

    Each file is tracked by its path relative to the repo root (e.g. novaconium/migrations/0001_x.sql) in that connection's own schema_migrations table (created automatically in that connection's database) and only ever run once — default and legacy each track their own applied migrations independently. Tracking the repo-relative path rather than the bare filename matters once a connection has more than one migrations_dir root: two roots can each contain a same-named file (e.g. a framework 0001_...sql and an unrelated project 0001_...sql) without one being mistaken for the other already having run. Migrations for a given connection apply automatically the first time it's used in a process — zero-config, the same "just works" philosophy as static caching — or explicitly for every configured connection at once, without serving a request first:

    php novaconium/bin/migrate.php
    -

    Point two connections' migrations_dir at different directories (e.g. App/migrations/ for default, App/migrations/legacy/ for legacy) if their SQL genuinely diverges between drivers; otherwise the same directory works for both as long as the SQL in it is portable. The framework itself ships no core tables yet, so there's no second novaconium/migrations/ root to merge in — if a future framework feature needs a shipped migration (e.g. admin login's user table), this can extend to the same App-over-novaconium two-root scan used for pages and lib.

    +

    When a connection has multiple migrations_dir roots, each root is fully processed in the order given — the default connection's own framework root (novaconium/migrations/) always applies completely before its project root (App/migrations/), not interleaved by filename across the two. novaconium/migrations/0001_create_content_index.sql (see {{ icons.search() }}Content index) is the first framework-shipped migration — the reason this two-root support exists at all, extending the same App-over-novaconium override pattern used for pages and lib to migrations too. Point two connections' migrations_dir at different directories (e.g. App/migrations/ for default, App/migrations/legacy/ for legacy) if their SQL genuinely diverges between drivers; otherwise the same directory works for both as long as the SQL in it is portable.

    {% endblock %} diff --git a/novaconium/pages/admin/docs/index.twig b/novaconium/pages/admin/docs/index.twig index 3bf8c32..c097ff7 100644 --- a/novaconium/pages/admin/docs/index.twig +++ b/novaconium/pages/admin/docs/index.twig @@ -4,7 +4,7 @@ {% block title %}Docs{% endblock %} -{% block description %}Framework documentation: routing, sidecars, forms, libraries, database, session, admin authentication, draft pages, layouts, caching, styling.{% endblock %} +{% block description %}Framework documentation: routing, sidecars, forms, libraries, database, session, content index, admin authentication, draft pages, layouts, caching, styling.{% endblock %} {% block robots %}noindex, nofollow{% endblock %} @@ -20,6 +20,7 @@
  • {{ icons.book() }}Configuration — override framework settings from App/config.php.
  • {{ icons.book() }}Database — Lib\Db, a thin PDO wrapper (SQLite and MySQL) with named, simultaneous connections and a plain-SQL migration convention.
  • {{ icons.book() }}Session — Lib\Session, a thin wrapper around native PHP sessions, with CodeIgniter-style flash data.
    • +
  • {{ icons.search() }}Content index — the shared crawler behind /sitemap.xml, /search, and blog tag browsing. Off by default.
  • {{ icons.lock() }}Admin authentication — gate /admin/* behind HTTP Basic Auth, reusable for any future admin page.
  • {{ icons.lock() }}Draft pages — let an admin preview a page before the public can see it, reusing the same auth check.
  • {{ icons.book() }}Layouts — pages and layouts are overridable, just like Lib\.
    • diff --git a/novaconium/pages/admin/docs/seo/index.twig b/novaconium/pages/admin/docs/seo/index.twig index 3fc2c4f..db24dc0 100644 --- a/novaconium/pages/admin/docs/seo/index.twig +++ b/novaconium/pages/admin/docs/seo/index.twig @@ -9,7 +9,7 @@ {% block docs_content %}

    SEO boilerplate

    -

    novaconium/pages/_layout/layout.twig (the root layout every page extends, directly or via a nested layout) renders a full set of SEO meta tags in <head>: viewport, description, robots, canonical link, Open Graph, and Twitter Card. Each piece is a named Twig block with a sensible default, so any page can override just the piece it needs without touching the rest of <head>.

    +

    novaconium/pages/_layout/layout.twig (the root layout every page extends, directly or via a nested layout) renders a full set of SEO meta tags in <head>: viewport, description, robots, keywords, canonical link, Open Graph, and Twitter Card. Each piece is a named Twig block with a sensible default, so any page can override just the piece it needs without touching the rest of <head>. Three more blocks — tags, changefreq, priority — are declared the same way but never rendered into the page at all; see Content index for what reads them.

    Blocks you can override

    @@ -21,6 +21,10 @@ titlesite_name config value<title>, and reused by og:title / twitter:title descriptiongeneric site description<meta name="description">, and reused by og:description / twitter:description robotsindex, follow<meta name="robots"> + keywordsempty<meta name="keywords"> + tagsemptynot rendered — comma-separated, harvested by the content index for blog tag browsing + changefreqmonthlynot rendered — harvested for /sitemap.xml's <changefreq> + priority0.5not rendered — harvested for /sitemap.xml's <priority> canonical{{ '{{ request_path }}' }}<link rel="canonical">, and reused by og:url og_typewebsite<meta property="og:type"> og_title{{ '{{ block(\'title\') }}' }}<meta property="og:title"> @@ -60,6 +64,10 @@ {% block description %}One or two sentences describing this page.{% endblock %} {% block robots %}index, follow{% endblock %} +{% block keywords %}{% endblock %} +{% block tags %}{% endblock %} +{% block changefreq %}monthly{% endblock %} +{% block priority %}0.5{% endblock %} {% block canonical %}{{ request_path|default('/') }}{% endblock %} {% block og_type %}website{% endblock %} diff --git a/novaconium/pages/admin/docs/sidecars/index.twig b/novaconium/pages/admin/docs/sidecars/index.twig index 632a2fb..88a373f 100644 --- a/novaconium/pages/admin/docs/sidecars/index.twig +++ b/novaconium/pages/admin/docs/sidecars/index.twig @@ -63,7 +63,7 @@ $all = Input::post(); // the whole cleaned $_POST array    <

    Calling post()/get() with no key returns the entire cleaned array (handy for passing straight to SpamGuard::isSpam(), as below); calling it with a key returns that key's cleaned value, or the given default if it's absent. Nested arrays (e.g. a checkbox group posted as tags[]) are cleaned recursively. The result is memoized per request, so calling Input::post() repeatedly across a sidecar doesn't re-clean the superglobal each time.

    -

    This is not SQL-injection protection. The cleaning Input does (trim + strip tags, via Lib\Validate::clean(), plus null-byte stripping) is defense-in-depth against HTML/script injection in output contexts — Twig already autoescapes {{ }} output by default (see novaconium/src/Renderer.php), so this is a second layer, not the only one. No string transform makes arbitrary input safe to concatenate into a SQL query; the real defense is parameterized queries (PDO prepared statements). There's no database layer in this framework yet (SQLite groundwork is Backlog — see novaconium/ISSUES.md); when one lands, use prepared statements exclusively. Input deliberately has no sqlSafe()-style method, since a method implying "cleaned = safe to interpolate into SQL" would be actively dangerous.

    +

    This is not SQL-injection protection. The cleaning Input does (trim + strip tags, via Lib\Validate::clean(), plus null-byte stripping) is defense-in-depth against HTML/script injection in output contexts — Twig already autoescapes {{ '{{ }}' }} output by default (see novaconium/src/Renderer.php), so this is a second layer, not the only one. No string transform makes arbitrary input safe to concatenate into a SQL query; the real defense is parameterized queries. Lib\Db's query() method uses PDO prepared statements exclusively for exactly this reason. Input deliberately has no sqlSafe()-style method, since a method implying "cleaned = safe to interpolate into SQL" would be actively dangerous.

    One documented exception: a field that needs an exact, unmodified value — a password about to be hashed, say — should read $_POST directly instead of going through Input::post(). Cleaning would silently strip characters like </> before hashing, producing a hash that doesn't match what's actually typed later. See novaconium/pages/admin/password-hash/index.php for the one place this framework does that on purpose.

    diff --git a/novaconium/pages/search/index.php b/novaconium/pages/search/index.php new file mode 100644 index 0000000..6bd563f --- /dev/null +++ b/novaconium/pages/search/index.php @@ -0,0 +1,103 @@ +fetchAll(PDO::FETCH_ASSOC); + + // A second query rather than joining content_pages into the FTS5 + // query directly — content_search is a virtual table, and mixing a + // real table JOIN into an FTS5 MATCH query is more fragile than doing + // the description lookup separately. IN (...) with one placeholder + // per route, built from the exact route set the first query returned + // — never string-interpolating $query or user input into this SQL, + // only the already-fetched, already-trusted route values. + if ($results !== []) { + $routes = array_column($results, 'route'); + $placeholders = implode(',', array_fill(0, count($routes), '?')); + $descriptions = Db::query( + "SELECT route, description FROM content_pages WHERE route IN ({$placeholders})", + $routes + )->fetchAll(PDO::FETCH_KEY_PAIR); + + foreach ($results as &$result) { + $result['description'] = $descriptions[$result['route']] ?? ''; + } + unset($result); // break the foreach-by-reference alias, standard PHP gotcha + } +} + +// $results stays [] (not an error) for a blank query or one with zero +// matches — the twig template only distinguishes those two cases by +// checking $query itself, not by any error flag. +return [ + 'query' => $query, + 'results' => $results, +]; diff --git a/novaconium/pages/search/index.twig b/novaconium/pages/search/index.twig new file mode 100644 index 0000000..45509ec --- /dev/null +++ b/novaconium/pages/search/index.twig @@ -0,0 +1,31 @@ +{% extends layout %} + +{% import '_layout/icons.twig' as icons %} + +{% block title %}Search{% endblock %} +{% block description %}Search this site.{% endblock %} +{% block robots %}noindex, follow{% endblock %} + +{% block content %} +

    {{ icons.search() }}Search

    + +
    + + +
    + + {% if query != '' %} + {% if results|length > 0 %} + + {% else %} +

    No results for “{{ query }}”.

    + {% endif %} + {% endif %} +{% endblock %} diff --git a/novaconium/pages/sitemap.xml/index.php b/novaconium/pages/sitemap.xml/index.php new file mode 100644 index 0000000..fb8dfe5 --- /dev/null +++ b/novaconium/pages/sitemap.xml/index.php @@ -0,0 +1,73 @@ + so it reflects when the content actually last changed, not +// when the index was last rebuilt. +$pages = Db::query('SELECT route, changefreq, priority, source_mtime FROM content_pages ORDER BY route') + ->fetchAll(PDO::FETCH_ASSOC); + +// Plain string concatenation rather than DOMDocument — this site's scale +// doesn't need a real XML builder, but every value is still +// htmlspecialchars()-escaped (ENT_XML1, not the HTML default) since +// route/changefreq/priority all ultimately come from page-author-controlled +// Twig blocks, not hardcoded constants. +$xml = '' . "\n"; +$xml .= '' . "\n"; + +foreach ($pages as $page) { + $xml .= ' ' . "\n"; + $xml .= ' ' . htmlspecialchars($page['route'], ENT_XML1) . '' . "\n"; + $xml .= ' ' . gmdate('Y-m-d', (int) $page['source_mtime']) . '' . "\n"; + $xml .= ' ' . htmlspecialchars($page['changefreq'], ENT_XML1) . '' . "\n"; + $xml .= ' ' . htmlspecialchars($page['priority'], ENT_XML1) . '' . "\n"; + $xml .= ' ' . "\n"; +} + +$xml .= '' . "\n"; + +return Response::xml($xml); diff --git a/novaconium/src/ContentIndexer.php b/novaconium/src/ContentIndexer.php new file mode 100644 index 0000000..cecf617 --- /dev/null +++ b/novaconium/src/ContentIndexer.php @@ -0,0 +1,234 @@ +beginTransaction(); + + $pdo->exec('DELETE FROM content_pages'); + $pdo->exec('DELETE FROM content_tags'); + $pdo->exec('DELETE FROM content_search'); + + $insertPage = $pdo->prepare( + 'INSERT INTO content_pages (route, title, description, keywords, changefreq, priority, source_mtime) ' . + 'VALUES (?, ?, ?, ?, ?, ?, ?)' + ); + $insertTag = $pdo->prepare('INSERT INTO content_tags (route, tag) VALUES (?, ?)'); + $insertSearch = $pdo->prepare('INSERT INTO content_search (route, title, body) VALUES (?, ?, ?)'); + + $newestMtime = 0; + + foreach ($routes as $dir) { + if (in_array($dir, $config['draft_routes'], true)) { + continue; + } + + $mtime = self::sourceMtime($config['pages_dirs'], $dir); + $newestMtime = max($newestMtime, $mtime); + + $route = new Route($dir, [], true); + $indexed = $renderer->renderForIndex($route); + + if ($indexed === null || str_contains($indexed->robots, 'noindex')) { + continue; + } + + $routeUrl = $dir === '' ? '/' : '/' . $dir; + + $insertPage->execute([ + $routeUrl, + $indexed->title, + $indexed->description, + $indexed->keywords, + $indexed->changefreq, + $indexed->priority, + $mtime, + ]); + + foreach (self::splitList($indexed->tags) as $tag) { + $insertTag->execute([$routeUrl, $tag]); + } + + $insertSearch->execute([$routeUrl, $indexed->title, strip_tags($indexed->html)]); + } + + $pdo->prepare('DELETE FROM content_index_meta')->execute(); + $pdo->prepare('INSERT INTO content_index_meta (id, newest_source_mtime, indexed_at) VALUES (1, ?, ?)') + ->execute([$newestMtime, gmdate('Y-m-d\TH:i:s\Z')]); + + $pdo->commit(); + } catch (\Throwable $e) { + if ($pdo->inTransaction()) { + $pdo->rollBack(); + } + throw $e; + } finally { + if ($originalMethod === null) { + unset($_SERVER['REQUEST_METHOD']); + } else { + $_SERVER['REQUEST_METHOD'] = $originalMethod; + } + self::$indexing = false; + } + } + + /** + * @param array $config + */ + private static function isStale(array $config): bool + { + $pdo = Db::connection(); + + $meta = $pdo->query('SELECT newest_source_mtime FROM content_index_meta WHERE id = 1')->fetch(); + if ($meta === false) { + return true; + } + + $newest = 0; + foreach (Overlay::listPageDirs($config['pages_dirs']) as $dir) { + $newest = max($newest, self::sourceMtime($config['pages_dirs'], $dir)); + } + + return $newest > (int) $meta['newest_source_mtime']; + } + + /** + * @param string[] $pagesDirs + */ + private static function sourceMtime(array $pagesDirs, string $dir): int + { + $twig = Overlay::findFile($pagesDirs, $dir === '' ? 'index.twig' : $dir . '/index.twig'); + $php = Overlay::findFile($pagesDirs, $dir === '' ? 'index.php' : $dir . '/index.php'); + + $mtimes = array_filter([ + $twig !== null ? filemtime($twig) : false, + $php !== null ? filemtime($php) : false, + ]); + + return $mtimes === [] ? 0 : (int) max($mtimes); + } + + /** + * @return string[] + */ + private static function splitList(string $value): array + { + $items = array_map('trim', explode(',', $value)); + + return array_values(array_filter($items, fn (string $item) => $item !== '')); + } + + /** + * @param array $config + */ + private static function renderer(array $config): Renderer + { + return new Renderer($config['pages_dirs'], new Cache($config['cache_dir'])); + } + + /** + * @return array + */ + private static function config(): array + { + $config = require __DIR__ . '/../config.php'; + + $appConfigFile = __DIR__ . '/../../App/config.php'; + if (is_file($appConfigFile)) { + $appConfig = require $appConfigFile; + $defaultConnections = $config['db_connections']; + $appConnections = $appConfig['db_connections'] ?? []; + $config = array_merge($config, $appConfig); + $config['db_connections'] = array_merge($defaultConnections, $appConnections); + } + + return $config; + } +} diff --git a/novaconium/src/IndexedPage.php b/novaconium/src/IndexedPage.php new file mode 100644 index 0000000..1b00978 --- /dev/null +++ b/novaconium/src/IndexedPage.php @@ -0,0 +1,26 @@ + $found + */ + private static function walk(string $root, string $relative, array &$found): void + { + $dir = self::join($root, $relative); + + if (!is_dir($dir)) { + return; + } + + if (is_file($dir . '/index.twig') || is_file($dir . '/index.php')) { + $found[$relative] = true; + } + + foreach (scandir($dir) ?: [] as $entry) { + if ($entry === '.' || $entry === '..' || !is_dir($dir . '/' . $entry)) { + continue; + } + + if ($entry[0] === '_' || $entry[0] === '[' || $entry === '404') { + continue; + } + + $childRelative = $relative === '' ? $entry : $relative . '/' . $entry; + self::walk($root, $childRelative, $found); + } + } + private static function join(string $root, string $relative): string { $root = rtrim($root, '/'); diff --git a/novaconium/src/Renderer.php b/novaconium/src/Renderer.php index aac2057..3ef8e1c 100644 --- a/novaconium/src/Renderer.php +++ b/novaconium/src/Renderer.php @@ -83,6 +83,47 @@ final class Renderer } } + /** + * Renders $route the same way render() does (runs its sidecar if any, + * resolves the nearest layout) but returns the result instead of + * emitting HTTP output or writing the static cache — used by + * App\ContentIndexer to crawl the site for /sitemap.xml, /search, and + * blog tag browsing (see /admin/docs/content-index). Returns null if + * the sidecar returns a Response (a redirect/JSON/XML endpoint has + * nothing meaningful to index). There's no real request URI to derive + * request_path from here, so it's synthesized from the route itself. + */ + public function renderForIndex(Route $route): ?IndexedPage + { + $sidecarRel = $this->withFile($route->dir, 'index.php'); + $sidecar = Overlay::findFile($this->pagesDirs, $sidecarRel); + $hasSidecar = $sidecar !== null; + + $result = $hasSidecar ? $this->runSidecar($sidecar, $route->params) : []; + + if ($result instanceof Response) { + return null; + } + + $data = array_merge((array) $result, ['params' => $route->params]); + $data['layout'] = $this->relativeLayoutPath($route->dir); + $data['request_path'] = $route->dir === '' ? '/' : '/' . $route->dir; + + $templateName = $this->withFile($route->dir, 'index.twig'); + $wrapper = $this->twig->load($templateName); + + return new IndexedPage( + html: $wrapper->render($data), + title: $wrapper->renderBlock('title', $data), + description: $wrapper->renderBlock('description', $data), + keywords: $wrapper->renderBlock('keywords', $data), + tags: $wrapper->renderBlock('tags', $data), + robots: $wrapper->renderBlock('robots', $data), + changefreq: $wrapper->renderBlock('changefreq', $data), + priority: $wrapper->renderBlock('priority', $data), + ); + } + public function renderNotFound(string $requestUri): void { http_response_code(404);