site/build.py
back to source
# SPDX-License-Identifier: MIT
"""Build the numericalos.pages.dev static site.
Produces, into public/:
- the content pages, wrapped in a common shell
- numericalos.git/ a real dumb-HTTP git remote (git clone works)
- browse/ generated file tree, blobs, and commit log
- skills/ the agentic build skills, served as plain text
- index.json machine-readable skill index
- llms.txt agent discovery entry point
- AGENTS.md the multi-agent protocol
- skills.zip the whole skill bundle
Python 3 stdlib only. Run from the repo root: py site/build.py
"""
import hashlib
import html
import io
import json
import os
import shutil
import subprocess
import tempfile
import sys
import zipfile
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Run as `py site/build.py` and sys.path[0] is site/, not the repo root, so
# the numos package the artifact step needs is not importable without this.
if ROOT not in sys.path:
sys.path.insert(0, ROOT)
SITE = os.path.join(ROOT, "site")
# Overridable: on Windows another process can hold a handle on the git pack
# files under the output tree, which makes the pre-build clean fail. Building
# to a fresh directory sidesteps that without killing anyone else's process.
OUT = os.environ.get("NUMOS_OUT") or os.path.join(ROOT, "public")
SITE_URL = "https://numericalos.com"
NAV = [
("/", "home"),
("/docs/", "docs"),
("/graph/", "graph"),
("/whitepaper/", "whitepaper"),
("/principles/", "principles"),
("/skills/", "skills"),
("/browse/", "source"),
]
def _force_remove(func, path, _exc):
"""rmtree onerror hook: git packs are read-only, and Windows honors that."""
os.chmod(path, 0o700)
func(path)
def rmtree(path):
if os.path.isdir(path):
shutil.rmtree(path, onerror=_force_remove)
def git(*args):
"""Run a git command in the repo and return stripped stdout."""
return subprocess.run(
["git", "-C", ROOT] + list(args),
capture_output=True, text=True, check=True,
).stdout.strip()
def write(relpath, text):
path = os.path.join(OUT, relpath)
parent = os.path.dirname(path)
if parent and not os.path.isdir(parent):
os.makedirs(parent)
with open(path, "w", encoding="utf-8", newline="\n") as handle:
handle.write(text)
def shell(title, body, active="", description=""):
"""Wrap a body fragment in the common page shell."""
nav = "".join(
'<a href="%s"%s>%s</a>' % (href, ' class="on"' if label == active else "", label)
for href, label in NAV
)
return PAGE % {
"title": html.escape(title),
"description": html.escape(description or title),
"nav": nav,
"body": body,
}
PAGE = """<!doctype html>
<html lang="en"><head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<title>%(title)s</title>
<meta name="description" content="%(description)s">
<link rel="stylesheet" href="/assets/site.css">
</head><body>
<header><a class="brand" href="/">NumericalOS</a><nav>%(nav)s</nav></header>
<main>%(body)s</main>
<footer>
<p>NumericalOS — a Linux userspace whose init is a graph.
<strong>Status: bootstrap logic tested; x86_64 QEMU self-init PASS (degraded=0) on kernel+initrd and hybrid ISO/GRUB serial; fleet bare-QEMU PASS-degraded; metal and non-x86_64 unverified.</strong></p>
<p><a href="/LICENSE">MIT</a> ·
<a href="/numericalos.git">git</a> ·
<a href="/llms.txt">llms.txt</a> ·
<a href="/AGENTS.md">AGENTS.md</a> ·
<a href="/index.json">index.json</a> ·
<a href="/skills.zip">skills.zip</a></p>
</footer>
</body></html>
"""
# ---------------------------------------------------------------- skills
def parse_frontmatter(text):
"""Return (meta, body) for a SKILL.md with YAML-ish frontmatter."""
if not text.startswith("---"):
return {}, text
end = text.find("\n---", 3)
if end == -1:
return {}, text
raw = text[3:end].strip()
body = text[end + 4:].lstrip("\n")
meta = {}
key = None
for line in raw.split("\n"):
if line and not line[0].isspace() and ":" in line:
key, _, value = line.partition(":")
key = key.strip()
meta[key] = value.strip()
elif key and line.strip():
meta[key] = (meta[key] + " " + line.strip()).strip()
return meta, body
def collect_skills():
"""Read every skills/*/SKILL.md into a sorted list of dicts."""
skills = []
base = os.path.join(ROOT, "skills")
for name in sorted(os.listdir(base)):
path = os.path.join(base, name, "SKILL.md")
if not os.path.isfile(path):
continue
with open(path, encoding="utf-8") as handle:
text = handle.read()
meta, _ = parse_frontmatter(text)
skills.append({
"name": meta.get("name", name),
"description": meta.get("description", ""),
"path": "skills/%s/SKILL.md" % name,
"url": "%s/skills/%s/SKILL.md" % (SITE_URL, name),
"bytes": len(text.encode("utf-8")),
"text": text,
})
return skills
def build_skills(skills):
"""Serve each SKILL.md verbatim, plus an index page."""
for skill in skills:
rel = skill["path"].replace("skills/", "skills/", 1)
write(rel, skill["text"])
rows = "".join(
'<tr><td><a href="/%s">%s</a></td><td>%s</td></tr>'
% (skill["path"], html.escape(skill["name"]),
html.escape(skill["description"]))
for skill in skills
)
body = """
<h1>Agentic build skills</h1>
<p class="lead">These skills support <em>you</em> building NumericalOS targets on
your own machine, or via Cloudflare Sandbox + R2 when the host has no local
Linux/WSL. This site does not ship prebuilt OS images as the primary product.</p>
<p>Point your agent at <a href="/AGENTS.md">AGENTS.md</a> for the protocol, or
take the whole bundle: <a href="/skills.zip">skills.zip</a>.</p>
<table class="skills"><thead><tr><th>skill</th><th>when it fires</th></tr></thead>
<tbody>%s</tbody></table>
<h2>The honesty contract</h2>
<p>Only <code>numericalos-verify-boot</code> may claim a boot, and only for the
artifact and architecture it observed. Build skills produce artifacts and say
exactly that. A prior QEMU observation on x86_64 does not generalize to other
arches, metal, or a new ISO you just built — re-run verify-boot.</p>
""" % rows
write("skills/index.html", shell("Skills - NumericalOS", body, "skills",
"Agentic skills for building NumericalOS targets locally or via Cloudflare."))
def build_bundle(skills):
"""skills.zip - the whole bundle including AGENTS.md."""
buffer = io.BytesIO()
with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive:
for skill in skills:
archive.writestr(skill["path"], skill["text"])
with open(os.path.join(ROOT, "skills", "AGENTS.md"), encoding="utf-8") as handle:
archive.writestr("skills/AGENTS.md", handle.read())
with open(os.path.join(OUT, "skills.zip"), "wb") as handle:
handle.write(buffer.getvalue())
def build_discovery(skills, head):
"""index.json + llms.txt + AGENTS.md at the root."""
index = {
"name": "numericalos",
"description": "A Linux userspace whose init is a graph export.",
"site": SITE_URL,
"status": "bootstrap logic tested; x86_64 QEMU self-init PASS (degraded=0); fleet bare-QEMU PASS-degraded; metal and non-x86_64 unverified",
"license": "MIT",
"license_url": "%s/LICENSE" % SITE_URL,
"commit": head,
"clone": "%s/numericalos.git" % SITE_URL,
"agents": "%s/AGENTS.md" % SITE_URL,
"bundle": "%s/skills.zip" % SITE_URL,
"skills": [
{k: s[k] for k in ("name", "description", "path", "url", "bytes")}
for s in skills
],
}
write("index.json", json.dumps(index, indent=2, sort_keys=True) + "\n")
lines = [
"# NumericalOS",
"",
"> A Linux userspace whose init is a graph export. No unit files: every",
"> supervisable thing, boot ordering constraint, health rule, and",
"> remediation path is a node in an M/G/S/MGS graph quartet.",
"",
"Status: bootstrap logic tested; x86_64 QEMU self-init PASS (degraded=0)",
"on kernel+initrd and hybrid ISO/GRUB serial; fleet bare-QEMU",
"PASS-degraded; metal and non-x86_64 unverified.",
"See /docs/status for scope.",
"",
"## Build it yourself",
"",
"These skills drive YOUR agent to build targets on YOUR machine, or via",
"Cloudflare Sandbox + R2 (numericalos-build-iso-cf) when you have no WSL.",
"This site does not ship prebuilt OS images as the primary product.",
"",
]
for skill in skills:
lines.append("- [%s](%s): %s" % (skill["name"], skill["url"], skill["description"]))
lines += [
"",
"## Protocol",
"",
"- [AGENTS.md](%s/AGENTS.md): multi-agent build protocol and honesty contract" % SITE_URL,
"- [index.json](%s/index.json): machine-readable index" % SITE_URL,
"- [skills.zip](%s/skills.zip): the whole bundle" % SITE_URL,
"",
"## Source",
"",
" git clone %s/numericalos.git" % SITE_URL,
"",
"Served as static files over dumb-HTTP. No forge, no account, no",
"server-side code. Verify what you got before you run it.",
"",
]
write("llms.txt", "\n".join(lines))
shutil.copyfile(os.path.join(ROOT, "skills", "AGENTS.md"),
os.path.join(OUT, "AGENTS.md"))
# ---------------------------------------------------------------- git
def build_git_remote():
"""A real dumb-HTTP git remote: git clone <site>/numericalos.git works.
Packed into a single packfile so the object count stays small, then
update-server-info writes the indexes the dumb protocol needs.
"""
dest = os.path.join(OUT, "numericalos.git")
rmtree(dest)
subprocess.run(["git", "clone", "--bare", "--quiet", ROOT, dest], check=True)
# Explode the pack into loose objects.
#
# The dumb protocol probes objects/<sha> directly. With everything packed
# that probe 404s, and git inflates the 404 body before checking status --
# printing "inflate: data stream error ... corrupt" and only THEN falling
# back to the pack. The clone succeeds, but a documented clone command
# that prints "corrupt" is not shippable. Loose objects make the probe hit.
# The pack must be MOVED OUT of the object store before unpacking.
# git unpack-objects skips any object already present, and an object
# inside a pack in the same repo counts as present -- so unpacking in
# place is a silent no-op. Earlier builds only worked by accident: the
# source repo had loose objects, so a local clone hardlinked them loose
# and there was no pack to trip over. After a `git gc` in the source,
# the clone hardlinks a pack instead, unpack does nothing, and deleting
# the pack leaves a repository with ZERO objects that still looks
# plausible on disk.
packdir = os.path.join(dest, "objects", "pack")
stash = tempfile.mkdtemp()
try:
moved = []
for name in sorted(os.listdir(packdir)):
src = os.path.join(packdir, name)
os.chmod(src, 0o700)
if name.endswith(".pack"):
target = os.path.join(stash, name)
shutil.move(src, target)
moved.append(target)
else:
os.unlink(src)
for pack in moved:
with open(pack, "rb") as handle:
subprocess.run(["git", "-C", dest, "unpack-objects", "-q"],
stdin=handle, check=True)
finally:
shutil.rmtree(stash, ignore_errors=True)
loose = sum(len(files) for root, _d, files in os.walk(os.path.join(dest, "objects"))
if os.sep + "info" not in root and os.sep + "pack" not in root)
if loose == 0:
raise SystemExit("git remote has no objects: unpack produced nothing")
subprocess.run(["git", "-C", dest, "update-server-info"], check=True)
# A bare clone carries hooks samples and a local-path origin; neither
# belongs on a public remote.
rmtree(os.path.join(dest, "hooks"))
subprocess.run(["git", "-C", dest, "remote", "remove", "origin"],
capture_output=True)
return dest
# ---------------------------------------------------------------- browse
TEXT_EXT = {".py", ".sh", ".md", ".json", ".txt", ".toml", ".cfg", ".yml", ".yaml"}
def build_browse(head):
"""Generated source browser: file tree, blobs, and commit log."""
listing = git("ls-tree", "-r", "--name-only", "HEAD").split("\n")
listing = [p for p in listing if p]
rows = []
for path in listing:
size = git("cat-file", "-s", "HEAD:%s" % path)
rows.append(
'<tr><td><a href="/browse/%s.html">%s</a></td><td class="num">%s</td></tr>'
% (html.escape(path), html.escape(path), size)
)
_build_blob(path)
log = git("log", "--pretty=format:%h\x1f%an\x1f%ad\x1f%s", "--date=short").split("\n")
entries = []
for line in log:
parts = line.split("\x1f")
if len(parts) != 4:
continue
sha, _author, date, subject = parts
entries.append(
'<tr><td class="mono">%s</td><td class="num">%s</td><td>%s</td></tr>'
% (html.escape(sha), html.escape(date), html.escape(subject))
)
body = """
<h1>Source</h1>
<p class="lead">Generated from commit <code>%s</code>. This is a rendering; the
repository itself is the authority:</p>
<pre class="cmd">git clone %s/numericalos.git</pre>
<h2>Files (%d)</h2>
<table class="files"><thead><tr><th>path</th><th class="num">bytes</th></tr></thead>
<tbody>%s</tbody></table>
<h2>History (%d commits)</h2>
<table class="files"><thead><tr><th>commit</th><th class="num">date</th><th>subject</th></tr></thead>
<tbody>%s</tbody></table>
""" % (head, SITE_URL, len(listing), "".join(rows), len(entries), "".join(entries))
write("browse/index.html", shell("Source - NumericalOS", body, "source",
"Browse the NumericalOS source and history."))
def _build_blob(path):
ext = os.path.splitext(path)[1]
if ext not in TEXT_EXT:
return
try:
content = git("show", "HEAD:%s" % path)
except subprocess.CalledProcessError:
return
body = """
<h1 class="mono">%s</h1>
<p><a href="/browse/">back to source</a></p>
<pre class="code">%s</pre>
""" % (html.escape(path), html.escape(content))
write("browse/%s.html" % path, shell(path + " - NumericalOS", body, "source"))
# ---------------------------------------------------------------- content
def build_content(head):
"""Wrap the hand-written content fragments in the site shell."""
content_dir = os.path.join(SITE, "content")
for name in sorted(os.listdir(content_dir)):
if not name.endswith(".html"):
continue
with open(os.path.join(content_dir, name), encoding="utf-8") as handle:
raw = handle.read()
title, _, fragment = raw.partition("\n")
title = title.replace("<!--", "").replace("-->", "").strip()
fragment = fragment.replace("{{COMMIT}}", head)
stem = name[:-5]
if stem == "index":
rel, active = "index.html", "home"
elif stem == "whitepaper":
rel, active = "whitepaper/index.html", "whitepaper"
elif stem == "principles":
rel, active = "principles/index.html", "principles"
elif stem == "graph":
# Top-level pure-graph product page (not under /docs/).
rel, active = "graph/index.html", "graph"
else:
rel, active = "docs/%s/index.html" % stem, "docs"
if stem == "docs":
rel = "docs/index.html"
write(rel, shell(title, fragment, active))
def build_artifacts():
"""Publish what bootstrap.sh fetches, with real checksums.
The manifest is line-oriented rather than JSON for the same reason
numos.state is: bootstrap.sh reads it from a POSIX shell, and shipping a
JSON parser into the smallest-bootstrap path would defeat the point.
Returns the manifest text so the caller can report on it.
"""
from numos.export_state import build_state
from numos.state import render
state_text = render(build_state([]))
sources = [
("bootstrap.sh", os.path.join(ROOT, "boot", "bootstrap.sh")),
("numinit.sh", os.path.join(ROOT, "boot", "numinit.sh")),
("arch_table.sh", os.path.join(ROOT, "boot", "lib", "arch_table.sh")),
# Floor control binary named by seeded unit execs (identity-init, join).
# Not a control socket; see boot/numctl header.
("numctl", os.path.join(ROOT, "boot", "numctl")),
]
entries = []
for name, path in sources:
with open(path, "rb") as handle:
blob = handle.read()
# Shell sources may carry UTF-8 in comments (em-dashes, etc.). Hash is
# over the raw bytes; the published text is UTF-8, never silently
# mojibaked to ASCII replacement characters.
write("artifacts/" + name, blob.decode("utf-8"))
entries.append((name, hashlib.sha256(blob).hexdigest(), len(blob)))
state_blob = state_text.encode("ascii")
write("artifacts/numos.state", state_text)
write("data/numos.state", state_text)
entries.append(("numos.state", hashlib.sha256(state_blob).hexdigest(),
len(state_blob)))
lines = ["V 1"] + [
"F %s %s %d" % (name, digest, size)
for name, digest, size in sorted(entries)
]
manifest = "\n".join(lines) + "\n"
write("artifacts/manifest.txt", manifest)
# The curl | sh entry point, served at /boot with no extension.
with open(os.path.join(SITE, "boot.sh"), encoding="utf-8") as handle:
write("boot", handle.read())
# The licence travels with what it licenses. A project that publishes
# source and argues people should be able to build it must serve the
# grant that makes that legal, not only carry it in the repository.
for name in ("LICENSE", "NOTICE"):
with open(os.path.join(ROOT, name), encoding="utf-8") as handle:
write(name, handle.read())
return entries
def build_404():
"""A real 404 page - load-bearing, not decoration.
Without 404.html, Cloudflare Pages answers unmatched paths with the root
index.html and status 200. Git's dumb-HTTP protocol probes for loose
objects that legitimately do not exist (everything is packed); receiving
HTML with a 200 instead of a 404, it reports 'inflate: data stream error'
and the clone breaks. A real 404 lets git fall through to the packfile.
"""
body = """
<h1>404</h1>
<p class="lead">No such page.</p>
<p><a href="/">home</a> · <a href="/docs/">docs</a> ·
<a href="/skills/">skills</a> · <a href="/browse/">source</a></p>
"""
write("404.html", shell("404 - NumericalOS", body))
def build_headers():
write("_headers", """/*
X-Content-Type-Options: nosniff
Referrer-Policy: strict-origin-when-cross-origin
/numericalos.git/*
Content-Type: application/octet-stream
Cache-Control: no-cache
/skills/*
Content-Type: text/markdown; charset=utf-8
/llms.txt
Content-Type: text/plain; charset=utf-8
/boot
Content-Type: text/plain; charset=utf-8
Cache-Control: no-cache
/LICENSE
Content-Type: text/plain; charset=utf-8
/NOTICE
Content-Type: text/plain; charset=utf-8
/artifacts/*
Content-Type: text/plain; charset=utf-8
Cache-Control: no-cache
/data/*
Content-Type: text/plain; charset=utf-8
Cache-Control: no-cache
/AGENTS.md
Content-Type: text/markdown; charset=utf-8
""")
write("robots.txt", "User-agent: *\nAllow: /\nSitemap: %s/sitemap.xml\n" % SITE_URL)
def main():
rmtree(OUT)
os.makedirs(OUT)
head = git("rev-parse", "--short", "HEAD")
skills = collect_skills()
shutil.copytree(os.path.join(SITE, "assets"), os.path.join(OUT, "assets"))
build_content(head)
build_skills(skills)
build_bundle(skills)
build_discovery(skills, head)
build_browse(head)
artifacts = build_artifacts()
build_git_remote()
build_404()
build_headers()
count = sum(len(files) for _, _, files in os.walk(OUT))
print("built %s at commit %s: %d files, %d skills, %d artifacts"
% (OUT, head, count, len(skills), len(artifacts)))
if __name__ == "__main__":
main()