0 reported it broke · the ▶ count is distinct visitors who ran it, counted once each, anonymously
Language models never see letters — they see tokens, and every bill you get from one is denominated in them. This page runs a real byte-pair encoder in your browser: an 800-merge table trained offline, applied with the same byte-level pre-tokenization GPT-2-style tokenizers use, so the coloured chips update live and the trace replays exactly how each token was assembled, one ranked merge at a time. Start with replay training — the slider walks the vocabulary from zero merges, where every UTF-8 byte is its own token, up to all 800, and the curve underneath shows why the first handful of merges buy enormous savings while the eight-hundredth barely registers. Then load the other languages sample and watch chars-per-token drop from 2.09 to 0.85: with a 1,056-token vocabulary — roughly a hundred times smaller than a production one — this is an honest demonstration of the mechanism, not a replica of any shipping tokenizer.
Source — the code you see is the code that just ran 990 lines 48.1 KB index.html
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Tokenizer Visualizer — sloprun.dev</title>
<style>
/* sloprun design tokens — inline this block into every demo (self-contained rule).
Identity: instrument-panel. Machine facts in mono; human words in sans.
Green is EARNED: only for "it ran / it worked" states, never decoration. */
:root {
--bg: #F6F6F3; /* neutral paper, not cream */
--surface: #FFFFFF;
--ink: #1A1D21;
--muted: #5A6068;
--line: #E3E2DC;
--accent: #C05600; /* phosphor amber, darkened for light bg */
--accent-soft: #FFF3E6;
--run: #1A7F37; /* earned green */
--run-soft: #E7F4EA;
--danger: #C93C37;
--radius: 10px;
--font-sans: system-ui, -apple-system, "Segoe UI", sans-serif;
--font-mono: ui-monospace, "SF Mono", "Cascadia Code", Menlo, Consolas, monospace;
}
@media (prefers-color-scheme: dark) {
:root:not([data-theme="light"]) {
--bg: #14161A; --surface: #1C1F26; --ink: #E8E6E1; --muted: #9AA0A8;
--line: #2A2E36; --accent: #FFB454; --accent-soft: #2A2118;
--run: #3FB950; --run-soft: #16281B; --danger: #F47067;
}
}
:root[data-theme="dark"] {
--bg: #14161A; --surface: #1C1F26; --ink: #E8E6E1; --muted: #9AA0A8;
--line: #2A2E36; --accent: #FFB454; --accent-soft: #2A2118;
--run: #3FB950; --run-soft: #16281B; --danger: #F47067;
}
body { background: var(--bg); color: var(--ink); font-family: var(--font-sans); }
/* ---- demo-local supplements (same three theme states) ---- */
:root {
--canvas: #FBFBF8; --shadow: 0 1px 2px rgba(20,22,26,.05); --track: #ECEBE5;
--t0: #FFE7CC; --t1: #DCE7FA; --t2: #EDE1FB; --t3: #FBDFE4; --t4: #E4E7EB; --t5: #D8EDF6;
}
@media (prefers-color-scheme: dark) {
:root:not([data-theme="light"]) {
--canvas: #171A20; --shadow: 0 1px 2px rgba(0,0,0,.3); --track: #23272F;
--t0: #4A3318; --t1: #1E3251; --t2: #32254B; --t3: #4A2530; --t4: #2E333B; --t5: #17384A;
}
}
:root[data-theme="dark"] {
--canvas: #171A20; --shadow: 0 1px 2px rgba(0,0,0,.3); --track: #23272F;
--t0: #4A3318; --t1: #1E3251; --t2: #32254B; --t3: #4A2530; --t4: #2E333B; --t5: #17384A;
}
* { box-sizing: border-box; }
html, body { margin: 0; padding: 0; }
body { line-height: 1.5; -webkit-text-size-adjust: 100%; overflow-x: hidden; }
.wrap { max-width: 1140px; margin: 0 auto; padding: 22px 16px 40px; }
/* ---- header ---- */
h1 { font-size: 1.35rem; margin: 0 0 6px; letter-spacing: -0.01em; }
.lede { margin: 0 0 10px; color: var(--muted); max-width: 66ch; font-size: .95rem; }
.spec { display: flex; flex-wrap: wrap; gap: 6px; margin-bottom: 16px; }
.spec b {
font-family: var(--font-mono); font-size: .7rem; font-weight: 500; letter-spacing: .02em;
padding: 3px 8px; border: 1px solid var(--line); border-radius: 999px;
color: var(--muted); background: var(--surface); white-space: nowrap;
}
.spec b.warn { color: var(--accent); border-color: var(--accent); background: var(--accent-soft); }
/* ---- layout ---- */
.grid { display: grid; gap: 14px; grid-template-columns: 1fr; align-items: start; }
.grid > div { min-width: 0; display: flex; flex-direction: column; gap: 14px; }
@media (min-width: 920px) {
/* rows: the left stack sizes row 1, row 2 soaks up whatever the taller right column adds,
so the gap between the budget and cost cards never inflates */
.grid { grid-template-columns: minmax(0, 0.95fr) minmax(0, 1.05fr); grid-template-rows: auto 1fr; }
.colA { grid-column: 1; grid-row: 1; }
.colB { grid-column: 2; grid-row: 1 / span 2; align-self: stretch; }
.cost { grid-column: 1; grid-row: 2; }
/* the trace panel takes up whatever height the left stack leaves over */
.colB > .tracecard { flex: 1 1 auto; min-height: 250px; }
.colB > .tracecard .trace { max-height: none; }
}
.card {
background: var(--surface); border: 1px solid var(--line);
border-radius: var(--radius); padding: 13px; box-shadow: var(--shadow);
}
.card h2 {
font-family: var(--font-mono); font-size: .7rem; font-weight: 600; letter-spacing: .08em;
text-transform: uppercase; color: var(--muted); margin: 0 0 10px;
}
.row { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; }
.spread { justify-content: space-between; }
.head { display: flex; align-items: baseline; justify-content: space-between; gap: 8px; margin-bottom: 9px; flex-wrap: wrap; }
.head h2 { margin: 0; }
/* ---- stat strip ---- */
.stats {
display: flex; flex-wrap: wrap; gap: 1px; background: var(--line);
border: 1px solid var(--line); border-radius: var(--radius); overflow: hidden; margin-bottom: 14px;
}
.stat { background: var(--surface); padding: 9px 11px; flex: 1 1 110px; min-width: 0; }
.stat .k { font-family: var(--font-mono); font-size: .61rem; letter-spacing: .09em; text-transform: uppercase; color: var(--muted); }
.stat .v { font-family: var(--font-mono); font-size: 1.15rem; font-weight: 600; letter-spacing: -0.02em; }
.stat.lead .v { color: var(--accent); }
/* ---- controls ---- */
button, input, textarea, select { font: inherit; color: inherit; }
.btn {
font-family: var(--font-mono); font-size: .73rem; padding: 5px 9px;
background: var(--surface); color: var(--muted);
border: 1px solid var(--line); border-radius: 7px; cursor: pointer;
}
.btn:hover { border-color: var(--accent); color: var(--ink); }
.btn[aria-pressed="true"], .btn.on {
background: var(--accent-soft); border-color: var(--accent); color: var(--accent); font-weight: 600;
}
:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; border-radius: 4px; }
textarea {
width: 100%; min-height: 224px; resize: vertical; display: block;
background: var(--canvas); color: var(--ink);
border: 1px solid var(--line); border-radius: 8px; padding: 10px;
font-family: var(--font-mono); font-size: .81rem; line-height: 1.55;
}
textarea:focus { border-color: var(--accent); }
.sr { position: absolute; width: 1px; height: 1px; overflow: hidden; clip: rect(0 0 0 0); white-space: nowrap; }
/* ---- token chips ---- */
.tokens { display: flex; flex-wrap: wrap; gap: 3px; max-height: 300px; overflow-y: auto; padding: 3px; }
.tok {
font-family: var(--font-mono); font-size: .79rem; line-height: 1.35;
padding: 2px 4px; border-radius: 5px; border: 1px solid transparent;
background: var(--t0); color: var(--ink); cursor: pointer;
white-space: pre-wrap; word-break: break-word; text-align: left; max-width: 100%;
}
.tok.c1 { background: var(--t1); } .tok.c2 { background: var(--t2); }
.tok.c3 { background: var(--t3); } .tok.c4 { background: var(--t4); }
.tok.c5 { background: var(--t5); }
.tok:hover { border-color: var(--accent); }
.tok[aria-current="true"] { border-color: var(--accent); box-shadow: inset 0 -2px 0 var(--accent); }
.tok .hexb { color: var(--muted); font-size: .7rem; }
.tok .ws { color: var(--muted); }
.tok.idmode { color: var(--muted); }
/* ---- merge budget ---- */
.plot { position: relative; margin: 2px 0 4px; }
.plot svg { display: block; width: 100%; height: 84px; }
.axis {
display: flex; justify-content: space-between; font-family: var(--font-mono);
font-size: .62rem; color: var(--muted); margin-top: 2px;
}
.dot {
position: absolute; width: 9px; height: 9px; margin: -4.5px 0 0 -4.5px;
border-radius: 50%; background: var(--accent); pointer-events: none;
}
.flag {
position: absolute; transform: translateX(-50%);
font-family: var(--font-mono); font-size: .64rem; color: var(--accent);
background: var(--surface); border: 1px solid var(--accent); border-radius: 5px;
padding: 0 4px; white-space: nowrap; pointer-events: none;
}
input[type="range"] { width: 100%; accent-color: var(--accent); background: transparent; margin: 6px 0 2px; }
.readout { font-family: var(--font-mono); font-size: .76rem; color: var(--muted); }
.readout b { color: var(--ink); }
/* ---- merge trace ---- */
.tracecard { display: flex; flex-direction: column; }
.trace { max-height: 300px; overflow-y: auto; min-height: 0; }
.tracehead { font-family: var(--font-mono); font-size: .76rem; margin: 0 0 8px; color: var(--muted); }
.tracehead b { color: var(--ink); }
.step { display: flex; gap: 8px; align-items: baseline; padding: 5px 0; border-top: 1px solid var(--line); }
.step:first-child { border-top: 0; }
.rk { font-family: var(--font-mono); font-size: .67rem; color: var(--muted); flex: 0 0 auto; min-width: 4.4em; }
.seq { display: flex; flex-wrap: wrap; gap: 2px; min-width: 0; }
.pc {
font-family: var(--font-mono); font-size: .73rem; padding: 1px 3px;
border-radius: 4px; background: var(--t4); white-space: pre-wrap; word-break: break-word;
}
.pc.new { background: var(--accent-soft); color: var(--accent); font-weight: 600; outline: 1px solid var(--accent); }
/* ---- cost table ---- */
.tblwrap { overflow-x: auto; }
table { border-collapse: collapse; width: 100%; font-size: .81rem; min-width: 300px; }
th, td { text-align: right; padding: 6px 8px; border-bottom: 1px solid var(--line); }
@media (max-width: 460px) { th, td { padding: 6px 4px; } input[type="number"] { width: 4.6em; } }
th:first-child, td:first-child { text-align: left; }
thead th {
font-family: var(--font-mono); font-size: .63rem; letter-spacing: .06em; text-transform: uppercase;
color: var(--muted); font-weight: 600;
}
tbody td { font-family: var(--font-mono); }
tbody td:first-child { font-family: var(--font-sans); }
tbody tr:last-child td { border-bottom: 0; }
input[type="number"] {
width: 5.4em; text-align: right; font-family: var(--font-mono); font-size: .79rem;
background: var(--canvas); border: 1px solid var(--line); border-radius: 6px; padding: 3px 5px;
}
input[type="number"]:focus { border-color: var(--accent); }
.status { font-family: var(--font-mono); font-size: .71rem; color: var(--muted); }
.status.ok { color: var(--run); }
.note { font-size: .78rem; color: var(--muted); margin: 9px 0 0; }
.note code, .body code { font-family: var(--font-mono); font-size: .95em; }
details { margin-top: 14px; }
summary { cursor: pointer; font-family: var(--font-mono); font-size: .73rem; color: var(--muted); list-style: none; }
summary::-webkit-details-marker { display: none; }
summary:hover { color: var(--ink); }
details[open] > summary .mk { transform: rotate(90deg); }
summary .mk { display: inline-block; }
@media (prefers-reduced-motion: no-preference) { summary .mk { transition: transform .15s ease; } }
details .body { font-size: .84rem; color: var(--muted); max-width: 78ch; }
details .body p { margin: 10px 0; }
details .body b { color: var(--ink); }
#idsOut {
font-family: var(--font-mono); font-size: .7rem; color: var(--muted);
max-height: 130px; overflow: auto; margin-top: 8px; word-break: break-all;
background: var(--canvas); border: 1px solid var(--line); border-radius: 6px; padding: 8px;
}
footer { margin-top: 24px; font-family: var(--font-mono); font-size: .7rem; color: var(--muted); }
@media (prefers-reduced-motion: no-preference) {
.btn, .tok, input[type="number"] { transition: background-color .12s ease, border-color .12s ease, color .12s ease; }
}
</style>
</head>
<body>
<div class="wrap">
<header>
<h1>Tokenizer Visualizer</h1>
<p class="lede">Language models never see letters — they see <em>tokens</em>, and every bill is counted in them.
This runs a real byte-pair encoder in your browser. Type anything, then drag the merge slider to watch the
vocabulary get built one merge at a time.</p>
<div class="spec">
<b>800 merges</b>
<b>1,056-token vocab</b>
<b>trained offline on English docs + code</b>
<b class="warn">approximation, not a real LLM tokenizer</b>
</div>
</header>
<div class="stats">
<div class="stat lead"><div class="k">tokens</div><div class="v" id="sTok">0</div></div>
<div class="stat"><div class="k">characters</div><div class="v" id="sChar">0</div></div>
<div class="stat"><div class="k">words</div><div class="v" id="sWord">0</div></div>
<div class="stat"><div class="k">utf-8 bytes</div><div class="v" id="sByte">0</div></div>
<div class="stat"><div class="k">chars / token</div><div class="v" id="sRatio">0</div></div>
</div>
<div class="grid">
<!-- LEFT -->
<div class="colA">
<section class="card">
<div class="head">
<h2>Input text</h2>
<span class="status" id="charline">—</span>
</div>
<label class="sr" for="input">Text to tokenize</label>
<textarea id="input" name="text" spellcheck="false"></textarea>
<div class="row" style="margin-top:9px">
<span class="status" style="margin-right:2px">samples:</span>
<button class="btn" type="button" data-sample="mixed" aria-pressed="false">prose + emoji + code</button>
<button class="btn" type="button" data-sample="rare" aria-pressed="false">rare words</button>
<button class="btn" type="button" data-sample="numbers" aria-pressed="false">numbers & URLs</button>
<button class="btn" type="button" data-sample="world" aria-pressed="false">other languages</button>
</div>
</section>
<section class="card">
<div class="head">
<h2>Vocabulary budget</h2>
<button class="btn" type="button" id="btnPlay">▶ replay training</button>
</div>
<div class="plot" id="plot">
<svg id="curve" viewBox="0 0 100 40" preserveAspectRatio="none" role="img"
aria-label="Token count for this text as merges are added — steep at first, then flattening"></svg>
<div class="dot" id="dot" hidden></div>
<div class="flag" id="flag" hidden></div>
</div>
<div class="axis"><span>0 merges</span><span>800 merges</span></div>
<label class="sr" for="lim">Merges applied</label>
<input type="range" id="lim" name="merges" min="0" max="800" value="800" step="1">
<p class="readout">merges applied <b id="rLim">800</b> · vocabulary <b id="rVocab">1,056</b> tokens ·
this text <b id="rTok">0</b> tokens</p>
<p class="note">At zero merges every UTF-8 byte is its own token. Each merge glues the most frequent
adjacent pair in the training corpus into one new symbol — the first few buy enormous savings, the
eight-hundredth barely registers. That curve is why production vocabularies stop around 100k.</p>
</section>
</div>
<!-- RIGHT -->
<div class="colB">
<section class="card">
<div class="head">
<h2>Tokens</h2>
<span class="status" id="runstat" aria-live="polite">—</span>
</div>
<div class="row" style="margin-bottom:9px">
<button class="btn" type="button" id="btnIds" aria-pressed="false">show token IDs</button>
<button class="btn" type="button" id="btnWs" aria-pressed="true">show whitespace</button>
</div>
<div class="tokens" id="tokens" role="group" aria-label="Tokens — arrow keys to inspect, Enter to trace"></div>
<p class="note" id="capnote" hidden></p>
<p class="note"><code>⟨F0⟩</code> is a raw byte with no vocabulary entry of its own — emoji and non-Latin
scripts land there. <code>·</code> is a space, <code>⏎</code> a newline.</p>
</section>
<section class="card tracecard">
<div class="head">
<h2>Merge trace</h2>
<span class="status">click any chip</span>
</div>
<div id="trace" class="trace"></div>
<p class="note">Every step is one entry from the embedded merge table, applied lowest rank first —
exactly the algorithm, just with a much smaller table.</p>
</section>
</div>
<section class="card cost">
<h2>Cost estimate</h2>
<div class="row" style="margin-bottom:10px">
<span class="status">count basis:</span>
<button class="btn on" type="button" id="basisProd" aria-pressed="true">production ≈ chars ÷ 4</button>
<button class="btn" type="button" id="basisDemo" aria-pressed="false">this demo's vocab</button>
</div>
<div class="tblwrap">
<table>
<thead>
<tr><th scope="col">Example model</th><th scope="col">$ / 1M in</th><th scope="col">this text</th><th scope="col">×10,000 runs</th></tr>
</thead>
<tbody id="costBody"></tbody>
</table>
</div>
<p class="note">Billed on <b id="basisTok">0</b> input tokens. Rates are editable examples (public list
prices, August 2026) — always check your provider's current pricing.</p>
</section>
</div>
<details>
<summary><span class="mk">▸</span> token IDs for this text</summary>
<div id="idsOut"></div>
</details>
<details>
<summary><span class="mk">▸</span> how this differs from a real LLM tokenizer</summary>
<div class="body">
<p><b>Same mechanism, tiny scale.</b> This is genuine byte-pair encoding: text is split by a GPT-2-style
pre-tokenizer regex, each piece is converted to UTF-8 bytes, and merges from the embedded table are applied
in the order they were learned. The merge table was trained offline on a few megabytes of English
documentation, prose and Python source.</p>
<p><b>What's different.</b> 800 merges give a <b>1,056-token</b> vocabulary. Production tokenizers carry
<b>100,000–200,000</b> tokens learned from terabytes of multilingual text. So counts here run roughly
1.8–2× higher than a real tokenizer on the same English text, and far higher on other scripts. There are
also no special tokens — no BOS/EOS, no chat template.</p>
<p><b>What still transfers.</b> Whitespace belongs to the <em>following</em> word, so <code>" the"</code>
and <code>"the"</code> are different tokens. Emoji are four UTF-8 bytes and shatter into byte tokens. Rare
words fragment while common ones survive whole. Non-English scripts pay a steep tax. Long digit runs are
expensive. Those effects are all real, and they are why token counts never match character counts.</p>
</div>
</details>
<footer>demo · sloprun.dev</footer>
</div>
<script>
(function () {
"use strict";
/* ---------- embedded merge table (trained offline, ranked) ---------- */
var MERGES = "\u0120 \u0120\n\u0120 t\n\u0120\u0120 \u0120\u0120\ni n\nh e\n\u0120 a\nr e\no n\ne r\n\u0120t he\no r\n\u0120 s\na t\ni s\ne s\n\u0120 c\ne d\n\u0120 f\ne n\ni t\n\u0120 o\ni on\n\u0120 b\nl e\na l\n\u0120\u0120 \u0120\n\u0120 d\n\u0120 p\na n\na r\nin g\n\u0120 w\n\u0120 in\n\u0120\u0120\u0120\u0120 \u0120\u0120\u0120\u0120\n\u0120t o\ns t\nc t\nr o\n\u0120 m\ne t\n\u0120a n\n\u0120 is\n\u0120 n\ni l\nu s\n\u0120 re\ni c\n\u0120o f\nu t\n\u0120 l\n\u0120 -\na m\no m\ne l\nen t\n\u0120t h\ni f\ne x\n\u0120an d\na s\n\u0120f or\nu n\ne c\no t\n\u0120\u0120\u0120\u0120 \u0120\u0120\u0120\nu l\nu r\np t\na b\na c\n\u0120 T\nd e\ni m\n\u0120 us\n\u0120b e\ne m\nl y\n\u010a \u0120\u0120\u0120\n\u0120 v\nat ion\nl o\na g\n\u010a \u0120\u0120\u0120\u0120\u0120\u0120\u0120\n\u0120 on\na d\nit h\n\u0120 g\n\u0120d e\n\u0120 (\ne ct\ni b\n\u0120 S\no l\n\u0120 C\n\u0120s t\ni le\nu m\ni g\n\u0120c om\n\u0120c on\n\u0120p ro\n\u0120 ex\n\u0120 h\n\u0120 or\n\u0120w ith\ns e\n\u0120 \"\npt ion\n\u010a \u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120\nc e\np p\n\u0120th at\nc h\ni v\n\u0120 I\n\u0120b y\nt er\nab le\nre s\n\u0120 '\nan d\n\u010a \u010a\n\u0120a s\n\u0120a l\n\u0120n ot\nul t\n\u0120 0\nam e\n\u0120 #\nh is\n\u0120 it\no d\nel f\nor t\nes s\n\u0120a re\n\u0120 if\n\u0120 A\np ec\ner s\n\u0120 =\ni d\nat e\n\u0120 D\n\u0120 L\nr a\n\u0120de f\nk e\ny st\nyst em\n\u0120s u\no p\nhe r\no w\n\u0120o ption\n\u0120th is\no c\nu e\nac k\nI N\na y\nal l\nis t\nv er\n\u0120T he\nen d\no de\np er\npec if\nE R\nil l\nur n\n\u0120 e\nhe n\n\u0120f ile\nct ion\nes t\ni z\n( )\no re\nag e\n\u0120 P\ni es\nq u\nro m\ng et\n\u0120 F\na ult\n\u0120c an\n\u0120s et\n\u0120com p\ni r\na v\nor m\n\u0120w h\nin t\nc on\np l\ni re\n\u010a\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120 \u0120\u0120\u0120\n\u0120 G\nj ect\nex t\n\u0120s elf\n\u00e2 \u0122\nin e\n\u0120v al\n\u0120w ill\na in\nu p\nM A\nat ed\n\u0120us e\nan g\n\u0120T his\n\u0120us ed\n\u0120c h\n\u0120f rom\n\u0120con t\n0 0\nen er\np ut\n\u0120 N\n\u0120 X\n\u0120 en\nas s\ny p\no u\n\u0120s pecif\n\u0120 un\nre n\nt urn\n\u0120com m\nf t\n\u0120- >\n\u0120 r\np le\nr i\n\u0120s h\nd d\nA T\ns o\nystem d\n\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120 \u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120\n\u0120 y\nic h\nu re\nar i\n\u0120def ault\n\u0120 M\n\u0120re turn\n\u0120a r\nf orm\n\u0120in st\nT he\nm ent\ni x\nc om\nres s\nab l\n\u0120 1\n\u0120f il\nal ly\nt her\nf f\nb ject\n\u0120w hen\n\u0120 R\nc l\nE T\nO N\nar y\nlo w\niv e\n\" \"\na il\nor y\nig n\n\u0120a t\nas e\nr u\n\u0120n e\n\u0120s e\np ro\n# #\na re\nin k\n\u0120wh ich\nc ess\no ut\nt o\ni ed\n\u0120 O\n\u0120 W\n\u0120 U\nR E\nar get\n) ,\nT his\n) :\ners ion\nn o\n\u0120l ib\nlo c\n\u0120s o\nI T\nire ct\n\u0120v ari\nr or\n\u0120 E\n\u0120 <\nat a\nun ction\niz e\ns elf\n\u0120b u\na ke\nit y\n\u0120 im\nar t\n\u0120o ut\no s\n\u0120val ue\nic e\nper t\na p\n\u0120 B\npp ort\n) .\np ort\n\u0120d o\nat h\nar d\ns ystemd\nT ER\n\u0120a dd\nim e\nt h\nul d\no uld\n\u0120on ly\ni al\n\u0120- -\n\u0120comm and\n\u0120al l\num ent\n\u0120t arget\no un\nyp e\n\u0120t r\non t\n\u0120re s\nren t\n\u0120fil es\non e\n\u0120 2\n\u0120I f\n\u0120g ener\nt he\nav e\n\u0120' \\\nt r\n\u010a \u010a\u0120\u0120\u0120\nu st\n\u0120m ay\n\u0120o ther\nA L\n\u0120y ou\n\u0120l o\n\u0120p re\num b\n\u0120c ode\n\u0120f unction\n\u0120n ame\nac e\ni p\n\u0120su pport\nd er\nf e\nor g\nor k\n\u0120o bject\nil d\nin d\n\u0120pro pert\nv ice\n\u0120l ink\n\u0120\u0120\u0120\u0120 \u0120\nt e\nA R\no v\nf ile\n\u0120 *\nL E\n\u0120 \u00e2\u0122\nf ig\n\u0120h as\nA C\n\u0120a pp\n\u0120us ing\nit ion\nh t\n\u0120d irect\n\u0120l ist\n\u0120c all\n\u010a\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120 \u0120\u0120\u0120\u0120\u0120\u0120\u0120\n\u0120 he\nu g\nan t\n\u0120v ersion\nac h\ns er\nK E\n\u0120X ft\n\u0120b ut\n\u0120ch ang\n- -\no st\nl ib\ns et\nS T\n\u0120out put\n\u0120an y\nom e\n\u0120d is\nET TER\n\u0120a b\nt op\nc he\nO R\nr ing\n\u0120L ETTER\n\u0120specif ied\numb er\nar n\nation s\na ct\nke y\n\u0120m od\n\u0120s y\nur ce\nD E\nect ion\nde f\n\u0120for m\n' s\n\u0120h ave\n\u0120vari able\n' ,\nan s\n\u0120al so\nf re\nat ch\n. .\ncl u\n\u0120lib r\nat or\nn ame\np h\n> >\n\u0120 H\nes k\nesk top\ni o\n\u0120propert y\nre am\nr int\n\u0120on e\nMA KE\n\u0120c l\nis e\nr it\nI n\n\u0120d ata\n\u0120re qu\nM E\nL L\nor d\nre ad\n\u0120m a\n\u0120p er\n( '\n\u0120option s\nter n\nent s\n\u0120 V\ni de\n\u0120s ystem\nr y\n\u0120 le\nro l\n\u0120comp il\n\u0120\" \"\"\ng r\ner ror\nre d\n\u0120n o\n\u0120n umber\n\u0120 [\nu al\n\u0120ar g\ne e\n\u0120 up\nen s\noc k\n\u0120I n\nabl es\ned esktop\n' t\nra w\nfre edesktop\n\u0120r un\nam es\nabl ed\np ar\nform ation\ns u\n\u0120ex p\nM P\n\u0120sh ould\np re\n\u0120do es\nim al\nen c\n\u0120m ore\ne ad\nAT IN\nm b\n\u0120L ATIN\nol low\n\u0120p ack\n\u0120ne w\nam ple\n\u0120a c\no o\nI G\nus e\nI ON\nec imal\np end\nc ri\nre e\n## ##\n\u0120bu ild\na st\n\u0120w e\ncom p\n\u0120c re\n\u0120l ine\ncon t\nc k\nat ing\n' )\nur rent\nv el\nR O\ness age\nl ag\nv al\nAC K\nin ed\n\u0120in formation\nl ic\n\u0120a v\n\u0120f ollow\n\u0120p ath\n\u0120m an\nt ing\n\u0120t est\na k\nU T\nru ct\nru ction\n\u0120the n\non g\nion s\nm p\n\u0120\u0120\u0120\u0120 \u0120\u0120\ns h\ne w\n\" ,\nE N\nb us\net h\not e\ngr am\nA N\nib le\n\u0120p ar\n\u0120re ad\nig ht\n\u0120p ass\nloc k\n\u0120re g\nl ine\nit s\n\u0120d et\nA P\n\u0120w arn\n\u0120cont ain\n\u0120form at\n\u00e2 \u0136\n\u0120\u00e2\u0122 \u0136\nib ut\nag es\n\u0120t ime\n\u00e2\u0136 \u0124\n\u0120ex t\nar g\n\u0120 3\n\u0120t ype\n\u0120arg ument\n\u0120 /\n\u0120pro cess\n\u0120s ome\n\u0120st ring\n\u0120con fig\n\u0120th an\nov e\n\u0120w he\nu es\n\u0120 k\nel y\n\u0120libr ary\nle ase\nd s\nat ic\no g\nz er\neth od\ner m\ni el\n\u0120 key\ni an\n\u0120 ent\n\u0120res ult\n\u0120w as\nmb ol\nec ut\nP ACK\n\u0120p l\n\u0120s ame\nt es\np s\nI C\now n\n\u0120ne ed\n\u0120su b\n\u0120 j\nu le\n\u0120re m\n\u0120 error\n\u0120s ize\n\u0120g et\no ur\nir st\n\u0120direct ory\n\u0120inst ruction\nf ter\n\u0120e v\nit ial\n\u0120o ver\n6 4\nce pt\n\u0120m em\nin ter\nr ror\n\u0120a ut\n\u0120in to\n\u0120w ork\n\u0120g iv\n\u0120in d\net w\no se\n* *\nn ot\n\u0120 J\n\u0120d es\no int\nf ore\nat es\nd ing\nIN G\n\u0120ex ample\n\u0120 ro\nan ce\nare d\n\u0120m ust\n\u0120 el\n\u0120I t\n\u0120in clu\n\u0120in t\n\u0120cont ext\nS E\nple ment\n\u010a \u010a\u0120\u0120\u0120\u0120\u0120\u0120\u0120\nle ct\n\u0120s er\ncri pt\nu res\n\u0120pro gram\na x\n\u0120c urrent\n\u0120\u0120\u0120\u0120\u0120\u0120\u0120\u0120 \u0120\u0120\u0120\n\u0120l oc\n\u0120in put\nb ug\nI f\nu x\nMA LL\n\u0120F or\n\u0120as s\n\u0120be en\n\u0120it s\n\u0120in ter\nff ect\noun t\nil ity\nC O\n\u0120so urce\nI B\nfe ren\nin es\n\u0120o per\nf o\nE S\n\u0120us er\nl l\num p\n\u0120S MALL\n\u0120ch ar\nend ed\nail able\n\u0120d ec\nf il\n\u0120n ames\n\u0120en abled\n\u0120en c\n\u0120 \u00e2\u0136\u0124\n\u0120p art\nE rror\nad d\nlo g\npl ay\n\u0120val ues\nel l\n\u0120su ch\n\u0120N one\n\u0120 +\n\u0120f lag\n\u0120m e\ni ke\n\u0120giv en\nc es\n\u0120s ystemd\nx t\n\u0120m ode\nA G\nd u\n\u0120follow ing\n\u0120re c\nat ive\n\u0120pro v\nM ake\nt ime\nar ch\nf ul\n\u0120ex ecut\n\u0120warn ing\nc re\nf ix\noun d\n\u0120f irst\na pp\nin st\n+ +\nult i\ny n\n\u0120o p\n\u0120sy mbol\n\u0120C AP\nand ard\nhe ck\nC T\n\u0120p os\nv ir\nI D\nad er\nam et\n\u0120 .\n\u0120 x\n\u0120ar ch\n\u0120inst ead\np os\n\u0120m ethod\nb it\n\u0120 $\n\u0120d if\n\u0120d oc\n\u0120inst all".split("\n");
var RANK = new Map();
for (var i = 0; i < MERGES.length; i++) RANK.set(MERGES[i], i);
var MAX_MERGES = MERGES.length;
/* ---------- byte <-> printable-char alphabet (GPT-2 scheme) ---------- */
function bytesToUnicode() {
var bs = [], b;
for (b = 33; b <= 126; b++) bs.push(b);
for (b = 161; b <= 172; b++) bs.push(b);
for (b = 174; b <= 255; b++) bs.push(b);
var have = new Set(bs), cs = bs.slice(), n = 0;
for (b = 0; b < 256; b++) if (!have.has(b)) { bs.push(b); cs.push(256 + n); n++; }
var out = new Array(256);
for (var i = 0; i < bs.length; i++) out[bs[i]] = String.fromCharCode(cs[i]);
return out;
}
var B2U = bytesToUnicode();
var U2B = new Map();
for (var bi = 0; bi < 256; bi++) U2B.set(B2U[bi], bi);
/* ---------- vocabulary: 256 raw bytes, then each merge result ---------- */
var VOCAB = B2U.slice();
var IDS = new Map();
VOCAB.forEach(function (t, i) { IDS.set(t, i); });
var VOCAB_AT = new Array(MERGES.length + 1); // vocabulary size after k merges
VOCAB_AT[0] = 256;
MERGES.forEach(function (m, k) {
var sp = m.indexOf(" ");
var t = m.slice(0, sp) + m.slice(sp + 1);
if (!IDS.has(t)) { IDS.set(t, VOCAB.length); VOCAB.push(t); }
VOCAB_AT[k + 1] = VOCAB.length;
});
/* ---------- pre-tokenizer + BPE ---------- */
var PAT = /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu;
var encoder = new TextEncoder();
var strictDec = new TextDecoder("utf-8", { fatal: true });
function toByteChars(s) {
var bytes = encoder.encode(s), out = "";
for (var i = 0; i < bytes.length; i++) out += B2U[bytes[i]];
return out;
}
var cache = new Map(); // key: limit + " " + word
function bpe(word, limit, trace) {
var key;
if (!trace) {
key = limit + " " + word;
var hit = cache.get(key);
if (hit) return hit;
}
var parts = Array.from(word);
if (limit > 0) {
while (parts.length > 1) {
var best = Infinity, bIdx = -1;
for (var i = 0; i < parts.length - 1; i++) {
var r = RANK.get(parts[i] + " " + parts[i + 1]);
if (r !== undefined && r < limit && r < best) { best = r; bIdx = i; }
}
if (bIdx < 0) break;
if (trace) trace.push({ rank: best, at: bIdx });
parts.splice(bIdx, 2, parts[bIdx] + parts[bIdx + 1]);
}
}
if (!trace) {
if (cache.size > 60000) cache.clear();
cache.set(key, parts);
}
return parts;
}
var MAX_CHARS = 20000; // tokenize at most this much
var MAX_CHIPS = 1200; // render at most this many chips
var CURVE_CHARS = 3000; // curve is computed on at most this much
function splitPieces(text) {
var out = [];
PAT.lastIndex = 0;
var m;
while ((m = PAT.exec(text)) !== null) out.push(m[0]);
return out;
}
function tokenize(pieces, limit) {
var groups = [], total = 0;
for (var i = 0; i < pieces.length; i++) {
var w = toByteChars(pieces[i]);
var parts = bpe(w, limit, null);
groups.push({ raw: pieces[i], word: w, parts: parts });
total += parts.length;
}
return { groups: groups, total: total };
}
function countTokens(words, limit) {
var n = 0;
for (var i = 0; i < words.length; i++) n += bpe(words[i], limit, null).length;
return n;
}
/* ---------- display helpers ---------- */
var decCache = new Map();
function decodeToken(tok) {
var hit = decCache.get(tok);
if (hit) return hit;
var arr = new Uint8Array(Array.from(tok).map(function (c) { return U2B.get(c); }));
var res;
try { res = { text: strictDec.decode(arr), ok: true }; }
catch (e) {
var hex = [];
for (var i = 0; i < arr.length; i++) hex.push(arr[i].toString(16).toUpperCase().padStart(2, "0"));
res = { hex: hex, ok: false };
}
if (decCache.size < 8000) decCache.set(tok, res);
return res;
}
function esc(s) {
return s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """);
}
var showWs = true;
function renderText(t) {
var out = "";
for (var ch of t) {
if (ch === "\n") out += '<span class="ws">⏎</span>';
else if (ch === "\t") out += '<span class="ws">⇥</span>';
else if (ch === " ") out += showWs ? '<span class="ws">·</span>' : " ";
else if (ch === "\r") out += '<span class="ws">␍</span>';
else {
var cp = ch.codePointAt(0);
if (cp < 32 || cp === 127) out += '<span class="ws">⟨' + cp.toString(16).toUpperCase().padStart(2, "0") + '⟩</span>';
else out += esc(ch);
}
}
return out === "" ? '<span class="ws">∅</span>' : out;
}
function chipInner(tok, showIds) {
if (showIds) {
var id = IDS.get(tok);
return String(id === undefined ? "?" : id);
}
var d = decodeToken(tok);
if (d.ok) return renderText(d.text);
return d.hex.map(function (h) { return '<span class="hexb">⟨' + h + '⟩</span>'; }).join("");
}
function tokLabel(tok) {
var d = decodeToken(tok);
var shown = d.ok ? JSON.stringify(d.text) : "bytes " + d.hex.join(" ");
return "token " + IDS.get(tok) + ": " + shown;
}
function fmt(n) { return n.toLocaleString(); }
/* ---------- DOM ---------- */
var $ = function (id) { return document.getElementById(id); };
var input = $("input"), tokensEl = $("tokens"), traceEl = $("trace"),
limEl = $("lim"), curveEl = $("curve"), flagEl = $("flag"), dotEl = $("dot");
var sampleBtns = document.querySelectorAll("[data-sample]");
var showIds = false, limit = MAX_MERGES, animating = false, chipCap = 1200;
var pieces = [], curveWords = [];
var current = { groups: [], total: 0, chars: 0, bytes: 0, words: 0, truncated: false };
var selectedGroup = 0, focusIdx = 0, flatChips = [];
var SAMPLES = {
mixed:
"The quick brown fox jumps over the lazy dog — and every tokenizer sees that sentence differently.\n\n" +
"A language model never reads letters. It reads tokens: frequent fragments fused into one symbol, rare ones\n" +
"shattered back into bytes. Watch antidisestablishmentarianism fall apart 🧩 while \" the\" survives whole.\n\n" +
"def tokenize(text):\n" +
" return [vocab[t] for t in split(text)]\n\n" +
"Emoji are four UTF-8 bytes each 🚀🌍, so they land here as raw byte tokens — no vocabulary entry at all.",
rare:
"antidisestablishmentarianism pneumonoultramicroscopicsilicovolcanoconiosis\n" +
"floccinaucinihilipilification supercalifragilisticexpialidocious\n\n" +
"Compare with the words a corpus sees constantly: the, and, of, to, is, that, with, from, this.\n\n" +
"Proper nouns fragment too: Anthropic, Kubernetes, Reykjavík, Xochimilco, Nguyen, Þórsmörk.",
numbers:
"Order #A-77219 shipped 2026-08-13 at 09:41:07Z for $1,284.95 (+8.25% tax).\n" +
"sha256: 9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08\n" +
"https://sloprun.dev/demos/tokenizer?utm_source=share&ref=42\n" +
"3.14159265358979 · 6.02214076e23 · 0xDEADBEEF · 192.168.0.117:8443\n\n" +
"Long digit runs are expensive: 100000000000 costs more tokens than one hundred billion.",
world:
"English: A tokenizer trained on English is unfair to everything else.\n" +
"Español: Un tokenizador entrenado en inglés no es justo con lo demás.\n" +
"Français: Un tokeniseur entraîné sur l'anglais désavantage les autres langues.\n" +
"Deutsch: Donaudampfschifffahrtsgesellschaftskapitän.\n" +
"Ελληνικά: Η γλώσσα δεν χωράει στο λεξιλόγιο.\n" +
"हिन्दी: यह वाक्य बाइट-दर-बाइट टूट जाएगा।\n" +
"日本語: このモデルは日本語の語彙を持っていません。\n" +
"العربية: هذا النص سيتحول إلى وحدات بايت."
};
/* ---------- text -> pieces (only when the text changes) ---------- */
function reparse() {
var raw = input.value;
var truncated = raw.length > MAX_CHARS;
var text = raw;
if (truncated) {
text = raw.slice(0, MAX_CHARS);
var last = text.charCodeAt(text.length - 1);
if (last >= 0xD800 && last <= 0xDBFF) text = text.slice(0, -1); // never split a surrogate pair
}
pieces = splitPieces(text);
var curveText = text.slice(0, CURVE_CHARS);
curveSampled = curveText.length !== text.length;
if (!curveSampled) curveWords = wordsFrom(pieces);
else curveWords = wordsFrom(splitPieces(curveText));
current.chars = Array.from(text).length;
current.bytes = encoder.encode(text).length;
var trimmed = text.trim();
current.words = trimmed === "" ? 0 : trimmed.split(/\s+/).length;
current.truncated = truncated;
current.rawChars = truncated ? Array.from(raw).length : current.chars;
// counting a piece once per occurrence keeps the curve honest
curveCounts = null;
}
function wordsFrom(ps) {
var out = [];
for (var i = 0; i < ps.length; i++) out.push(toByteChars(ps[i]));
return out;
}
/* ---------- curve ---------- */
var CURVE_LIMITS = [0, 2, 5, 10, 18, 30, 50, 75, 110, 160, 220, 300, 400, 520, 660, 800];
var curveCounts = null, curveSampled = false;
function xOf(l) { return 100 * l / MAX_MERGES; }
function renderCurve() {
if (!curveCounts) {
curveCounts = CURVE_LIMITS.map(function (l) { return countTokens(curveWords, l); });
}
var maxT = curveCounts[0] || 1;
var pts = CURVE_LIMITS.map(function (l, i) {
return xOf(l).toFixed(2) + "," + (40 - 38 * (curveCounts[i] / maxT)).toFixed(2);
});
var line = pts.join(" ");
var area = "0,40 " + line + " 100,40";
var mx = xOf(limit);
var myTokens = countTokens(curveWords, limit);
var my = 40 - 38 * (maxT ? myTokens / maxT : 0);
curveEl.innerHTML =
'<polygon points="' + area + '" fill="var(--accent-soft)"></polygon>' +
'<polyline points="' + line + '" fill="none" stroke="var(--accent)" stroke-width="1.6" ' +
'vector-effect="non-scaling-stroke" stroke-linejoin="round"></polyline>' +
'<line x1="' + mx.toFixed(2) + '" y1="0" x2="' + mx.toFixed(2) + '" y2="40" stroke="var(--accent)" ' +
'stroke-width="1" stroke-dasharray="3 3" vector-effect="non-scaling-stroke"></line>';
var h = curveEl.clientHeight || 84;
var topPx = (my / 40) * h;
dotEl.hidden = false;
dotEl.style.left = mx.toFixed(2) + "%";
dotEl.style.top = topPx.toFixed(1) + "px";
flagEl.hidden = false;
flagEl.textContent = (curveSampled ? "\u2248 " : "") + fmt(myTokens) + " tok";
flagEl.style.left = Math.min(92, Math.max(8, mx)) + "%";
flagEl.style.top = Math.max(0, Math.min(h - 17, topPx + 9)).toFixed(1) + "px";
}
/* ---------- render ---------- */
function run(opts) {
var light = opts && opts.light;
var t0 = performance.now();
var res = tokenize(pieces, limit);
current.groups = res.groups;
current.total = res.total;
renderChips();
renderStats();
renderBudget();
renderCost();
if (!light) renderIds();
if (!light) { // #runstat is aria-live: never rewrite it 60x/second mid-replay
var ms = Math.max(1, Math.round(performance.now() - t0));
var st = $("runstat");
if (res.total > 0) { st.className = "status ok"; st.textContent = "✓ tokenized in " + ms + " ms"; }
else { st.className = "status"; st.textContent = "waiting for text"; }
}
$("charline").textContent = fmt(current.rawChars) + " chars in";
var cap = $("capnote"), msgs = [];
if (current.truncated) msgs.push("Input truncated to " + fmt(MAX_CHARS) + " characters for this demo.");
if (res.total > chipCap && !animating) msgs.push("Showing the first " + fmt(chipCap) + " of " + fmt(res.total) +
" chips; the counts above cover everything.");
var capText = msgs.join(" ");
if (capText !== cap.textContent) cap.textContent = capText; // no attribute churn mid-replay
if (cap.hidden === !!capText) cap.hidden = !capText;
if (selectedGroup >= current.groups.length) selectedGroup = pickInteresting();
markCurrent();
if (!light) renderTrace();
}
function pickInteresting() {
var best = 0, bestLen = -1;
var lim = Math.min(current.groups.length, 400);
for (var i = 0; i < lim; i++) {
var g = current.groups[i];
var span = Array.from(g.word).length - g.parts.length;
if (span > bestLen) { bestLen = span; best = i; }
}
return best;
}
function renderChips() {
var html = [], n = 0, cap = animating ? 400 : MAX_CHIPS;
chipCap = cap;
flatChips = [];
for (var gi = 0; gi < current.groups.length && n < cap; gi++) {
var parts = current.groups[gi].parts;
for (var pi = 0; pi < parts.length && n < cap; pi++) {
var tok = parts[pi];
var id = IDS.get(tok);
var cls = "tok c" + ((id === undefined ? 0 : id) % 6) + (showIds ? " idmode" : "");
var lbl = esc(tokLabel(tok));
html.push('<button type="button" class="' + cls + '" data-g="' + gi + '" tabindex="-1" title="' +
lbl + '" aria-label="' + lbl + '">' + chipInner(tok, showIds) + '</button>');
flatChips.push(gi);
n++;
}
}
var hadFocus = tokensEl.contains(document.activeElement);
tokensEl.innerHTML = html.join("");
if (focusIdx >= flatChips.length) focusIdx = 0;
var kids = tokensEl.children;
if (kids.length) {
kids[focusIdx].tabIndex = 0;
if (hadFocus) kids[focusIdx].focus();
}
markCurrent();
}
function markCurrent() {
var kids = tokensEl.children;
for (var i = 0; i < kids.length; i++) {
if (flatChips[i] === selectedGroup) kids[i].setAttribute("aria-current", "true");
else kids[i].removeAttribute("aria-current");
}
}
function renderStats() {
$("sTok").textContent = fmt(current.total);
$("sChar").textContent = fmt(current.chars);
$("sWord").textContent = fmt(current.words);
$("sByte").textContent = fmt(current.bytes);
var ratio = current.total ? (current.chars / current.total) : 0;
$("sRatio").textContent = ratio ? ratio.toFixed(2) : "—";
}
function renderBudget() {
$("rLim").textContent = fmt(limit);
$("rVocab").textContent = fmt(VOCAB_AT[limit]);
$("rTok").textContent = fmt(current.total);
limEl.setAttribute("aria-valuetext", limit + " merges, " + VOCAB_AT[limit] + "-token vocabulary, " +
current.total + " tokens for this text");
renderCurve();
}
/* ---------- merge trace ---------- */
function renderTrace() {
var g = current.groups[selectedGroup];
if (!g) { traceEl.innerHTML = '<p class="tracehead">Type something, then click a token chip.</p>'; return; }
var trace = [];
var parts = bpe(g.word, limit, trace);
var seq = Array.from(g.word);
var head = '<p class="tracehead">piece <b>' + esc(JSON.stringify(g.raw)) + '</b> · ' +
seq.length + ' byte' + (seq.length === 1 ? '' : 's') + ' → ' + parts.length + ' token' + (parts.length === 1 ? '' : 's') + ' · ' +
trace.length + ' merge' + (trace.length === 1 ? '' : 's') + '</p>';
var rows = [step("bytes", seq, -1)];
for (var i = 0; i < trace.length; i++) {
var s = trace[i];
seq = seq.slice();
seq.splice(s.at, 2, seq[s.at] + seq[s.at + 1]);
rows.push(step("#" + String(s.rank).padStart(3, "0"), seq, s.at));
}
if (!trace.length) rows.push('<p class="note" style="margin:6px 0 0">No merge in the budget applies — ' +
'this piece stays as raw bytes.</p>');
traceEl.innerHTML = head + rows.join("");
}
function step(label, seq, hi) {
var chips = seq.map(function (p, i) {
var d = decodeToken(p);
var inner = d.ok ? renderText(d.text) : d.hex.map(function (h) { return "⟨" + h + "⟩"; }).join("");
return '<span class="pc' + (i === hi ? ' new' : '') + '">' + inner + '</span>';
}).join("");
return '<div class="step"><span class="rk">' + esc(label) + '</span><span class="seq">' + chips + '</span></div>';
}
/* ---------- cost ---------- */
var MODELS = [
{ name: "Claude Opus 5", rate: 5.00 },
{ name: "Claude Sonnet 5", rate: 3.00 },
{ name: "Claude Haiku 4.5", rate: 1.00 }
];
var basis = "prod";
function basisTokens() {
if (basis === "demo") return current.total;
return current.chars ? Math.max(1, Math.round(current.chars / 4)) : 0;
}
function money(v) {
if (v === 0) return "$0.00";
if (v < 0.01) return "$" + v.toFixed(5);
if (v < 1) return "$" + v.toFixed(4);
return "$" + v.toLocaleString(undefined, { minimumFractionDigits: 2, maximumFractionDigits: 2 });
}
function renderCost() {
var toks = basisTokens();
$("basisTok").textContent = fmt(toks);
var body = $("costBody");
if (!body.dataset.built) {
body.innerHTML = MODELS.map(function (m, i) {
return '<tr><td>' + esc(m.name) + '</td>' +
'<td><input type="number" id="rate' + i + '" name="rate' + i + '" min="0" step="0.01" value="' +
m.rate.toFixed(2) + '" data-rate="' + i +
'" aria-label="Dollars per million input tokens for ' + esc(m.name) + '"></td>' +
'<td data-one="' + i + '">—</td><td data-many="' + i + '">—</td></tr>';
}).join("");
body.dataset.built = "1";
body.addEventListener("input", function (e) {
var idx = e.target.getAttribute("data-rate");
if (idx === null) return;
var v = parseFloat(e.target.value);
MODELS[+idx].rate = isFinite(v) && v >= 0 ? v : 0;
renderCost();
});
}
MODELS.forEach(function (m, i) {
var one = toks / 1e6 * m.rate;
body.querySelector('[data-one="' + i + '"]').textContent = money(one);
body.querySelector('[data-many="' + i + '"]').textContent = money(one * 10000);
});
}
function renderIds() {
var out = [], n = 0;
for (var gi = 0; gi < current.groups.length && n < 4000; gi++) {
var parts = current.groups[gi].parts;
for (var pi = 0; pi < parts.length && n < 4000; pi++) { out.push(IDS.get(parts[pi])); n++; }
}
$("idsOut").textContent = "[" + out.join(", ") + (current.total > n ? ", … " + (current.total - n) + " more" : "") + "]";
}
/* ---------- events ---------- */
var timer = null;
input.addEventListener("input", function () {
markSample(null);
if (timer) clearTimeout(timer);
timer = setTimeout(function () { reparse(); selectedGroup = 0; run(); }, 110);
});
limEl.addEventListener("input", function () {
limit = +limEl.value;
stopPlay();
run({ light: true });
renderTrace();
});
limEl.addEventListener("change", function () { // slider released: full run so token IDs aren't stale
limit = +limEl.value;
run();
});
tokensEl.addEventListener("click", function (e) {
var btn = e.target.closest ? e.target.closest(".tok") : null;
if (!btn) return;
selectedGroup = +btn.getAttribute("data-g");
var idx = Array.prototype.indexOf.call(tokensEl.children, btn);
if (idx >= 0) { // keep the roving tabindex on the chip just clicked
var kids = tokensEl.children;
if (kids[focusIdx]) kids[focusIdx].tabIndex = -1;
focusIdx = idx;
btn.tabIndex = 0;
}
markCurrent();
renderTrace();
});
tokensEl.addEventListener("keydown", function (e) {
var kids = tokensEl.children;
if (!kids.length) return;
var next = null;
if (e.key === "ArrowRight" || e.key === "ArrowDown") next = Math.min(focusIdx + 1, kids.length - 1);
else if (e.key === "ArrowLeft" || e.key === "ArrowUp") next = Math.max(focusIdx - 1, 0);
else if (e.key === "Home") next = 0;
else if (e.key === "End") next = kids.length - 1;
if (next === null) return;
e.preventDefault();
kids[focusIdx].tabIndex = -1;
focusIdx = next;
kids[focusIdx].tabIndex = 0;
kids[focusIdx].focus();
selectedGroup = flatChips[focusIdx];
markCurrent();
renderTrace();
});
function markSample(key) {
for (var i = 0; i < sampleBtns.length; i++) {
var on = sampleBtns[i].getAttribute("data-sample") === key;
sampleBtns[i].classList.toggle("on", on);
sampleBtns[i].setAttribute("aria-pressed", String(on));
}
}
Array.prototype.forEach.call(sampleBtns, function (b) {
b.addEventListener("click", function () {
input.value = SAMPLES[b.getAttribute("data-sample")];
markSample(b.getAttribute("data-sample"));
focusIdx = 0;
reparse();
selectedGroup = 0;
run();
selectedGroup = pickInteresting();
markCurrent();
renderTrace();
});
});
$("btnIds").addEventListener("click", function () {
showIds = !showIds;
this.setAttribute("aria-pressed", String(showIds));
renderChips();
});
$("btnWs").addEventListener("click", function () {
showWs = !showWs;
this.setAttribute("aria-pressed", String(showWs));
renderChips(); renderTrace();
});
function setBasis(b) {
basis = b;
var p = $("basisProd"), d = $("basisDemo");
p.classList.toggle("on", b === "prod"); p.setAttribute("aria-pressed", String(b === "prod"));
d.classList.toggle("on", b === "demo"); d.setAttribute("aria-pressed", String(b === "demo"));
renderCost();
}
$("basisProd").addEventListener("click", function () { setBasis("prod"); });
$("basisDemo").addEventListener("click", function () { setBasis("demo"); });
/* ---------- replay ---------- */
var playing = false, rafId = 0, playBtn = $("btnPlay");
function reduceMotion() {
return !!(window.matchMedia && window.matchMedia("(prefers-reduced-motion: reduce)").matches);
}
function stopPlay() {
if (!playing) return;
playing = false; animating = false;
if (rafId) cancelAnimationFrame(rafId);
playBtn.textContent = "▶ replay training";
run();
renderTrace();
}
playBtn.addEventListener("click", function () {
if (playing) { stopPlay(); return; }
if (reduceMotion()) { // no animation: show the two endpoints instead
limit = limit === 0 ? MAX_MERGES : 0;
limEl.value = limit;
run(); renderTrace();
return;
}
playing = true; animating = true;
playBtn.textContent = "■ stop";
var t0 = performance.now(), DUR = 2600;
(function frame(now) {
if (!playing) return;
// a rAF timestamp can predate t0 — clamp, or Math.pow of a negative base gives NaN
var p = Math.min(1, Math.max(0, (now - t0) / DUR));
limit = Math.round(Math.pow(p, 2.2) * MAX_MERGES); // slow start: the early merges matter most
limEl.value = limit;
run({ light: true });
if (p < 1) rafId = requestAnimationFrame(frame);
else { playing = false; animating = false; playBtn.textContent = "▶ replay training"; run(); renderTrace(); }
})(t0);
});
/* ---------- platform theme ---------- */
window.addEventListener("message", function (e) {
var d = e.data;
if (d && d.type === "sloprun:theme" && (d.theme === "light" || d.theme === "dark")) {
document.documentElement.setAttribute("data-theme", d.theme);
}
});
window.addEventListener("resize", function () { if (!playing) renderCurve(); });
/* ---------- boot ---------- */
input.value = SAMPLES.mixed;
markSample("mixed");
reparse();
run();
selectedGroup = pickInteresting();
markCurrent();
renderTrace();
})();
</script>
</body>
</html>
source-visible by construction · nothing is published here without its code
Comments
0 totalNo comments yet. If you ran it, say what happened.