Mirror a site you own. One file, zero installs.

1// universal-scrape.mjs — one file, zero dependencies (Node 20+).2// -----------------------------------------------------------------------------3// Mirror a site you own or are authorised to archive — your own dashboards, your4// SaaS content, public documentation, a migration export — to disk, using your own5// logged-in session. Everything the browser can see with your cookie, this saves.6//7// AUTHORISED USE ONLY. Run this only against sites you own or have explicit8// permission to copy, and respect each site's terms of service and copyright.9// The tool sends requests only to the URLs you give it and writes only to the10// output folder you choose. No dependencies, no telemetry, nothing phones home.11//12// DevLune · https://devlune.in · MIT licence. Use at your own risk.13// -----------------------------------------------------------------------------14// what it does:15//   1) reads config from scrape.config.json (or CLI flags) — seeds, auth, filters16//   2) enumerates URLs via sitemap.xml + BFS link discovery from seeds17//   3) fetches every page with your cookie/bearer, mirrors HTML to disk18//   4) extracts clean text + Next.js RSC flight prose19//   5) mirrors CSS + JS + fonts + images + audio/video referenced from the HTML20//   6) resumable: keeps state in .cache/seen.jsonl, safe to re-run21//22// usage:23//   node universal-scrape.mjs                             (uses scrape.config.json)24//   node universal-scrape.mjs --seed https://site/x       (ad-hoc single seed)25//   node universal-scrape.mjs --config other.json26//27// scrape.config.json shape:28//   {29//     "name": "myproject",30//     "seeds": ["https://site.com/dashboard","https://site.com/courses"],31//     "cookieFile": "cookie.txt",           // OR "cookie": "raw cookie string"32//     "bearerFile": "bearer.txt",           // OR "bearer": "Bearer eyJ…"33//     "headers": { "x-api-key": "…" },34//     "outDir": "output",35//     "include": ["^/(courses|lessons|packs|articles)"],   // regex on pathname36//     "exclude": ["/logout","/api/track","\\.png$"],37//     "maxPages": 5000,38//     "delayMs": 500,39//     "concurrency": 1,                     // sequential is safer for auth cookies40//     "assets": true,                       // mirror css/js/fonts/media41//     "sitemap": true,                      // try /sitemap.xml first42//     "sameHostOnly": true,43//     "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) …",44//     "referer": "https://site.com/dashboard",45//     "authProbe": { "loginPath": "/login", "minBodyBytes": 500 }46//   }47// -----------------------------------------------------------------------------48 49import fs from 'node:fs/promises';50import { existsSync, createWriteStream } from 'node:fs';51import path from 'node:path';52import { URL } from 'node:url';53import { setTimeout as sleep } from 'node:timers/promises';54import { parseArgs } from 'node:util';55 56// ─── args + config ──────────────────────────────────────────────────────────57const { values: A } = parseArgs({58  options: {59    config: { type: 'string', default: 'scrape.config.json' },60    seed:   { type: 'string', multiple: true },61    out:    { type: 'string' },62    max:    { type: 'string' },63    cookie: { type: 'string' },64    bearer: { type: 'string' },65    'no-assets':  { type: 'boolean' },66    'no-sitemap': { type: 'boolean' },67    reset:  { type: 'boolean' },68    verbose:{ type: 'boolean' },69  },70});71 72let cfg = {};73try { cfg = JSON.parse(await fs.readFile(A.config, 'utf8')); }74catch { if (!A.seed?.length) throw new Error(`no ${A.config} and no --seed passed`); }75 76if (A.seed?.length) cfg.seeds = A.seed;77if (A.out)          cfg.outDir = A.out;78if (A.max)          cfg.maxPages = +A.max;79if (A.cookie)       cfg.cookie = A.cookie;80if (A.bearer)       cfg.bearer = A.bearer;81if (A['no-assets'])  cfg.assets = false;82if (A['no-sitemap']) cfg.sitemap = false;83 84// A config with no seeds (and no --seed) would blow up on `cfg.seeds[0]` below with an85// opaque TypeError. Fail early with a message that says exactly what to fix.86if (!cfg.seeds?.length) {87  console.error('No seeds. Add "seeds" to your config, or pass --seed <url>.');88  process.exit(1);89}90 91cfg.name         ??= new URL(cfg.seeds[0]).host.replace(/^www\./, '');92cfg.outDir       ??= 'output-' + cfg.name;93cfg.maxPages     ??= 5000;94cfg.delayMs      ??= 500;95cfg.concurrency  ??= 1;96cfg.assets       ??= true;97cfg.sitemap      ??= true;98cfg.sameHostOnly ??= true;99cfg.include      ??= [];100cfg.exclude      ??= [];101cfg.userAgent    ??= 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36';102cfg.referer      ??= cfg.seeds[0];103cfg.authProbe    ??= { minBodyBytes: 500 };104 105// ─── auth loading ───────────────────────────────────────────────────────────106if (!cfg.cookie && cfg.cookieFile && existsSync(cfg.cookieFile)) cfg.cookie = (await fs.readFile(cfg.cookieFile, 'utf8')).trim();107if (!cfg.bearer && cfg.bearerFile && existsSync(cfg.bearerFile)) cfg.bearer = (await fs.readFile(cfg.bearerFile, 'utf8')).trim();108 109const AUTH_HEADERS = {110  'user-agent': cfg.userAgent,111  'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',112  'accept-language': 'en-US,en;q=0.9',113  'referer': cfg.referer,114  ...(cfg.cookie ? { 'cookie': cfg.cookie } : {}),115  ...(cfg.bearer ? { 'authorization': cfg.bearer.startsWith('Bearer ') ? cfg.bearer : `Bearer ${cfg.bearer}` } : {}),116  ...(cfg.headers || {}),117};118 119const seedHosts = new Set(cfg.seeds.map(s => new URL(s).host));120const CACHE_DIR = path.join(cfg.outDir, '.cache');121await fs.mkdir(CACHE_DIR, { recursive: true });122const SEEN_FILE = path.join(CACHE_DIR, 'seen.jsonl');123if (A.reset && existsSync(SEEN_FILE)) await fs.unlink(SEEN_FILE);124 125// ─── seen / resume ──────────────────────────────────────────────────────────126const seen = new Map(); // url -> { status, bytes, ts }127if (existsSync(SEEN_FILE)) {128  const raw = await fs.readFile(SEEN_FILE, 'utf8');129  for (const line of raw.split('\n')) if (line.trim()) {130    try { const j = JSON.parse(line); seen.set(j.url, j); } catch {}131  }132  console.log('[resume] loaded', seen.size, 'seen entries');133}134const seenStream = createWriteStream(SEEN_FILE, { flags: 'a' });135const markSeen = (rec) => { seen.set(rec.url, rec); seenStream.write(JSON.stringify(rec) + '\n'); };136 137// ─── URL policy ─────────────────────────────────────────────────────────────138const INC = cfg.include.map(r => new RegExp(r));139const EXC = cfg.exclude.map(r => new RegExp(r));140const shouldFetch = (u) => {141  try {142    const url = new URL(u);143    if (cfg.sameHostOnly && !seedHosts.has(url.host)) return false;144    const p = url.pathname + url.search;145    if (EXC.some(rx => rx.test(p))) return false;146    if (INC.length && !INC.some(rx => rx.test(p))) return false;147    return true;148  } catch { return false; }149};150 151// ─── HTTP with retries ──────────────────────────────────────────────────────152async function get(u, tries = 3) {153  let lastErr;154  for (let i = 1; i <= tries; i++) {155    try {156      const controller = new AbortController();157      const t = setTimeout(() => controller.abort(), 30000);158      const res = await fetch(u, { headers: AUTH_HEADERS, redirect: 'follow', signal: controller.signal });159      clearTimeout(t);160      const buf = Buffer.from(await res.arrayBuffer());161      if (res.status === 429) throw new Error('rate limited');162      const isHtml = (res.headers.get('content-type') || '').includes('html');163      if (res.status === 401 || res.status === 403) return { url: u, status: res.status, buf, headers: Object.fromEntries(res.headers), authFail: true };164      if (isHtml && buf.length < (cfg.authProbe.minBodyBytes || 0) && !cfg.cookie && !cfg.bearer) {165        return { url: u, status: res.status, buf, headers: Object.fromEntries(res.headers), suspiciousShort: true };166      }167      return { url: u, status: res.status, buf, headers: Object.fromEntries(res.headers) };168    } catch (e) {169      lastErr = e;170      if (i < tries) await sleep(1000 * i);171    }172  }173  throw lastErr;174}175 176// ─── extract links + assets from HTML ───────────────────────────────────────177const LINK_RX = /(?:href|action|src|data-href|data-url)\s*=\s*["']([^"'#]+)/gi;178const CSS_URL_RX = /url\(["']?([^"')]+)["']?\)/gi;179const SRCSET_RX = /srcset\s*=\s*["']([^"']+)["']/gi;180function extractLinks(html, baseUrl) {181  const out = new Set();182  let m;183  while ((m = LINK_RX.exec(html)))  { try { out.add(new URL(m[1], baseUrl).toString()); } catch {} }184  while ((m = CSS_URL_RX.exec(html))) { try { out.add(new URL(m[1], baseUrl).toString()); } catch {} }185  while ((m = SRCSET_RX.exec(html))) {186    for (const part of m[1].split(',')) { const u = part.trim().split(/\s+/)[0]; try { out.add(new URL(u, baseUrl).toString()); } catch {} }187  }188  return [...out];189}190 191const ASSET_EXT = /\.(css|js|mjs|woff2?|ttf|otf|svg|png|jpe?g|gif|webp|avif|ico|mp3|mp4|m3u8|webm|ogg|pdf|json|xml|txt)($|\?)/i;192const isAsset = (u) => ASSET_EXT.test(u) || /\/(_next\/static|_next\/image|fonts?|media|images?|audio|video|assets?)\//i.test(u);193const isDocument = (u) => !isAsset(u);194 195// ─── content extraction (Next.js RSC + generic) ─────────────────────────────196function extractContent(html) {197  const flightChunks = [];198  const rx = /self\.__next_f\.push\(\[1,"((?:[^"\\]|\\.)*)"\]\)/g;199  let m; while ((m = rx.exec(html))) {200    flightChunks.push(m[1]201      .replace(/\\n/g, '\n').replace(/\\r/g, '').replace(/\\t/g, '\t')202      .replace(/\\"/g, '"').replace(/\\\\/g, '\\')203      .replace(/\\u([0-9a-fA-F]{4})/g, (_, h) => String.fromCharCode(parseInt(h, 16))));204  }205  const flight = flightChunks.join('\n');206  const stripped = html207    .replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, '')208    .replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, '')209    .replace(/<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi, '')210    .replace(/<[^>]+>/g, ' ')211    .replace(/&nbsp;/g, ' ').replace(/&amp;/g, '&').replace(/&lt;/g, '<')212    .replace(/&gt;/g, '>').replace(/&quot;/g, '"').replace(/&#39;/g, "'")213    .replace(/&#x27;/g, "'").replace(/&#x2F;/g, '/')214    .replace(/\s+\n/g, '\n').replace(/\n\s+/g, '\n').replace(/[ \t]+/g, ' ').trim();215  const title = (html.match(/<title[^>]*>([^<]+)/i) || [])[1]?.trim() || null;216  const h1 = (html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i) || [])[1]?.replace(/<[^>]+>/g, ' ').trim() || null;217  return { text: stripped, flight, title, h1 };218}219 220// ─── file layout: mirror URL path onto disk ─────────────────────────────────221function urlToPath(u, root) {222  const url = new URL(u);223  let p = url.pathname;224  if (p.endsWith('/') || p === '') p += 'index';225  let name = path.join(root, url.host, p);226  const q = url.search ? '__' + Buffer.from(url.search).toString('base64url').slice(0, 32) : '';227  name = name.replace(/[?#].*$/, '') + q;228  return name.replace(/[<>:"|*]/g, '_');229}230 231async function writeArtifact(u, buf, headers) {232  const base = urlToPath(u, cfg.outDir);233  await fs.mkdir(path.dirname(base), { recursive: true });234  const ct = (headers['content-type'] || '').toLowerCase();235  if (ct.includes('html')) {236    await fs.writeFile(base + '.html', buf);237    const html = buf.toString('utf8');238    const c = extractContent(html);239    if (c.text)   await fs.writeFile(base + '.txt', c.text);240    if (c.flight && c.flight.length > 500) await fs.writeFile(base + '.flight.txt', c.flight);241    await fs.writeFile(base + '.json', JSON.stringify({ url: u, title: c.title, h1: c.h1, bytes: buf.length, textBytes: c.text.length }, null, 2));242    return { html, extracted: c };243  }244  await fs.writeFile(base, buf);245  return {};246}247 248// ─── sitemap enumeration ────────────────────────────────────────────────────249async function trySitemap(hostBase) {250  const found = new Set();251  for (const p of ['/sitemap.xml', '/sitemap_index.xml', '/robots.txt']) {252    try {253      const r = await get(hostBase + p);254      if (r.status !== 200) continue;255      const body = r.buf.toString('utf8');256      const locs = [...body.matchAll(/<loc>([^<]+)/gi)].map(m => m[1]);257      const smaps = locs.filter(u => u.endsWith('.xml'));258      for (const u of locs) found.add(u);259      for (const sm of smaps) {260        try {261          const rr = await get(sm);262          const inner = rr.buf.toString('utf8');263          for (const m of inner.matchAll(/<loc>([^<]+)/gi)) found.add(m[1]);264        } catch {}265      }266      if (p === '/robots.txt') {267        for (const m of body.matchAll(/Sitemap:\s*(\S+)/gi)) {268          try {269            const rr = await get(m[1]);270            for (const mm of rr.buf.toString('utf8').matchAll(/<loc>([^<]+)/gi)) found.add(mm[1]);271          } catch {}272        }273      }274    } catch {}275  }276  return [...found];277}278 279// ─── crawl loop ─────────────────────────────────────────────────────────────280const docQueue = [];281const assetQueue = new Set();282const queued = new Set();283 284const enqueueDoc = (u) => {285  if (queued.has(u) || seen.has(u)) return;286  if (!shouldFetch(u)) return;287  queued.add(u); docQueue.push(u);288};289const enqueueAsset = (u) => { if (!seen.has(u)) assetQueue.add(u); };290 291for (const s of cfg.seeds) enqueueDoc(s);292 293if (cfg.sitemap) {294  for (const host of seedHosts) {295    const base = 'https://' + host;296    console.log('[sitemap] probing', base);297    const found = await trySitemap(base);298    console.log('[sitemap] found', found.length, 'urls');299    for (const u of found) enqueueDoc(u);300  }301}302 303const stats = { docs: 0, docErr: 0, assets: 0, assetErr: 0, authFail: 0, bytes: 0 };304const startedAt = Date.now();305 306console.log(`[crawl] starting — ${docQueue.length} queued, max ${cfg.maxPages}`);307while (docQueue.length && stats.docs < cfg.maxPages) {308  const u = docQueue.shift();309  if (seen.has(u)) continue;310  try {311    const r = await get(u);312    stats.docs++; stats.bytes += r.buf.length;313    if (r.authFail) { stats.authFail++; console.warn(`  [${stats.docs}] AUTH ${r.status} ${u}`); markSeen({ url: u, status: r.status, bytes: r.buf.length, ts: Date.now(), authFail: true }); continue; }314    const { html, extracted } = await writeArtifact(u, r.buf, r.headers);315    if (A.verbose || stats.docs % 10 === 0) console.log(`  [${stats.docs}] ${r.status} ${r.buf.length.toString().padStart(7)}b ${extracted?.title ? '"'+extracted.title.slice(0,50)+'" ' : ''}${u}`);316    if (html) {317      const links = extractLinks(html, u);318      for (const l of links) {319        if (isDocument(l)) enqueueDoc(l);320        else if (cfg.assets) enqueueAsset(l);321      }322    }323    markSeen({ url: u, status: r.status, bytes: r.buf.length, ts: Date.now() });324  } catch (e) {325    stats.docErr++; console.warn(`  [ERR] ${u} — ${e.message}`);326    markSeen({ url: u, error: e.message, ts: Date.now() });327  }328  await sleep(cfg.delayMs);329}330 331console.log(`[crawl] docs done — ${stats.docs} fetched, ${stats.docErr} errors, ${stats.authFail} auth-blocked`);332 333// ─── assets ─────────────────────────────────────────────────────────────────334if (cfg.assets && assetQueue.size) {335  console.log(`[assets] downloading ${assetQueue.size}…`);336  for (const u of assetQueue) {337    if (seen.has(u)) continue;338    try {339      const r = await get(u);340      stats.assets++; stats.bytes += r.buf.length;341      const base = urlToPath(u, cfg.outDir);342      await fs.mkdir(path.dirname(base), { recursive: true });343      await fs.writeFile(base, r.buf);344      if (A.verbose || stats.assets % 25 === 0) console.log(`  [asset ${stats.assets}] ${r.status} ${r.buf.length}b ${u.slice(0, 90)}`);345      // If asset is CSS, mine it for further asset URLs346      if ((r.headers['content-type'] || '').includes('css')) {347        for (const m of r.buf.toString('utf8').matchAll(CSS_URL_RX)) {348          try { enqueueAsset(new URL(m[1], u).toString()); } catch {}349        }350      }351      markSeen({ url: u, status: r.status, bytes: r.buf.length, ts: Date.now(), asset: true });352    } catch (e) {353      stats.assetErr++;354      markSeen({ url: u, error: e.message, ts: Date.now(), asset: true });355    }356    await sleep(cfg.delayMs / 2);357  }358}359 360// ─── index ──────────────────────────────────────────────────────────────────361const idx = { name: cfg.name, generatedAt: new Date().toISOString(), stats, seeds: cfg.seeds, cfg: { ...cfg, cookie: !!cfg.cookie ? '[redacted]' : undefined, bearer: !!cfg.bearer ? '[redacted]' : undefined, headers: cfg.headers } };362await fs.writeFile(path.join(cfg.outDir, 'index.json'), JSON.stringify(idx, null, 2));363seenStream.end();364console.log(`\n[done] ${stats.docs} docs, ${stats.assets} assets, ${(stats.bytes/1024/1024).toFixed(1)} MB, ${Math.round((Date.now()-startedAt)/1000)}s`);365console.log(`       → ${cfg.outDir}/  (index.json holds the map; re-run to resume)`);