#!/usr/bin/env node /* Adds a LEGO set to the collection using Rebrickable API. Usage: node tools/add_lego.js Flow: 1) Search sets with the provided query 2) Interactive choice if multiple results 3) Fetch chosen set details and its theme 4) Create content/collections/lego/// - Always create images/ - Create data/images/ - Download all available set images from Rebrickable - Create data/images/.yaml with attribution - Create index.md with title and date (only if not existing), including markdown references to images found */ const fs = require('fs/promises'); const fsSync = require('fs'); const path = require('path'); const https = require('https'); const readline = require('readline'); const ROOT = process.cwd(); const LEGO_ROOT = path.join(ROOT, 'content', 'collections', 'lego'); const CONFIG_PATH = path.join(ROOT, 'tools', 'config.json'); const CACHE_ROOT = path.join(ROOT, 'tools', 'cache', 'rebrickable'); const ONE_WEEK_MS = 7 * 24 * 60 * 60 * 1000; function loadConfig() { const raw = fsSync.readFileSync(CONFIG_PATH, 'utf8'); return JSON.parse(raw); } function slugify(input) { return input .normalize('NFD') .replace(/\p{Diacritic}/gu, '') .toLowerCase() .replace(/[^a-z0-9]+/g, '-') .replace(/^-+|-+$/g, '') .replace(/-{2,}/g, '-'); } function prompt(question) { const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); return new Promise((resolve) => rl.question(question, (ans) => { rl.close(); resolve(ans.trim()); })); } function apiGetJson(host, path, apiKey) { return new Promise((resolve) => { const req = https.request({ host, path, method: 'GET', headers: { 'Authorization': `key ${apiKey}`, 'Accept': 'application/json', 'User-Agent': 'add_lego_script/1.0', }, }, (res) => { let data = ''; res.on('data', (chunk) => { data += chunk; }); res.on('end', () => { resolve(JSON.parse(data)); }); }); req.end(); }); } function downloadFile(url, destPath) { return new Promise((resolve) => { const file = fsSync.createWriteStream(destPath); https.get(url, (res) => { res.pipe(file); file.on('finish', () => file.close(() => resolve())); }); }); } async function ensureDir(dir) { await fs.mkdir(dir, { recursive: true }); } async function saveJson(relPath, data) { const out = path.join(CACHE_ROOT, relPath); await ensureDir(path.dirname(out)); await fs.writeFile(out, JSON.stringify(data, null, 2), 'utf8'); } async function cachedApiJson(relPath, host, apiPath, apiKey, label, options = {}) { const full = path.join(CACHE_ROOT, relPath); if (options.noCache) { const json = await apiGetJson(host, apiPath, apiKey); await ensureDir(path.dirname(full)); await fs.writeFile(full, JSON.stringify(json, null, 2), 'utf8'); if (label) console.log(`[fetch] ${label} <- ${apiPath} [nocache]`); return json; } try { const st = await fs.stat(full); const ageMs = Date.now() - st.mtimeMs; if (ageMs < ONE_WEEK_MS) { try { const raw = await fs.readFile(full, 'utf8'); const json = JSON.parse(raw); if (label) console.log(`[cache] ${label}`); return json; } catch (_) { // Corrupt cache, fall through to refetch if (label) console.log(`[stale] ${label} (cache unreadable) -> refetch`); } } else { if (label) console.log(`[stale] ${label} (age ${(ageMs / ONE_WEEK_MS).toFixed(2)} weeks) -> refetch`); } } catch (_) { // No cache, fetch } const json = await apiGetJson(host, apiPath, apiKey); await ensureDir(path.dirname(full)); await fs.writeFile(full, JSON.stringify(json, null, 2), 'utf8'); if (label) console.log(`[fetch] ${label} <- ${apiPath}`); return json; } function escapeMarkdownAlt(text) { return String(text || '') .replace(/\r?\n/g, ' ') .replace(/\\/g, '\\\\') .replace(/\[/g, '\\[') .replace(/\]/g, '\\]'); } async function readYamlTitle(yamlPath) { try { const content = await fs.readFile(yamlPath, 'utf8'); // Match title: "..." or title: '...' or title: text const m = content.match(/^\s*title\s*:\s*(?:\"([^\"]*)\"|'([^']*)'|([^\n\r#]+))/m); if (!m) return null; const val = (m[1] || m[2] || m[3] || '').trim(); return val; } catch (_) { return null; } } async function main() { // Args parsing: support --no-cache and a free-form query const argv = process.argv.slice(2); let noCache = false; const parts = []; for (const a of argv) { if (a === '--no-cache' || a === '--nocache' || a === '--fresh') { noCache = true; } else if (a === '--') { // everything after is query literal const idx = argv.indexOf('--'); parts.push(argv.slice(idx + 1).join(' ')); break; } else if (a.startsWith('--')) { // ignore unknown flags for now } else { parts.push(a); } } const query = parts.join(' ').trim(); if (!query) { console.log('Usage: node tools/add_lego.js [--no-cache] '); process.exit(1); } const config = loadConfig(); const apiKey = config.rebrickable.apiKey; const host = 'rebrickable.com'; // 1) Search (cached ≤ 1 week) const searchPath = `/api/v3/lego/sets/?search=${encodeURIComponent(query)}&page_size=25`; const search = await cachedApiJson( `search/${encodeURIComponent(query)}.json`, host, searchPath, apiKey, `search:${query}`, { noCache } ); const results = search.results || []; if (results.length === 0) { console.log('No sets found for query:', query); process.exit(0); } // 2) Selection let selected = results[0]; if (results.length > 1) { console.log(`Found ${results.length} sets:`); results.forEach((r, i) => { const id = (r.set_num || '').split('-')[0]; console.log(`${i + 1}. ${r.set_num} — ${r.name} (${r.year}) [id: ${id}]`); }); const ans = await prompt('Choose a set (1..N): '); const idx = Math.max(1, Math.min(results.length, parseInt(ans, 10) || 1)) - 1; selected = results[idx]; } const setNum = selected.set_num; // e.g., 10283-1 const setId = (setNum || '').split('-')[0]; // 3) Fetch exact set and theme (cached ≤ 1 week) const setDetails = await cachedApiJson( `sets/${setNum}.json`, host, `/api/v3/lego/sets/${encodeURIComponent(setNum)}/`, apiKey, `set:${setNum}`, { noCache } ); const themeId = setDetails.theme_id || selected.theme_id; const theme = await cachedApiJson( `themes/${themeId}.json`, host, `/api/v3/lego/themes/${themeId}/`, apiKey, `theme:${themeId}`, { noCache } ); const themeName = theme.name; const themeSlug = slugify(themeName); // 4) Prepare dirs const setDir = path.join(LEGO_ROOT, themeSlug, setId); const imagesDir = path.join(setDir, 'images'); const dataImagesDir = path.join(setDir, 'data', 'images'); await ensureDir(imagesDir); // created even if no images provided await ensureDir(dataImagesDir); // Images provided by Rebrickable (set images) const downloadedImages = new Set(); const imageUrls = []; if (setDetails.set_img_url) imageUrls.push(setDetails.set_img_url); if (setDetails.set_img_url_2) imageUrls.push(setDetails.set_img_url_2); for (const url of imageUrls) { const urlObj = new URL(url); const base = path.basename(urlObj.pathname); const dest = path.join(imagesDir, base); if (fsSync.existsSync(dest)) { console.log(`[skip] image exists: ${path.relative(ROOT, dest)}`); } else { await downloadFile(url, dest); downloadedImages.add(base); console.log(`[download] ${url} -> ${path.relative(ROOT, dest)}`); } const nameNoExt = base.replace(/\.[^.]+$/, ''); const yamlPath = path.join(dataImagesDir, `${nameNoExt}.yaml`); const titleForSetImage = (setDetails.name || selected.name || '').replace(/"/g, '\\"'); const yaml = `attribution: "© Rebrickable"\n` + `title: "${titleForSetImage}"\n`; await fs.writeFile(yamlPath, yaml, 'utf8'); } // Minifigs images (stored directly in images/) const figsResp = await cachedApiJson( `sets/${setNum}_minifigs.json`, host, `/api/v3/lego/sets/${encodeURIComponent(setNum)}/minifigs/?page_size=1000`, apiKey, `minifigs:${setNum}`, { noCache } ); const figs = figsResp.results || []; if (figs.length > 0) { for (const fig of figs) { const figNum = fig.fig_num || fig.set_num; let figImg = fig.set_img_url || fig.fig_img_url; let figTitle = fig.name || ''; if (!figImg && figNum) { const figDetails = await cachedApiJson( `minifigs/${figNum}.json`, host, `/api/v3/lego/minifigs/${encodeURIComponent(figNum)}/`, apiKey, `minifig:${figNum}`, { noCache } ); figImg = figDetails.set_img_url || figDetails.fig_img_url; figTitle = figTitle || figDetails.name || ''; } if (!figImg) continue; const urlObj = new URL(figImg); const base = path.basename(urlObj.pathname); const dest = path.join(imagesDir, base); if (fsSync.existsSync(dest)) { console.log(`[skip] image exists: ${path.relative(ROOT, dest)}`); } else { await downloadFile(figImg, dest); downloadedImages.add(base); console.log(`[download] ${figImg} -> ${path.relative(ROOT, dest)}`); } const nameNoExt = base.replace(/\.[^.]+$/, ''); const yamlPath = path.join(dataImagesDir, `${nameNoExt}.yaml`); const yaml = `attribution: "© Rebrickable"\n` + `title: "${(figTitle || '').replace(/"/g, '\\"')}"\n`; await fs.writeFile(yamlPath, yaml, 'utf8'); } } // index.md (do not overwrite if exists) const indexPath = path.join(setDir, 'index.md'); const indexExists = fsSync.existsSync(indexPath); if (!indexExists) { const pageTitle = setDetails.name || selected.name; const today = new Date(); const date = today.toISOString().slice(0, 10); // Collect images present in imagesDir (existing or just downloaded) let imageFiles = []; try { const allFiles = await fs.readdir(imagesDir); imageFiles = allFiles.filter((f) => /\.(png|jpe?g|gif|webp)$/i.test(f)); } catch (_) { imageFiles = Array.from(downloadedImages); } let body = ''; if (imageFiles.length > 0) { const ordered = [ ...Array.from(downloadedImages).filter((b) => imageFiles.includes(b)), ...imageFiles.filter((b) => !downloadedImages.has(b)), ]; const lines = []; for (const base of ordered) { const nameNoExt = base.replace(/\.[^.]+$/, ''); const yamlPath = path.join(dataImagesDir, `${nameNoExt}.yaml`); const altRaw = (await readYamlTitle(yamlPath)) || `${pageTitle}`; const altEsc = escapeMarkdownAlt(altRaw); const titleAttr = altRaw.replace(/\"/g, '\\"'); lines.push(`\n![${altEsc}](images/${base} \"${titleAttr}\")`); } body = lines.join('\n') + '\n'; } const md = `---\n` + `title: "${pageTitle.replace(/"/g, '\\"')}"\n` + `date: ${date}\n` + `---\n\n` + body; await fs.writeFile(indexPath, md, 'utf8'); console.log(`[create] ${path.relative(ROOT, indexPath)} with ${imageFiles.length} image reference(s)`); } // Report downloaded images that are not referenced in an existing index.md if (indexExists && downloadedImages.size > 0) { try { const mdContent = await fs.readFile(indexPath, 'utf8'); const refSet = new Set(); const imgRegex = /!\[[^\]]*\]\(([^)]+)\)/g; // capture URL inside () let m; while ((m = imgRegex.exec(mdContent)) !== null) { const p = m[1].trim(); const base = path.basename(p.split('?')[0].split('#')[0]); if (base) refSet.add(base); } const notRef = Array.from(downloadedImages).filter((b) => !refSet.has(b)); if (notRef.length > 0) { console.log(`\n[note] ${notRef.length} downloaded image(s) not referenced in index.md:`); notRef.forEach((b) => console.log(` - images/${b}`)); } } catch (e) { console.log(`[warn] could not analyze index.md references: ${e.message}`); } } console.log(`\nAdded set ${setNum} → ${path.relative(ROOT, setDir)}`); } main();