#!/usr/bin/env node /* Adds a LEGO set to the collection using Rebrickable API. Usage: node tools/add_lego.js Flow: 1) Search sets with the provided query 2) Interactive choice if multiple results 3) Fetch chosen set details and its theme 4) Create content/collections/lego/// - Always create images/ - Create data/images/ - Download all available set images from Rebrickable - Create data/images/.yaml with attribution - Create index.md with title and date (only if not existing), including markdown references to images found */ const fs = require('fs/promises'); const fsSync = require('fs'); const path = require('path'); const https = require('https'); const readline = require('readline'); const { loadToolsConfig } = require('./lib/config'); const ROOT = process.cwd(); const LEGO_ROOT = path.join(ROOT, 'content', 'collections', 'lego'); const CACHE_ROOT = path.join(ROOT, 'tools', 'cache', 'rebrickable'); const ONE_WEEK_MS = 7 * 24 * 60 * 60 * 1000; function slugify(input) { return input .normalize('NFD') .replace(/\p{Diacritic}/gu, '') .toLowerCase() .replace(/[^a-z0-9]+/g, '-') .replace(/^-+|-+$/g, '') .replace(/-{2,}/g, '-'); } function prompt(question) { const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); return new Promise((resolve) => rl.question(question, (ans) => { rl.close(); resolve(ans.trim()); })); } function apiGetJson(host, path, apiKey) { return new Promise((resolve) => { const req = https.request({ host, path, method: 'GET', headers: { 'Authorization': `key ${apiKey}`, 'Accept': 'application/json', 'User-Agent': 'add_lego_script/1.0', }, }, (res) => { let data = ''; res.on('data', (chunk) => { data += chunk; }); res.on('end', () => { resolve(JSON.parse(data)); }); }); req.end(); }); } function downloadFile(url, destPath) { return new Promise((resolve) => { const file = fsSync.createWriteStream(destPath); https.get(url, (res) => { res.pipe(file); file.on('finish', () => file.close(() => resolve())); }); }); } async function ensureDir(dir) { await fs.mkdir(dir, { recursive: true }); } async function saveJson(relPath, data) { const out = path.join(CACHE_ROOT, relPath); await ensureDir(path.dirname(out)); await fs.writeFile(out, JSON.stringify(data, null, 2), 'utf8'); } async function cachedApiJson(relPath, host, apiPath, apiKey, label, options = {}) { const full = path.join(CACHE_ROOT, relPath); if (options.noCache) { const json = await apiGetJson(host, apiPath, apiKey); await ensureDir(path.dirname(full)); await fs.writeFile(full, JSON.stringify(json, null, 2), 'utf8'); if (label) console.log(`[fetch] ${label} <- ${apiPath} [nocache]`); return json; } try { const st = await fs.stat(full); const ageMs = Date.now() - st.mtimeMs; if (ageMs < ONE_WEEK_MS) { try { const raw = await fs.readFile(full, 'utf8'); const json = JSON.parse(raw); if (label) console.log(`[cache] ${label}`); return json; } catch (_) { // Corrupt cache, fall through to refetch if (label) console.log(`[stale] ${label} (cache unreadable) -> refetch`); } } else { if (label) console.log(`[stale] ${label} (age ${(ageMs / ONE_WEEK_MS).toFixed(2)} weeks) -> refetch`); } } catch (_) { // No cache, fetch } const json = await apiGetJson(host, apiPath, apiKey); await ensureDir(path.dirname(full)); await fs.writeFile(full, JSON.stringify(json, null, 2), 'utf8'); if (label) console.log(`[fetch] ${label} <- ${apiPath}`); return json; } function escapeMarkdownAlt(text) { return String(text || '') .replace(/\r?\n/g, ' ') .replace(/\\/g, '\\\\') .replace(/\[/g, '\\[') .replace(/\]/g, '\\]'); } // Lightweight dimensions readers for PNG/JPEG/GIF. Fallback to file size. async function getImageDimensions(filePath) { const fd = await fs.open(filePath, 'r'); try { const stat = await fd.stat(); const size = stat.size || 0; const hdr = Buffer.alloc(64); const { bytesRead } = await fd.read(hdr, 0, hdr.length, 0); const b = hdr.slice(0, bytesRead); // PNG if (b.length >= 24 && b[0] === 0x89 && b[1] === 0x50 && b[2] === 0x4E && b[3] === 0x47) { const width = b.readUInt32BE(16); const height = b.readUInt32BE(20); return { width, height, area: width * height, size }; } // GIF if (b.length >= 10 && b[0] === 0x47 && b[1] === 0x49 && b[2] === 0x46) { const width = b.readUInt16LE(6); const height = b.readUInt16LE(8); return { width, height, area: width * height, size }; } // JPEG: scan segments for SOF markers if (b.length >= 4 && b[0] === 0xFF && b[1] === 0xD8) { let pos = 2; while (pos + 3 < size) { const markerBuf = Buffer.alloc(4); await fd.read(markerBuf, 0, 4, pos); if (markerBuf[0] !== 0xFF) { pos += 1; continue; } const marker = markerBuf[1]; const hasLen = marker !== 0xD8 && marker !== 0xD9; let segLen = 0; if (hasLen) segLen = markerBuf.readUInt16BE(2); // SOF0..SOF15 except DHT(0xC4), JPG(0xC8), DAC(0xCC) if (marker >= 0xC0 && marker <= 0xCF && ![0xC4, 0xC8, 0xCC].includes(marker)) { const segHdr = Buffer.alloc(7); await fd.read(segHdr, 0, segHdr.length, pos + 4); const height = segHdr.readUInt16BE(1); const width = segHdr.readUInt16BE(3); return { width, height, area: width * height, size }; } if (!hasLen || segLen < 2) { pos += 2; continue; } pos += 2 + segLen; } } return { width: 0, height: 0, area: 0, size }; } catch (_) { return { width: 0, height: 0, area: 0, size: 0 }; } finally { await fd.close(); } } async function readYamlTitle(yamlPath) { try { const content = await fs.readFile(yamlPath, 'utf8'); // Match title: "..." or title: '...' or title: text const m = content.match(/^\s*title\s*:\s*(?:\"([^\"]*)\"|'([^']*)'|([^\n\r#]+))/m); if (!m) return null; const val = (m[1] || m[2] || m[3] || '').trim(); return val; } catch (_) { return null; } } async function main() { // Args parsing: support --no-cache and a free-form query const argv = process.argv.slice(2); let noCache = false; const parts = []; for (const a of argv) { if (a === '--no-cache' || a === '--nocache' || a === '--fresh') { noCache = true; } else if (a === '--') { // everything after is query literal const idx = argv.indexOf('--'); parts.push(argv.slice(idx + 1).join(' ')); break; } else if (a.startsWith('--')) { // ignore unknown flags for now } else { parts.push(a); } } const query = parts.join(' ').trim(); if (!query) { console.log('Usage: node tools/add_lego.js [--no-cache] '); process.exit(1); } const config = await loadToolsConfig(); const apiKey = config.rebrickable.apiKey; const host = 'rebrickable.com'; // 1) Search (cached ≤ 1 week) const searchPath = `/api/v3/lego/sets/?search=${encodeURIComponent(query)}&page_size=25`; const search = await cachedApiJson( `search/${encodeURIComponent(query)}.json`, host, searchPath, apiKey, `search:${query}`, { noCache } ); const results = search.results || []; if (results.length === 0) { console.log('No sets found for query:', query); process.exit(0); } // 2) Selection let selected = results[0]; if (results.length > 1) { console.log(`Found ${results.length} sets:`); results.forEach((r, i) => { const id = (r.set_num || '').split('-')[0]; console.log(`${i + 1}. ${r.set_num} — ${r.name} (${r.year}) [id: ${id}]`); }); const ans = await prompt('Choose a set (1..N): '); const idx = Math.max(1, Math.min(results.length, parseInt(ans, 10) || 1)) - 1; selected = results[idx]; } const setNum = selected.set_num; // e.g., 10283-1 const setId = (setNum || '').split('-')[0]; // 3) Fetch exact set and theme (cached ≤ 1 week) const setDetails = await cachedApiJson( `sets/${setNum}.json`, host, `/api/v3/lego/sets/${encodeURIComponent(setNum)}/`, apiKey, `set:${setNum}`, { noCache } ); const themeId = setDetails.theme_id || selected.theme_id; const theme = await cachedApiJson( `themes/${themeId}.json`, host, `/api/v3/lego/themes/${themeId}/`, apiKey, `theme:${themeId}`, { noCache } ); const themeName = theme.name; const themeSlug = slugify(themeName); // 4) Prepare dirs const setDir = path.join(LEGO_ROOT, themeSlug, setId); const imagesDir = path.join(setDir, 'images'); const dataImagesDir = path.join(setDir, 'data', 'images'); await ensureDir(imagesDir); // created even if no images provided await ensureDir(dataImagesDir); // Images provided by Rebrickable (set images) const downloadedImages = new Set(); const imageUrls = []; if (setDetails.set_img_url) imageUrls.push(setDetails.set_img_url); if (setDetails.set_img_url_2) imageUrls.push(setDetails.set_img_url_2); for (const url of imageUrls) { const urlObj = new URL(url); const base = path.basename(urlObj.pathname); const dest = path.join(imagesDir, base); if (fsSync.existsSync(dest)) { console.log(`[skip] image exists: ${path.relative(ROOT, dest)}`); } else { await downloadFile(url, dest); downloadedImages.add(base); console.log(`[download] ${url} -> ${path.relative(ROOT, dest)}`); } const nameNoExt = base.replace(/\.[^.]+$/, ''); const yamlPath = path.join(dataImagesDir, `${nameNoExt}.yaml`); const titleForSetImage = (setDetails.name || selected.name || '').replace(/"/g, '\\"'); const yaml = `attribution: "© [Rebrickable](https://rebrickable.com/)"\n` + `title: "${titleForSetImage}"\n`; await fs.writeFile(yamlPath, yaml, 'utf8'); } // Minifigs images (stored directly in images/) const figsResp = await cachedApiJson( `sets/${setNum}_minifigs.json`, host, `/api/v3/lego/sets/${encodeURIComponent(setNum)}/minifigs/?page_size=1000`, apiKey, `minifigs:${setNum}`, { noCache } ); const figs = figsResp.results || []; if (figs.length > 0) { for (const fig of figs) { const figNum = fig.fig_num || fig.set_num; let figImg = fig.set_img_url || fig.fig_img_url; let figTitle = fig.name || ''; if (!figImg && figNum) { const figDetails = await cachedApiJson( `minifigs/${figNum}.json`, host, `/api/v3/lego/minifigs/${encodeURIComponent(figNum)}/`, apiKey, `minifig:${figNum}`, { noCache } ); figImg = figDetails.set_img_url || figDetails.fig_img_url; figTitle = figTitle || figDetails.name || ''; } if (!figImg) continue; const urlObj = new URL(figImg); const base = path.basename(urlObj.pathname); const dest = path.join(imagesDir, base); if (fsSync.existsSync(dest)) { console.log(`[skip] image exists: ${path.relative(ROOT, dest)}`); } else { await downloadFile(figImg, dest); downloadedImages.add(base); console.log(`[download] ${figImg} -> ${path.relative(ROOT, dest)}`); } const nameNoExt = base.replace(/\.[^.]+$/, ''); const yamlPath = path.join(dataImagesDir, `${nameNoExt}.yaml`); const cleanedTitle = (figTitle || '').trim(); const yamlLines = [`attribution: "© Rebrickable"`]; if (cleanedTitle) { yamlLines.push(`title: "${cleanedTitle.replace(/"/g, '\\"')}"`); } const yaml = `${yamlLines.join('\n')}\n`; await fs.writeFile(yamlPath, yaml, 'utf8'); } } // index.md (do not overwrite if exists) const indexPath = path.join(setDir, 'index.md'); const indexExists = fsSync.existsSync(indexPath); if (!indexExists) { const pageTitle = setDetails.name || selected.name; const today = new Date(); const date = today.toISOString().slice(0, 10); // Collect images present in imagesDir (existing or just downloaded) let imageFiles = []; try { const allFiles = await fs.readdir(imagesDir); imageFiles = allFiles.filter((f) => /\.(png|jpe?g|gif|webp)$/i.test(f)); } catch (_) { imageFiles = Array.from(downloadedImages); } // Choose cover: prefer downloaded this run; fallback to all images let coverFile = null; const coverCandidates = (downloadedImages.size > 0) ? Array.from(downloadedImages).filter((b) => imageFiles.includes(b)) : imageFiles.slice(); if (coverCandidates.length > 0) { let best = null; let bestScore = -1; for (const base of coverCandidates) { const full = path.join(imagesDir, base); const dim = await getImageDimensions(full); const score = (dim.area && dim.area > 0) ? dim.area : (dim.size || 0); if (score > bestScore) { bestScore = score; best = base; } } coverFile = best; } let body = ''; if (imageFiles.length > 0) { const ordered = [ ...Array.from(downloadedImages).filter((b) => imageFiles.includes(b)), ...imageFiles.filter((b) => !downloadedImages.has(b)), ]; const lines = []; for (const base of ordered) { if (coverFile && base === coverFile) continue; const nameNoExt = base.replace(/\.[^.]+$/, ''); const yamlPath = path.join(dataImagesDir, `${nameNoExt}.yaml`); const altRaw = (await readYamlTitle(yamlPath)) || `${pageTitle}`; const altEsc = escapeMarkdownAlt(altRaw); const titleAttr = altRaw.replace(/\"/g, '\\"'); lines.push(`\n![${altEsc}](images/${base} \"${titleAttr}\")`); } body = lines.join('\n') + '\n'; } const coverLine = coverFile ? `cover: images/${coverFile}\n` : ''; const md = `---\n` + `title: "${pageTitle.replace(/"/g, '\\"')}"\n` + `date: ${date}\n` + coverLine + `---\n\n` + body; await fs.writeFile(indexPath, md, 'utf8'); const coverInfo = coverFile ? `, cover=images/${coverFile}` : ''; console.log(`[create] ${path.relative(ROOT, indexPath)} with ${imageFiles.length} image reference(s)${coverInfo}`); } // Report downloaded images that are not referenced in an existing index.md if (indexExists && downloadedImages.size > 0) { try { const mdContent = await fs.readFile(indexPath, 'utf8'); const refSet = new Set(); const imgRegex = /!\[[^\]]*\]\(([^)]+)\)/g; // capture URL inside () let m; while ((m = imgRegex.exec(mdContent)) !== null) { const p = m[1].trim(); const base = path.basename(p.split('?')[0].split('#')[0]); if (base) refSet.add(base); } const notRef = Array.from(downloadedImages).filter((b) => !refSet.has(b)); if (notRef.length > 0) { console.log(`\n[note] ${notRef.length} downloaded image(s) not referenced in index.md:`); notRef.forEach((b) => console.log(` - images/${b}`)); } } catch (e) { console.log(`[warn] could not analyze index.md references: ${e.message}`); } } console.log(`\nAdded set ${setNum} → ${path.relative(ROOT, setDir)}`); } main();