455 lines
15 KiB
JavaScript
455 lines
15 KiB
JavaScript
#!/usr/bin/env node
|
|
/*
|
|
Adds a LEGO set to the collection using Rebrickable API.
|
|
Usage: node tools/add_lego.js <query>
|
|
|
|
Flow:
|
|
1) Search sets with the provided query
|
|
2) Interactive choice if multiple results
|
|
3) Fetch chosen set details and its theme
|
|
4) Create content/collections/lego/<theme-slug>/<set-id>/
|
|
- Always create images/
|
|
- Create data/images/
|
|
- Download all available set images from Rebrickable
|
|
- Create data/images/<image-name>.yaml with attribution
|
|
- Create index.md with title and date (only if not existing),
|
|
including markdown references to images found
|
|
*/
|
|
|
|
const fs = require('fs/promises');
|
|
const fsSync = require('fs');
|
|
const path = require('path');
|
|
const https = require('https');
|
|
const readline = require('readline');
|
|
const { loadToolsConfig } = require('./lib/config');
|
|
|
|
const ROOT = process.cwd();
|
|
const LEGO_ROOT = path.join(ROOT, 'content', 'collections', 'lego');
|
|
const CACHE_ROOT = path.join(ROOT, 'tools', 'cache', 'rebrickable');
|
|
const ONE_WEEK_MS = 7 * 24 * 60 * 60 * 1000;
|
|
|
|
function slugify(input) {
|
|
return input
|
|
.normalize('NFD')
|
|
.replace(/\p{Diacritic}/gu, '')
|
|
.toLowerCase()
|
|
.replace(/[^a-z0-9]+/g, '-')
|
|
.replace(/^-+|-+$/g, '')
|
|
.replace(/-{2,}/g, '-');
|
|
}
|
|
|
|
function prompt(question) {
|
|
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
|
|
return new Promise((resolve) => rl.question(question, (ans) => { rl.close(); resolve(ans.trim()); }));
|
|
}
|
|
|
|
function apiGetJson(host, path, apiKey) {
|
|
return new Promise((resolve) => {
|
|
const req = https.request({
|
|
host,
|
|
path,
|
|
method: 'GET',
|
|
headers: {
|
|
'Authorization': `key ${apiKey}`,
|
|
'Accept': 'application/json',
|
|
'User-Agent': 'add_lego_script/1.0',
|
|
},
|
|
}, (res) => {
|
|
let data = '';
|
|
res.on('data', (chunk) => { data += chunk; });
|
|
res.on('end', () => { resolve(JSON.parse(data)); });
|
|
});
|
|
req.end();
|
|
});
|
|
}
|
|
|
|
function downloadFile(url, destPath) {
|
|
return new Promise((resolve) => {
|
|
const file = fsSync.createWriteStream(destPath);
|
|
https.get(url, (res) => {
|
|
res.pipe(file);
|
|
file.on('finish', () => file.close(() => resolve()));
|
|
});
|
|
});
|
|
}
|
|
|
|
async function ensureDir(dir) {
|
|
await fs.mkdir(dir, { recursive: true });
|
|
}
|
|
|
|
async function saveJson(relPath, data) {
|
|
const out = path.join(CACHE_ROOT, relPath);
|
|
await ensureDir(path.dirname(out));
|
|
await fs.writeFile(out, JSON.stringify(data, null, 2), 'utf8');
|
|
}
|
|
|
|
async function cachedApiJson(relPath, host, apiPath, apiKey, label, options = {}) {
|
|
const full = path.join(CACHE_ROOT, relPath);
|
|
if (options.noCache) {
|
|
const json = await apiGetJson(host, apiPath, apiKey);
|
|
await ensureDir(path.dirname(full));
|
|
await fs.writeFile(full, JSON.stringify(json, null, 2), 'utf8');
|
|
if (label) console.log(`[fetch] ${label} <- ${apiPath} [nocache]`);
|
|
return json;
|
|
}
|
|
try {
|
|
const st = await fs.stat(full);
|
|
const ageMs = Date.now() - st.mtimeMs;
|
|
if (ageMs < ONE_WEEK_MS) {
|
|
try {
|
|
const raw = await fs.readFile(full, 'utf8');
|
|
const json = JSON.parse(raw);
|
|
if (label) console.log(`[cache] ${label}`);
|
|
return json;
|
|
} catch (_) {
|
|
// Corrupt cache, fall through to refetch
|
|
if (label) console.log(`[stale] ${label} (cache unreadable) -> refetch`);
|
|
}
|
|
} else {
|
|
if (label) console.log(`[stale] ${label} (age ${(ageMs / ONE_WEEK_MS).toFixed(2)} weeks) -> refetch`);
|
|
}
|
|
} catch (_) {
|
|
// No cache, fetch
|
|
}
|
|
|
|
const json = await apiGetJson(host, apiPath, apiKey);
|
|
await ensureDir(path.dirname(full));
|
|
await fs.writeFile(full, JSON.stringify(json, null, 2), 'utf8');
|
|
if (label) console.log(`[fetch] ${label} <- ${apiPath}`);
|
|
return json;
|
|
}
|
|
|
|
function escapeMarkdownAlt(text) {
|
|
return String(text || '')
|
|
.replace(/\r?\n/g, ' ')
|
|
.replace(/\\/g, '\\\\')
|
|
.replace(/\[/g, '\\[')
|
|
.replace(/\]/g, '\\]');
|
|
}
|
|
|
|
// Lightweight dimensions readers for PNG/JPEG/GIF. Fallback to file size.
|
|
async function getImageDimensions(filePath) {
|
|
const fd = await fs.open(filePath, 'r');
|
|
try {
|
|
const stat = await fd.stat();
|
|
const size = stat.size || 0;
|
|
const hdr = Buffer.alloc(64);
|
|
const { bytesRead } = await fd.read(hdr, 0, hdr.length, 0);
|
|
const b = hdr.slice(0, bytesRead);
|
|
// PNG
|
|
if (b.length >= 24 && b[0] === 0x89 && b[1] === 0x50 && b[2] === 0x4E && b[3] === 0x47) {
|
|
const width = b.readUInt32BE(16);
|
|
const height = b.readUInt32BE(20);
|
|
return { width, height, area: width * height, size };
|
|
}
|
|
// GIF
|
|
if (b.length >= 10 && b[0] === 0x47 && b[1] === 0x49 && b[2] === 0x46) {
|
|
const width = b.readUInt16LE(6);
|
|
const height = b.readUInt16LE(8);
|
|
return { width, height, area: width * height, size };
|
|
}
|
|
// JPEG: scan segments for SOF markers
|
|
if (b.length >= 4 && b[0] === 0xFF && b[1] === 0xD8) {
|
|
let pos = 2;
|
|
while (pos + 3 < size) {
|
|
const markerBuf = Buffer.alloc(4);
|
|
await fd.read(markerBuf, 0, 4, pos);
|
|
if (markerBuf[0] !== 0xFF) { pos += 1; continue; }
|
|
const marker = markerBuf[1];
|
|
const hasLen = marker !== 0xD8 && marker !== 0xD9;
|
|
let segLen = 0;
|
|
if (hasLen) segLen = markerBuf.readUInt16BE(2);
|
|
// SOF0..SOF15 except DHT(0xC4), JPG(0xC8), DAC(0xCC)
|
|
if (marker >= 0xC0 && marker <= 0xCF && ![0xC4, 0xC8, 0xCC].includes(marker)) {
|
|
const segHdr = Buffer.alloc(7);
|
|
await fd.read(segHdr, 0, segHdr.length, pos + 4);
|
|
const height = segHdr.readUInt16BE(1);
|
|
const width = segHdr.readUInt16BE(3);
|
|
return { width, height, area: width * height, size };
|
|
}
|
|
if (!hasLen || segLen < 2) { pos += 2; continue; }
|
|
pos += 2 + segLen;
|
|
}
|
|
}
|
|
return { width: 0, height: 0, area: 0, size };
|
|
} catch (_) {
|
|
return { width: 0, height: 0, area: 0, size: 0 };
|
|
} finally {
|
|
await fd.close();
|
|
}
|
|
}
|
|
|
|
async function readYamlTitle(yamlPath) {
|
|
try {
|
|
const content = await fs.readFile(yamlPath, 'utf8');
|
|
// Match title: "..." or title: '...' or title: text
|
|
const m = content.match(/^\s*title\s*:\s*(?:\"([^\"]*)\"|'([^']*)'|([^\n\r#]+))/m);
|
|
if (!m) return null;
|
|
const val = (m[1] || m[2] || m[3] || '').trim();
|
|
return val;
|
|
} catch (_) {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async function main() {
|
|
// Args parsing: support --no-cache and a free-form query
|
|
const argv = process.argv.slice(2);
|
|
let noCache = false;
|
|
const parts = [];
|
|
for (const a of argv) {
|
|
if (a === '--no-cache' || a === '--nocache' || a === '--fresh') {
|
|
noCache = true;
|
|
} else if (a === '--') {
|
|
// everything after is query literal
|
|
const idx = argv.indexOf('--');
|
|
parts.push(argv.slice(idx + 1).join(' '));
|
|
break;
|
|
} else if (a.startsWith('--')) {
|
|
// ignore unknown flags for now
|
|
} else {
|
|
parts.push(a);
|
|
}
|
|
}
|
|
const query = parts.join(' ').trim();
|
|
if (!query) {
|
|
console.log('Usage: node tools/add_lego.js [--no-cache] <query|set-id>');
|
|
process.exit(1);
|
|
}
|
|
|
|
const config = await loadToolsConfig();
|
|
const apiKey = config.rebrickable.apiKey;
|
|
const host = 'rebrickable.com';
|
|
|
|
// 1) Search (cached ≤ 1 week)
|
|
const searchPath = `/api/v3/lego/sets/?search=${encodeURIComponent(query)}&page_size=25`;
|
|
const search = await cachedApiJson(
|
|
`search/${encodeURIComponent(query)}.json`,
|
|
host,
|
|
searchPath,
|
|
apiKey,
|
|
`search:${query}`,
|
|
{ noCache }
|
|
);
|
|
const results = search.results || [];
|
|
|
|
if (results.length === 0) {
|
|
console.log('No sets found for query:', query);
|
|
process.exit(0);
|
|
}
|
|
|
|
// 2) Selection
|
|
let selected = results[0];
|
|
if (results.length > 1) {
|
|
console.log(`Found ${results.length} sets:`);
|
|
results.forEach((r, i) => {
|
|
const id = (r.set_num || '').split('-')[0];
|
|
console.log(`${i + 1}. ${r.set_num} — ${r.name} (${r.year}) [id: ${id}]`);
|
|
});
|
|
const ans = await prompt('Choose a set (1..N): ');
|
|
const idx = Math.max(1, Math.min(results.length, parseInt(ans, 10) || 1)) - 1;
|
|
selected = results[idx];
|
|
}
|
|
|
|
const setNum = selected.set_num; // e.g., 10283-1
|
|
const setId = (setNum || '').split('-')[0];
|
|
|
|
// 3) Fetch exact set and theme (cached ≤ 1 week)
|
|
const setDetails = await cachedApiJson(
|
|
`sets/${setNum}.json`,
|
|
host,
|
|
`/api/v3/lego/sets/${encodeURIComponent(setNum)}/`,
|
|
apiKey,
|
|
`set:${setNum}`,
|
|
{ noCache }
|
|
);
|
|
const themeId = setDetails.theme_id || selected.theme_id;
|
|
const theme = await cachedApiJson(
|
|
`themes/${themeId}.json`,
|
|
host,
|
|
`/api/v3/lego/themes/${themeId}/`,
|
|
apiKey,
|
|
`theme:${themeId}`,
|
|
{ noCache }
|
|
);
|
|
const themeName = theme.name;
|
|
const themeSlug = slugify(themeName);
|
|
|
|
// 4) Prepare dirs
|
|
const setDir = path.join(LEGO_ROOT, themeSlug, setId);
|
|
const imagesDir = path.join(setDir, 'images');
|
|
const dataImagesDir = path.join(setDir, 'data', 'images');
|
|
|
|
await ensureDir(imagesDir); // created even if no images provided
|
|
await ensureDir(dataImagesDir);
|
|
|
|
// Images provided by Rebrickable (set images)
|
|
const downloadedImages = new Set();
|
|
const imageUrls = [];
|
|
if (setDetails.set_img_url) imageUrls.push(setDetails.set_img_url);
|
|
if (setDetails.set_img_url_2) imageUrls.push(setDetails.set_img_url_2);
|
|
|
|
for (const url of imageUrls) {
|
|
const urlObj = new URL(url);
|
|
const base = path.basename(urlObj.pathname);
|
|
const dest = path.join(imagesDir, base);
|
|
if (fsSync.existsSync(dest)) {
|
|
console.log(`[skip] image exists: ${path.relative(ROOT, dest)}`);
|
|
} else {
|
|
await downloadFile(url, dest);
|
|
downloadedImages.add(base);
|
|
console.log(`[download] ${url} -> ${path.relative(ROOT, dest)}`);
|
|
}
|
|
|
|
const nameNoExt = base.replace(/\.[^.]+$/, '');
|
|
const yamlPath = path.join(dataImagesDir, `${nameNoExt}.yaml`);
|
|
const titleForSetImage = (setDetails.name || selected.name || '').replace(/"/g, '\\"');
|
|
const yaml = `attribution: "© [Rebrickable](https://rebrickable.com/)"\n` +
|
|
`title: "${titleForSetImage}"\n`;
|
|
await fs.writeFile(yamlPath, yaml, 'utf8');
|
|
}
|
|
|
|
// Minifigs images (stored directly in images/)
|
|
const figsResp = await cachedApiJson(
|
|
`sets/${setNum}_minifigs.json`,
|
|
host,
|
|
`/api/v3/lego/sets/${encodeURIComponent(setNum)}/minifigs/?page_size=1000`,
|
|
apiKey,
|
|
`minifigs:${setNum}`,
|
|
{ noCache }
|
|
);
|
|
const figs = figsResp.results || [];
|
|
if (figs.length > 0) {
|
|
for (const fig of figs) {
|
|
const figNum = fig.fig_num || fig.set_num;
|
|
let figImg = fig.set_img_url || fig.fig_img_url;
|
|
let figTitle = fig.name || '';
|
|
if (!figImg && figNum) {
|
|
const figDetails = await cachedApiJson(
|
|
`minifigs/${figNum}.json`,
|
|
host,
|
|
`/api/v3/lego/minifigs/${encodeURIComponent(figNum)}/`,
|
|
apiKey,
|
|
`minifig:${figNum}`,
|
|
{ noCache }
|
|
);
|
|
figImg = figDetails.set_img_url || figDetails.fig_img_url;
|
|
figTitle = figTitle || figDetails.name || '';
|
|
}
|
|
if (!figImg) continue;
|
|
|
|
const urlObj = new URL(figImg);
|
|
const base = path.basename(urlObj.pathname);
|
|
const dest = path.join(imagesDir, base);
|
|
if (fsSync.existsSync(dest)) {
|
|
console.log(`[skip] image exists: ${path.relative(ROOT, dest)}`);
|
|
} else {
|
|
await downloadFile(figImg, dest);
|
|
downloadedImages.add(base);
|
|
console.log(`[download] ${figImg} -> ${path.relative(ROOT, dest)}`);
|
|
}
|
|
|
|
const nameNoExt = base.replace(/\.[^.]+$/, '');
|
|
const yamlPath = path.join(dataImagesDir, `${nameNoExt}.yaml`);
|
|
const cleanedTitle = (figTitle || '').trim();
|
|
const yamlLines = [`attribution: "© Rebrickable"`];
|
|
if (cleanedTitle) {
|
|
yamlLines.push(`title: "${cleanedTitle.replace(/"/g, '\\"')}"`);
|
|
}
|
|
const yaml = `${yamlLines.join('\n')}\n`;
|
|
await fs.writeFile(yamlPath, yaml, 'utf8');
|
|
}
|
|
}
|
|
|
|
// index.md (do not overwrite if exists)
|
|
const indexPath = path.join(setDir, 'index.md');
|
|
const indexExists = fsSync.existsSync(indexPath);
|
|
if (!indexExists) {
|
|
const pageTitle = setDetails.name || selected.name;
|
|
const today = new Date();
|
|
const date = today.toISOString().slice(0, 10);
|
|
|
|
// Collect images present in imagesDir (existing or just downloaded)
|
|
let imageFiles = [];
|
|
try {
|
|
const allFiles = await fs.readdir(imagesDir);
|
|
imageFiles = allFiles.filter((f) => /\.(png|jpe?g|gif|webp)$/i.test(f));
|
|
} catch (_) {
|
|
imageFiles = Array.from(downloadedImages);
|
|
}
|
|
|
|
// Choose cover: prefer downloaded this run; fallback to all images
|
|
let coverFile = null;
|
|
const coverCandidates = (downloadedImages.size > 0)
|
|
? Array.from(downloadedImages).filter((b) => imageFiles.includes(b))
|
|
: imageFiles.slice();
|
|
if (coverCandidates.length > 0) {
|
|
let best = null;
|
|
let bestScore = -1;
|
|
for (const base of coverCandidates) {
|
|
const full = path.join(imagesDir, base);
|
|
const dim = await getImageDimensions(full);
|
|
const score = (dim.area && dim.area > 0) ? dim.area : (dim.size || 0);
|
|
if (score > bestScore) { bestScore = score; best = base; }
|
|
}
|
|
coverFile = best;
|
|
}
|
|
|
|
let body = '';
|
|
if (imageFiles.length > 0) {
|
|
const ordered = [
|
|
...Array.from(downloadedImages).filter((b) => imageFiles.includes(b)),
|
|
...imageFiles.filter((b) => !downloadedImages.has(b)),
|
|
];
|
|
const lines = [];
|
|
for (const base of ordered) {
|
|
if (coverFile && base === coverFile) continue;
|
|
const nameNoExt = base.replace(/\.[^.]+$/, '');
|
|
const yamlPath = path.join(dataImagesDir, `${nameNoExt}.yaml`);
|
|
const altRaw = (await readYamlTitle(yamlPath)) || `${pageTitle}`;
|
|
const altEsc = escapeMarkdownAlt(altRaw);
|
|
const titleAttr = altRaw.replace(/\"/g, '\\"');
|
|
lines.push(`\n![${altEsc}](images/${base} \"${titleAttr}\")`);
|
|
}
|
|
body = lines.join('\n') + '\n';
|
|
}
|
|
|
|
const coverLine = coverFile ? `cover: images/${coverFile}\n` : '';
|
|
const md = `---\n` +
|
|
`title: "${pageTitle.replace(/"/g, '\\"')}"\n` +
|
|
`date: ${date}\n` +
|
|
coverLine +
|
|
`---\n\n` +
|
|
body;
|
|
await fs.writeFile(indexPath, md, 'utf8');
|
|
const coverInfo = coverFile ? `, cover=images/${coverFile}` : '';
|
|
console.log(`[create] ${path.relative(ROOT, indexPath)} with ${imageFiles.length} image reference(s)${coverInfo}`);
|
|
}
|
|
|
|
// Report downloaded images that are not referenced in an existing index.md
|
|
if (indexExists && downloadedImages.size > 0) {
|
|
try {
|
|
const mdContent = await fs.readFile(indexPath, 'utf8');
|
|
const refSet = new Set();
|
|
const imgRegex = /!\[[^\]]*\]\(([^)]+)\)/g; // capture URL inside ()
|
|
let m;
|
|
while ((m = imgRegex.exec(mdContent)) !== null) {
|
|
const p = m[1].trim();
|
|
const base = path.basename(p.split('?')[0].split('#')[0]);
|
|
if (base) refSet.add(base);
|
|
}
|
|
const notRef = Array.from(downloadedImages).filter((b) => !refSet.has(b));
|
|
if (notRef.length > 0) {
|
|
console.log(`\n[note] ${notRef.length} downloaded image(s) not referenced in index.md:`);
|
|
notRef.forEach((b) => console.log(` - images/${b}`));
|
|
}
|
|
} catch (e) {
|
|
console.log(`[warn] could not analyze index.md references: ${e.message}`);
|
|
}
|
|
}
|
|
|
|
console.log(`\nAdded set ${setNum} → ${path.relative(ROOT, setDir)}`);
|
|
}
|
|
|
|
main();
|