134 lines
3.5 KiB
JavaScript
134 lines
3.5 KiB
JavaScript
const { fetch } = require("undici");
|
|
const UserAgent = require("user-agents");
|
|
|
|
const DEFAULT_ACCEPT =
|
|
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8";
|
|
const DEFAULT_ACCEPT_LANGUAGE = "fr-FR,fr;q=0.9,en;q=0.7";
|
|
const DEFAULT_TIMEOUT_MS = 5000;
|
|
const DEFAULT_MAX_REDIRECTS = 5;
|
|
|
|
function buildUserAgent(preferred) {
|
|
if (typeof preferred === "string" && preferred.trim()) {
|
|
return preferred.trim();
|
|
}
|
|
const ua = new UserAgent();
|
|
return ua.toString();
|
|
}
|
|
|
|
async function fetchWithRedirects(targetUrl, options, maxRedirects) {
|
|
let currentUrl = targetUrl;
|
|
let response = null;
|
|
let redirects = 0;
|
|
|
|
while (redirects <= maxRedirects) {
|
|
response = await fetch(currentUrl, { ...options, redirect: "manual" });
|
|
const location = response.headers.get("location");
|
|
if (
|
|
response.status >= 300 &&
|
|
response.status < 400 &&
|
|
location &&
|
|
redirects < maxRedirects
|
|
) {
|
|
if (response.body && typeof response.body.cancel === "function") {
|
|
try {
|
|
await response.body.cancel();
|
|
} catch (_) {
|
|
// Ignore cancellation errors; we're moving to the next hop.
|
|
}
|
|
}
|
|
currentUrl = new URL(location, currentUrl).toString();
|
|
redirects += 1;
|
|
continue;
|
|
}
|
|
break;
|
|
}
|
|
|
|
return response;
|
|
}
|
|
|
|
async function probeUrl(url, options = {}) {
|
|
const method = typeof options.method === "string" ? options.method.toUpperCase() : "HEAD";
|
|
const timeoutMs = Number.isFinite(options.timeoutMs) ? options.timeoutMs : DEFAULT_TIMEOUT_MS;
|
|
const maxRedirects = Number.isFinite(options.maxRedirects)
|
|
? options.maxRedirects
|
|
: DEFAULT_MAX_REDIRECTS;
|
|
const userAgent = buildUserAgent(options.userAgent);
|
|
const headers = {
|
|
"user-agent": userAgent,
|
|
"accept-language": DEFAULT_ACCEPT_LANGUAGE,
|
|
"accept": DEFAULT_ACCEPT,
|
|
...(options.headers || {}),
|
|
};
|
|
|
|
const controller = new AbortController();
|
|
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
|
|
try {
|
|
const response = await fetchWithRedirects(
|
|
url,
|
|
{
|
|
method,
|
|
headers,
|
|
signal: controller.signal,
|
|
},
|
|
maxRedirects
|
|
);
|
|
const status = response ? response.status : null;
|
|
const finalUrl = response?.url || url;
|
|
if (response?.body && typeof response.body.cancel === "function") {
|
|
try {
|
|
await response.body.cancel();
|
|
} catch (_) {
|
|
// Ignore cancellation errors; the status is all we needed.
|
|
}
|
|
}
|
|
return {
|
|
status,
|
|
finalUrl,
|
|
method,
|
|
errorType: null,
|
|
};
|
|
} catch (error) {
|
|
if (error.name === "AbortError") {
|
|
return {
|
|
status: null,
|
|
finalUrl: url,
|
|
method,
|
|
errorType: "timeout",
|
|
};
|
|
}
|
|
return {
|
|
status: null,
|
|
finalUrl: url,
|
|
method,
|
|
errorType: "network",
|
|
message: error.message,
|
|
};
|
|
} finally {
|
|
clearTimeout(timer);
|
|
}
|
|
}
|
|
|
|
function shouldRetry(result) {
|
|
if (!result) return true;
|
|
if (result.errorType) return true;
|
|
if (typeof result.status !== "number") return true;
|
|
return result.status >= 400;
|
|
}
|
|
|
|
async function checkUrl(url, options = {}) {
|
|
const firstMethod = options.firstMethod || "HEAD";
|
|
let result = await probeUrl(url, { ...options, method: firstMethod });
|
|
if (options.retryWithGet !== false && shouldRetry(result)) {
|
|
result = await probeUrl(url, { ...options, method: "GET" });
|
|
}
|
|
return result;
|
|
}
|
|
|
|
module.exports = {
|
|
buildUserAgent,
|
|
checkUrl,
|
|
probeUrl,
|
|
shouldRetry,
|
|
};
|