import { HTTP_TIMEOUT, MAX_BYTES } from "../constants.js"; import { browserPool } from "../utils/browser-pool.js"; import { assertSafeUrl } from "./security.js"; import { fetchViaVettedAddress } from "./http.js"; const MAX_BROWSER_REQUESTS = 100; const MAX_BROWSER_TOTAL_BYTES = Math.min(MAX_BYTES, 20 * 1024 * 1024); function responseHeaders(headers) { const record = {}; headers.forEach((value, key) => { record[key] = value; }); return record; } export async function fetchBrowserResource(url, timeoutMs = HTTP_TIMEOUT) { await assertSafeUrl(url); return browserPool.withBrowser(async browser => { const page = await browser.newPage(); const devtools = await page.target().createCDPSession(); await devtools.send("Network.enable"); await devtools.send("Network.setBlockedURLs", { urls: ["ws://*", "wss://*", "file://*", "ftp://*"] }); await page.evaluateOnNewDocument(() => { const blockedTransport = name => class { constructor() { throw new DOMException(`${name} is disabled by the safe browser fetcher`, "SecurityError"); } }; for (const name of ["WebSocket", "WebTransport", "RTCPeerConnection", "webkitRTCPeerConnection"]) { if (name in globalThis) { Object.defineProperty(globalThis, name, { configurable: false, writable: false, value: blockedTransport(name) }); } } }); await page.setBypassServiceWorker(true); let requests = 0; let totalBytes = 0; let blockedError; await page.setRequestInterception(true); page.on("request", request => { void (async () => { try { const requestUrl = new URL(request.url()); if (!["http:", "https:"].includes(requestUrl.protocol)) throw new Error("unsupported browser request scheme"); if (request.method() !== "GET") throw new Error("browser fetch blocks non-GET requests"); requests += 1; if (requests > MAX_BROWSER_REQUESTS) throw new Error("browser request limit exceeded"); await assertSafeUrl(requestUrl); const upstream = await fetchViaVettedAddress(requestUrl, timeoutMs); const body = Buffer.from(await upstream.arrayBuffer()); totalBytes += body.byteLength; if (totalBytes > MAX_BROWSER_TOTAL_BYTES) throw new Error("browser byte limit exceeded"); await request.respond({ status: upstream.status, headers: responseHeaders(upstream.headers), body }); } catch (error) { blockedError ||= error; await request.abort("blockedbyclient").catch(() => undefined); } })(); }); try { const navigation = await page.goto(url.toString(), { waitUntil: "networkidle2", timeout: timeoutMs }); if (blockedError && !navigation) throw blockedError; const finalUrl = new URL(page.url()); await assertSafeUrl(finalUrl); const html = await page.content(); const buffer = Buffer.from(html); if (buffer.byteLength > MAX_BYTES) throw new Error("rendered content too large"); const headers = new Headers({ "content-type": "text/html; charset=utf-8" }); const status = navigation?.status() || 200; const response = new Response(new Uint8Array(buffer), { status, headers }); Object.defineProperty(response, "url", { value: finalUrl.toString() }); return { response, finalUrl: finalUrl.toString(), contentType: headers.get("content-type"), buffer, byteLength: buffer.byteLength }; } finally { await page.close(); } }); }