Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 | 16x 8x 8x 3x 2x 12x 12x 11x 10x 9x 1x 8x 6x 8x 8x 8x 6x 8x 8x 8x 6x 6x 6x 6x 5x 4x 4x 4x 3x 3x 6x 6x 6x 6x 6x 5x 5x 3x 3x 3x 3x 5x 5x 3x 3x 1x 1x 2x 2x 2x 2x 2x 2x 2x 7x 7x 1x 7x 7x 7x 7x | import dns from "node:dns/promises";
import { isIP } from "node:net";
import type { FullCommitSha, RepositorySlug } from "@shared/examples";
import { config } from "../../config";
import { ExamplesLoadController } from "./examples-load-controller";
import { CourseContentLoader, type LoadedCourseContentSnapshot } from "../course-content/course-content-loader";
import { githubApiBudget } from "./github-api-budget";
export type LoadedRevisionSnapshot = LoadedCourseContentSnapshot;
export interface TextFetcher {
fetchText(url: URL, maxBytes: number, signal?: AbortSignal): Promise<string>;
}
export class SecureExamplesFetcher implements TextFetcher {
constructor(private readonly controller?: ExamplesLoadController) {}
fetchText(url: URL, maxBytes: number, signal?: AbortSignal): Promise<string> {
const operation = () => fetchText(url, maxBytes, signal);
return this.controller ? this.controller.runOutbound(operation, signal) : operation();
}
}
export class RevisionProvider {
private readonly loader: CourseContentLoader;
constructor(
fetcher: TextFetcher,
maxFileFetchConcurrency = config.examples.maxFileFetchConcurrency,
) {
this.loader = new CourseContentLoader(fetcher, maxFileFetchConcurrency);
}
async load(
repository: RepositorySlug,
revision: FullCommitSha,
signal?: AbortSignal,
): Promise<LoadedRevisionSnapshot> {
return this.loader.load(repository, revision, signal);
}
}
export function validateSourceUrl(value: string): URL {
const url = new URL(value);
if (url.username || url.password || url.search || url.hash) throw new Error("Invalid examples source URL");
if (url.protocol !== "https:") throw new Error("External examples source must use HTTPS");
if (isIP(url.hostname) !== 0) throw new Error("IP literal examples sources are not allowed");
if (!config.examples.allowedHosts.includes(url.hostname.toLowerCase())) {
throw new Error("External examples source host is not allowlisted");
}
return url;
}
const GITHUB_API_HOST = "api.github.com";
async function fetchText(url: URL, maxBytes: number, requestSignal?: AbortSignal): Promise<string> {
requestSignal?.throwIfAborted();
const validated = validateSourceUrl(url.toString());
await assertPublicHost(validated.hostname);
requestSignal?.throwIfAborted();
const controller = new AbortController();
const abort = () => controller.abort(requestSignal?.reason);
Iif (requestSignal?.aborted) abort();
else requestSignal?.addEventListener("abort", abort, { once: true });
const timeout = setTimeout(() => controller.abort(), config.examples.timeoutMs);
try {
const response = await requestSource(validated, controller.signal);
if (response.status >= 300 && response.status < 400) throw new Error("Redirects are not allowed for examples");
Iif (!response.ok) throw new Error(`Examples source returned ${response.status}`);
const contentLength = response.headers.get("content-length");
if (contentLength && Number(contentLength) > maxBytes) throw new Error("Examples response exceeds size limit");
Iif (!response.body) throw new Error("Examples response has no body");
return await readLimitedText(response.body, maxBytes);
} finally {
clearTimeout(timeout);
requestSignal?.removeEventListener("abort", abort);
}
}
async function requestSource(url: URL, signal: AbortSignal): Promise<Response> {
const isGitHubApi = url.hostname.toLowerCase() === GITHUB_API_HOST;
// The optional token is sent to the GitHub REST API only, never to raw content hosts.
const token = isGitHubApi ? config.examples.githubToken : undefined;
const response = await fetch(url, {
signal,
redirect: "manual",
...(token ? { headers: { authorization: `Bearer ${token}` } } : {}),
});
if (isGitHubApi) githubApiBudget.observe(response.status, response.headers);
return response;
}
async function readLimitedText(body: ReadableStream<Uint8Array>, maxBytes: number): Promise<string> {
const reader = body.getReader();
const chunks: Uint8Array[] = [];
let total = 0;
while (true) {
const next = await reader.read();
if (next.done) break;
total += next.value.byteLength;
if (total > maxBytes) {
await reader.cancel();
throw new Error("Examples response exceeds size limit");
}
chunks.push(next.value);
}
const bytes = new Uint8Array(total);
let offset = 0;
for (const chunk of chunks) {
bytes.set(chunk, offset);
offset += chunk.byteLength;
}
return new TextDecoder("utf-8", { fatal: true }).decode(bytes);
}
async function assertPublicHost(hostname: string): Promise<void> {
const addresses = await dns.lookup(hostname, { all: true });
if (addresses.length === 0 || addresses.some(({ address }) => isPrivateAddress(address))) {
throw new Error("Examples source resolves to a private or reserved address");
}
}
function isPrivateAddress(address: string): boolean {
Eif (isIP(address) === 4) {
const octets = address.split(".").map(Number);
const [a, b] = octets;
return a === 10 || a === 127 || a === 0 || (a === 169 && b === 254) || (a === 172 && b >= 16 && b <= 31) || (a === 192 && b === 168) || (a === 100 && b >= 64 && b <= 127);
}
const normalized = address.toLowerCase();
if (normalized.startsWith("::ffff:")) return isPrivateAddress(normalized.slice("::ffff:".length));
return normalized === "::" || normalized === "::1" || normalized.startsWith("fc") || normalized.startsWith("fd") || normalized.startsWith("fe8") || normalized.startsWith("fe9") || normalized.startsWith("fea") || normalized.startsWith("feb");
}
|