Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 | 21x 21x 21x 19x 19x 19x 19x 19x 19x 19x 19x 30x 19x 30x 19x 19x 19x 26x 48x 48x 30x 30x 30x 3x 30x 26x 26x 19x 30x 19x 19x 2x 17x 13x 4x 18x 19x 6x 2x 19x 18x 13x 13x 10x 10x 9x 9x 8x 8x 8x 7x 1x 6x 9x 8x 8x 8x 8x 7x 8x 2x 2x 2x 2x 2x 36x 40x 9x 9x 9x 8x 8x 8x 8x 8x 8x 8x 8x 2x 3x 1x 7x 6x 6x 1x 7x 2x 2x 2x 2x 1x 18x 10x 10x 53x 53x 53x 19x 19x 19x 29387x 19x | import { createHash } from "node:crypto";
import { parse as parseYaml } from "yaml";
import { config } from "../../config";
import { ExamplesError } from "../examples/examples-error";
import { ExamplesLoadController } from "../examples/examples-load-controller";
import { mapWithConcurrency } from "../concurrency";
import type { TextFetcher } from "../examples/http-provider";
import {
courseContentTutorManifestSchema,
parseCourseContentManifest,
type CourseContentTutorManifest,
} from "./course-content-schema";
import {
extractEmbeddedTutorAnnotation,
type ExampleTutorAnnotation,
} from "./embedded-tutor-annotation";
import {
curriculumTopicSchema,
validateCurriculumTopic,
type CurriculumTopic,
} from "../tutor/curriculum/curriculum-schema";
import {
effectiveTutorStrategySchema,
type EffectiveTutorStrategy,
} from "../tutor/strategy/effective-tutor-strategy";
import type { ExampleRecord } from "../examples/examples-schema";
import type { FullCommitSha, RepositorySlug } from "@shared/examples";
export type TutorCapability =
| { readonly status: "absent" }
| { readonly status: "invalid"; readonly reason: string }
| {
readonly status: "valid";
readonly manifest: CourseContentTutorManifest;
readonly topics: readonly CurriculumTopic[];
readonly strategies: readonly EffectiveTutorStrategy[];
};
export interface LoadedCourseContentSnapshot {
readonly examples: ExampleRecord[];
readonly tutor: TutorCapability;
readonly exampleTutorAnnotations: ReadonlyMap<string, ExampleTutorAnnotation>;
readonly contentBytes: number;
}
export class CourseContentLoader {
constructor(
private readonly fetcher: TextFetcher,
private readonly maxFileFetchConcurrency = config.examples.maxFileFetchConcurrency,
private readonly loads?: ExamplesLoadController,
) {}
async load(
repository: RepositorySlug,
revision: FullCommitSha,
signal?: AbortSignal,
): Promise<LoadedCourseContentSnapshot> {
const base = new URL(`${revision}/`, `https://raw.githubusercontent.com/${repository}/`);
const manifestText = await this.fetcher.fetchText(new URL("manifest.json", base), config.examples.maxManifestBytes, signal);
let decoded: unknown;
try {
decoded = JSON.parse(manifestText) as unknown;
} catch {
throw new ExamplesError("INVALID_SNAPSHOT", "External examples manifest is invalid");
}
const validation = parseCourseContentManifest(decoded);
Iif (validation.examples.status !== "valid") {
throw new ExamplesError("INVALID_SNAPSHOT", validation.examples.reason);
}
const manifest = validation.examples.manifest;
const filesToLoad = manifest.examples.flatMap((exampleIndex, index) =>
exampleIndex.files.map((file) => ({ exampleIndex: index, file })),
);
Iif (filesToLoad.length > config.examples.maxFiles) {
throw new ExamplesError("INVALID_SNAPSHOT", "External examples exceed the file count limit");
}
const loaded = await this.map(filesToLoad, async ({ file }) => ({
...file,
content: await this.fetcher.fetchText(relativeUrl(base, file.path), config.examples.maxFileBytes, signal),
}));
const exampleTutorAnnotations = new Map<string, ExampleTutorAnnotation>();
let invalidEmbeddedAnnotation = false;
const examples = manifest.examples.map((example, exampleIndex) => {
const files = filesToLoad
.map((item, index) => ({ item, file: loaded[index] }))
.filter(({ item }) => item.exampleIndex === exampleIndex)
.map(({ file }) => {
const annotation = extractEmbeddedTutorAnnotation(file.content, file.name, { isMainFile: file.name === example.main });
if (annotation.status === "invalid") invalidEmbeddedAnnotation = true;
if (annotation.status === "valid" && file.name === example.main) {
exampleTutorAnnotations.set(example.id, annotation.annotation);
}
return annotation.status === "absent" ? file : { ...file, content: annotation.cleanedSource };
});
const tutorAnnotation = exampleTutorAnnotations.get(example.id);
return {
...example,
files,
source: "external" as const,
...(tutorAnnotation ? { tutorAnnotation } : {}),
};
});
const exampleBytes = examples.reduce(
(total, example) => total + example.files.reduce((sum, file) => sum + Buffer.byteLength(file.content, "utf8"), 0),
0,
);
Iif (exampleBytes > config.examples.maxTotalBytes) {
throw new ExamplesError("INVALID_SNAPSHOT", "External examples exceed the total size limit");
}
let tutor: TutorCapability;
if (invalidEmbeddedAnnotation) {
tutor = { status: "invalid", reason: "Tutor capability is invalid" };
} else if (validation.tutor.status === "valid") {
tutor = await this.loadTutor(base, validation.tutor.descriptor.manifest, exampleTutorAnnotations, signal);
} else {
tutor = validation.tutor;
}
const activeAnnotations = tutor.status === "invalid" ? new Map<string, ExampleTutorAnnotation>() : exampleTutorAnnotations;
const tutorBytes = tutor.status === "valid"
? tutor.topics.reduce((sum, topic) => sum + JSON.stringify(topic).length, 0)
+ tutor.strategies.reduce((sum, strategy) => sum + JSON.stringify(strategy).length, 0)
: 0;
Iif (exampleBytes + tutorBytes > config.examples.maxTotalBytes) {
throw new ExamplesError("INVALID_SNAPSHOT", "Course Content exceeds the total size limit");
}
return { examples, tutor, exampleTutorAnnotations: activeAnnotations, contentBytes: exampleBytes + tutorBytes };
}
private async loadTutor(
base: URL,
descriptorPath: string,
annotations: ReadonlyMap<string, ExampleTutorAnnotation>,
signal?: AbortSignal,
): Promise<TutorCapability> {
try {
const manifestText = await this.fetcher.fetchText(relativeUrl(base, descriptorPath), config.examples.maxManifestBytes, signal);
const manifestParsed = courseContentTutorManifestSchema.safeParse(parseSafeYaml(manifestText));
if (!manifestParsed.success) throw new Error("Tutor manifest is invalid");
const manifest = validateTutorManifest(manifestParsed.data);
const topicEntries = await this.loadTopics(base, manifest, signal);
const strategies = await this.loadStrategies(base, manifest, signal);
validateTutorReferences(manifest, topicEntries, strategies, annotations);
return { status: "valid", manifest, topics: topicEntries, strategies };
} catch (error) {
if (signal?.aborted) {
throw signal?.reason ?? error;
}
return { status: "invalid", reason: "Tutor capability is invalid" };
}
}
private loadTopics(
base: URL,
manifest: CourseContentTutorManifest,
signal?: AbortSignal,
): Promise<CurriculumTopic[]> {
return this.map(manifest.topics, async (entry) => {
const source = await this.fetcher.fetchText(relativeUrl(base, entry.path), config.examples.maxFileBytes, signal);
verifyDigest(source, entry.sha256, entry.id);
const parsed = curriculumTopicSchema.safeParse(parseSafeYaml(source));
Iif (!parsed.success || parsed.data.id !== entry.id) throw new Error(`Tutor topic is invalid: ${entry.id}`);
return validateCurriculumTopic(parsed.data);
});
}
private loadStrategies(
base: URL,
manifest: CourseContentTutorManifest,
signal?: AbortSignal,
): Promise<EffectiveTutorStrategy[]> {
return this.map(manifest.strategies, async (entry) => {
const source = await this.fetcher.fetchText(relativeUrl(base, entry.path), config.examples.maxFileBytes, signal);
verifyDigest(source, entry.sha256, entry.id);
const parsed = effectiveTutorStrategySchema.safeParse(parseSafeYaml(source));
Iif (!parsed.success || parsed.data.id !== entry.id) throw new Error(`Tutor strategy is invalid: ${entry.id}`);
return parsed.data;
});
}
private map<T, R>(values: readonly T[], mapper: (value: T) => Promise<R>): Promise<R[]> {
return mapWithConcurrency(values, this.maxFileFetchConcurrency, async (value) => {
Eif (!this.loads) return mapper(value);
return this.loads.runOutbound(() => mapper(value));
});
}
}
function validateTutorManifest(manifest: CourseContentTutorManifest): CourseContentTutorManifest {
assertUnique(manifest.topics.map(({ id }) => id), "Tutor topic");
assertUnique(manifest.strategies.map(({ id }) => id), "Tutor strategy");
return manifest;
}
function validateTutorReferences(
manifest: CourseContentTutorManifest,
topics: readonly CurriculumTopic[],
strategies: readonly EffectiveTutorStrategy[],
annotations: ReadonlyMap<string, ExampleTutorAnnotation>,
): void {
const topicIds = new Set(topics.map(({ id }) => id));
const strategyIds = new Set(strategies.map(({ id }) => id));
validateDefaultStrategy(manifest, strategyIds);
validatePhaseStrategies(manifest, strategyIds);
validateTopicExtensions(topics, topicIds);
validateAnnotationReferences(annotations, topicIds, strategyIds);
}
function validateDefaultStrategy(manifest: CourseContentTutorManifest, strategyIds: ReadonlySet<string>): void {
Iif (manifest.defaultStrategy !== undefined && !strategyIds.has(manifest.defaultStrategy)) {
throw new Error("Tutor default strategy is not enumerated");
}
}
function validatePhaseStrategies(manifest: CourseContentTutorManifest, strategyIds: ReadonlySet<string>): void {
if (manifest.schemaVersion !== 2) return;
for (const phaseStrategy of Object.values(manifest.phaseStrategies ?? {})) {
if (phaseStrategy !== undefined && !strategyIds.has(phaseStrategy)) {
throw new Error(`Tutor phase strategy is not enumerated: ${phaseStrategy}`);
}
}
}
function validateTopicExtensions(topics: readonly CurriculumTopic[], topicIds: ReadonlySet<string>): void {
for (const topic of topics) {
const extensions = topic.schemaVersion === 2 ? topic.extensions ?? [] : [];
for (const extension of extensions) {
Iif (!topicIds.has(extension.topic)) throw new Error(`Tutor extension target is not enumerated: ${extension.topic}`);
}
}
}
function validateAnnotationReferences(
annotations: ReadonlyMap<string, ExampleTutorAnnotation>,
topicIds: ReadonlySet<string>,
strategyIds: ReadonlySet<string>,
): void {
for (const annotation of annotations.values()) {
validateAnnotationTopics(annotation, topicIds);
Iif (annotation.strategy !== undefined && !strategyIds.has(annotation.strategy)) {
throw new Error(`Tutor annotation references unknown strategy: ${annotation.strategy}`);
}
}
}
function validateAnnotationTopics(annotation: ExampleTutorAnnotation, topicIds: ReadonlySet<string>): void {
for (const topicId of annotation.topics ?? []) {
if (!topicIds.has(topicId)) throw new Error(`Tutor annotation references unknown topic: ${topicId}`);
}
Iif (annotation.primaryTopic !== undefined && !topicIds.has(annotation.primaryTopic)) {
throw new Error(`Tutor annotation references unknown primary topic: ${annotation.primaryTopic}`);
}
}
function assertUnique(values: readonly string[], label: string): void {
Iif (new Set(values).size !== values.length) throw new Error(`Duplicate ${label} id`);
}
function verifyDigest(source: string, expected: string, id: string): void {
const actual = createHash("sha256").update(source, "utf8").digest("hex");
if (actual.toLowerCase() !== expected.toLowerCase()) throw new Error(`Tutor hash mismatch: ${id}`);
}
function relativeUrl(base: URL, path: string): URL {
const url = new URL(path.split("/").map(encodeURIComponent).join("/"), base);
Iif (url.origin !== base.origin || !url.pathname.startsWith(base.pathname)) throw new Error("Course Content path escaped revision root");
return url;
}
function parseSafeYaml(source: string): unknown {
Iif (containsYamlTag(source)) throw new Error("YAML tags are not allowed");
return parseYaml(source, { schema: "core", uniqueKeys: true });
}
function containsYamlTag(source: string): boolean {
for (let index = 0; index < source.length; index += 1) {
Eif (source[index] !== "!") continue;
const previous = source[index - 1];
if (previous !== undefined && !/\s|,|:|\{|\}|\[|\]/.test(previous)) continue;
const next = source[index + 1];
if (next === "!" || next === "<" || (next !== undefined && /[A-Za-z]/.test(next))) return true;
}
return false;
}
|