typescript공급자 중립 model gateway 실행 경계 의사코드
type Json = null | boolean | number | string | Json[] | { [key: string]: Json };
type GatewayErrorCode =
| "invalid_request"
| "auth_failed"
| "policy_denied"
| "rate_limited"
| "queue_full"
| "overloaded"
| "timeout"
| "canceled"
| "unavailable"
| "invalid_output"
| "unknown_outcome";
type VersionPins = {
requestContract: string;
gatewayConfig: string;
prompt: string;
systemPolicy: string;
outputSchema: string;
routingPolicy: string;
priceCard: string;
};
type ModelRequest = {
requestId: string;
tenantId: string;
principalId: string;
input: Json;
pins: VersionPins;
privacy: "public" | "tenant_private" | "confidential";
deadlineAt: number;
stream: boolean;
cache: "off" | "private" | "public";
};
type PinnedConfig = {
modelAlias: string;
generation: { temperature: number; maxOutputTokens: number };
requiredCapabilities: string[];
maxProviderAttempts: number;
maxRepairAttempts: number;
maxFallbacks: number;
maxQueuedMs: number;
maxBufferedBytes: number;
retryBaseMs: number;
cacheTtlMs: number;
credentialRef: string;
};
type RouteCandidate = {
providerId: string;
modelId: string;
region: string;
capabilities: string[];
};
type ProviderRequest = {
requestId: string;
modelId: string;
messages: Json;
outputSchema: Json;
generation: PinnedConfig["generation"];
credentialRef: string;
};
type ProviderUsage = { inputTokens: number; outputTokens: number };
type ProviderResult = { modelId: string; text: string; usage: ProviderUsage; providerRequestId: string };
type ProviderEvent =
| { kind: "text_delta"; sequence: number; text: string }
| { kind: "usage"; sequence: number; usage: ProviderUsage }
| { kind: "finish"; sequence: number; reason: string };
interface ProviderAdapter {
readonly id: string;
readonly capabilities: ReadonlySet<string>;
complete(request: ProviderRequest, signal: AbortSignal): Promise<ProviderResult>;
stream(request: ProviderRequest, signal: AbortSignal): AsyncIterable<ProviderEvent>;
normalizeError(error: unknown): GatewayFailure;
}
type GatewayFailure = Error & {
code: GatewayErrorCode;
retryable: boolean;
providerFault: boolean;
retryAfterMs?: number;
};
interface RuntimeDependencies {
registry: {
pin(request: ModelRequest): Promise<{ config: PinnedConfig; prompt: Json; schema: Json }>;
};
router: {
plan(request: ModelRequest, config: PinnedConfig): Promise<RouteCandidate[]>;
};
adapters: Map<string, ProviderAdapter>;
validator: {
parseAndValidate(text: string, schema: Json):
| { ok: true; value: Json }
| { ok: false; safeErrors: string[] };
};
admission: {
acquire(key: string, deadlineAt: number, signal: AbortSignal): Promise<() => void>;
};
circuit: {
allows(candidate: RouteCandidate): boolean;
recordSuccess(candidate: RouteCandidate, latencyMs: number): void;
recordFailure(candidate: RouteCandidate, error: GatewayFailure): void;
};
cache: {
get(key: string, privacy: ModelRequest["privacy"]): Promise<Json | undefined>;
put(key: string, value: Json, ttlMs: number, privacy: ModelRequest["privacy"]): Promise<void>;
};
trace: {
event(name: string, fields: Record<string, Json>): void;
};
clock: { now(): number; sleep(ms: number, signal: AbortSignal): Promise<void> };
hash(value: Json): string;
}
function remainingMs(request: ModelRequest, now: number): number {
return Math.max(0, request.deadlineAt - now);
}
function fullJitter(baseMs: number, attempt: number, retryAfterMs = 0): number {
const exponentialCap = baseMs * Math.pow(2, attempt);
return Math.max(retryAfterMs, Math.floor(Math.random() * exponentialCap));
}
function canRetry(error: GatewayFailure): boolean {
return error.retryable && ["rate_limited", "overloaded", "timeout", "unavailable"].includes(error.code);
}
function canFallback(error: GatewayFailure): boolean {
return ["rate_limited", "overloaded", "timeout", "unavailable"].includes(error.code);
}
function normalizeFailure(adapter: ProviderAdapter, error: unknown): GatewayFailure {
if (
error instanceof Error &&
typeof (error as Partial<GatewayFailure>).code === "string" &&
typeof (error as Partial<GatewayFailure>).retryable === "boolean" &&
typeof (error as Partial<GatewayFailure>).providerFault === "boolean"
) {
return error as GatewayFailure;
}
return adapter.normalizeError(error);
}
class ModelGateway {
constructor(private readonly d: RuntimeDependencies) {}
async complete(request: ModelRequest, signal: AbortSignal): Promise<Json> {
const startedAt = this.d.clock.now();
const pinned = await this.d.registry.pin(request);
const cacheKey = this.cacheKey(request, pinned.config);
this.d.trace.event("gateway.request", {
requestId: request.requestId,
tenantId: request.tenantId,
pins: request.pins as unknown as Json,
inputHash: this.d.hash(request.input),
privacy: request.privacy,
});
if (request.cache !== "off") {
const cached = await this.d.cache.get(cacheKey, request.privacy);
if (cached !== undefined) {
this.d.trace.event("cache.hit", { requestId: request.requestId });
return cached;
}
}
const release = await this.d.admission.acquire(
request.tenantId + ":" + pinned.config.modelAlias,
Math.min(request.deadlineAt, startedAt + pinned.config.maxQueuedMs),
signal,
);
try {
const route = await this.d.router.plan(request, pinned.config);
let lastFailure: GatewayFailure | undefined;
let fallbackCount = 0;
for (const candidate of route) {
if (fallbackCount > pinned.config.maxFallbacks) break;
if (!this.d.circuit.allows(candidate)) continue;
const adapter = this.requireAdapter(candidate, pinned.config);
try {
const value = await this.callWithRetryAndRepair(
adapter,
candidate,
request,
pinned,
signal,
);
if (request.cache !== "off") {
await this.d.cache.put(cacheKey, value, pinned.config.cacheTtlMs, request.privacy);
}
this.d.trace.event("gateway.completed", {
requestId: request.requestId,
latencyMs: this.d.clock.now() - startedAt,
providerId: candidate.providerId,
modelId: candidate.modelId,
fallbackCount,
});
return value;
} catch (error) {
const failure = normalizeFailure(adapter, error);
lastFailure = failure;
if (failure.providerFault) this.d.circuit.recordFailure(candidate, failure);
if (!canFallback(failure)) throw failure;
fallbackCount += 1;
}
}
throw lastFailure ?? Object.assign(new Error("No eligible route"), {
code: "unavailable" as const,
retryable: true,
providerFault: false,
});
} finally {
release();
}
}
async stream(
request: ModelRequest,
sink: { write(event: ProviderEvent): Promise<void>; commit(value: Json): Promise<void>; abort(error: GatewayFailure): Promise<void> },
signal: AbortSignal,
): Promise<void> {
const startedAt = this.d.clock.now();
const pinned = await this.d.registry.pin(request);
const release = await this.d.admission.acquire(
request.tenantId + ":" + pinned.config.modelAlias,
Math.min(request.deadlineAt, startedAt + pinned.config.maxQueuedMs),
signal,
);
try {
const route = await this.d.router.plan(request, pinned.config);
const candidate = route.find((item) => this.d.circuit.allows(item));
if (!candidate) throw new Error("No streaming route");
const adapter = this.requireAdapter(candidate, pinned.config);
const providerRequest = this.providerRequest(request, pinned, candidate);
const deadlineSignal = AbortSignal.timeout(remainingMs(request, this.d.clock.now()));
const combinedSignal = AbortSignal.any([signal, deadlineSignal]);
let firstTokenAt: number | undefined;
let buffered = "";
try {
// Do not switch providers after exposing a partial stream to the consumer.
for await (const event of adapter.stream(providerRequest, combinedSignal)) {
if (combinedSignal.aborted) throw Object.assign(new Error("Canceled or timed out"), {
code: signal.aborted ? "canceled" as const : "timeout" as const,
retryable: false,
providerFault: false,
});
if (event.kind === "text_delta") {
firstTokenAt ??= this.d.clock.now();
buffered += event.text;
if (new TextEncoder().encode(buffered).byteLength > pinned.config.maxBufferedBytes) {
throw Object.assign(new Error("Stream buffer exceeded"), {
code: "invalid_output" as const,
retryable: false,
providerFault: false,
});
}
}
await sink.write(event); // Pull the next provider event only after the consumer accepts this one.
}
const validated = this.d.validator.parseAndValidate(buffered, pinned.schema);
if (!validated.ok) {
throw Object.assign(new Error("Invalid streamed output"), {
code: "invalid_output" as const,
retryable: false,
providerFault: false,
});
}
await sink.commit(validated.value);
if (request.cache !== "off") {
await this.d.cache.put(
this.cacheKey(request, pinned.config),
validated.value,
pinned.config.cacheTtlMs,
request.privacy,
);
}
this.d.circuit.recordSuccess(candidate, this.d.clock.now() - startedAt);
this.d.trace.event("stream.completed", {
requestId: request.requestId,
ttftMs: firstTokenAt === undefined ? -1 : firstTokenAt - startedAt,
latencyMs: this.d.clock.now() - startedAt,
});
} catch (error) {
const failure = normalizeFailure(adapter, error);
if (failure.providerFault) this.d.circuit.recordFailure(candidate, failure);
await sink.abort(failure);
throw failure;
}
} finally {
release();
}
}
private async callWithRetryAndRepair(
adapter: ProviderAdapter,
candidate: RouteCandidate,
request: ModelRequest,
pinned: Awaited<ReturnType<RuntimeDependencies["registry"]["pin"]>>,
signal: AbortSignal,
): Promise<Json> {
let messages = pinned.prompt;
let repairCount = 0;
const invoke = async (currentMessages: Json): Promise<ProviderResult> => {
for (let attempt = 0; attempt < pinned.config.maxProviderAttempts; attempt += 1) {
if (remainingMs(request, this.d.clock.now()) <= 0) {
throw Object.assign(new Error("Request deadline exceeded"), {
code: "timeout" as const,
retryable: false,
providerFault: false,
});
}
try {
return await adapter.complete(
{ ...this.providerRequest(request, pinned, candidate), messages: currentMessages },
signal,
);
} catch (error) {
const failure = normalizeFailure(adapter, error);
if (!canRetry(failure) || attempt + 1 >= pinned.config.maxProviderAttempts) throw failure;
const waitMs = fullJitter(pinned.config.retryBaseMs, attempt, failure.retryAfterMs);
if (waitMs >= remainingMs(request, this.d.clock.now())) throw failure;
await this.d.clock.sleep(waitMs, signal);
}
}
throw new Error("Unreachable retry state");
};
while (true) {
const callStartedAt = this.d.clock.now();
const result = await invoke(messages);
this.d.circuit.recordSuccess(candidate, this.d.clock.now() - callStartedAt);
const checked = this.d.validator.parseAndValidate(result.text, pinned.schema);
this.d.trace.event("provider.success", {
requestId: request.requestId,
providerRequestId: result.providerRequestId,
providerId: candidate.providerId,
modelId: result.modelId,
inputTokens: result.usage.inputTokens,
outputTokens: result.usage.outputTokens,
schemaValid: checked.ok,
repairCount,
});
if (checked.ok) return checked.value;
if (repairCount >= pinned.config.maxRepairAttempts) {
throw Object.assign(new Error("Structured output failed validation"), {
code: "invalid_output" as const,
retryable: false,
providerFault: false,
});
}
repairCount += 1;
messages = {
task: pinned.prompt,
previousOutput: result.text,
validationErrors: checked.safeErrors,
instruction: "Return a complete replacement matching the pinned schema; do not add commentary.",
};
}
}
private requireAdapter(candidate: RouteCandidate, config: PinnedConfig): ProviderAdapter {
const adapter = this.d.adapters.get(candidate.providerId);
if (!adapter) throw new Error("Missing provider adapter: " + candidate.providerId);
for (const capability of config.requiredCapabilities) {
if (!adapter.capabilities.has(capability) || !candidate.capabilities.includes(capability)) {
throw new Error("Route lacks capability: " + capability);
}
}
return adapter;
}
private providerRequest(
request: ModelRequest,
pinned: Awaited<ReturnType<RuntimeDependencies["registry"]["pin"]>>,
candidate: RouteCandidate,
): ProviderRequest {
return {
requestId: request.requestId,
modelId: candidate.modelId,
messages: pinned.prompt,
outputSchema: pinned.schema,
generation: pinned.config.generation,
credentialRef: pinned.config.credentialRef, // Resolve the secret only inside the adapter boundary.
};
}
private cacheKey(request: ModelRequest, config: PinnedConfig): string {
return this.d.hash({
tenantId: request.tenantId,
principalId: request.principalId,
privacy: request.privacy,
pins: request.pins,
generation: config.generation,
input: request.input,
} as unknown as Json);
}
}
METRICS
숫자가 나빠질 때 무엇을 의심할까
- First-pass schema validity
- repair 없이 pinned schema와 업무 불변식을 통과한 최초 출력 비율을 schema version·model·route별로 본다.repair 이후 성공만 보면 처음부터 불안정한 prompt·model과 추가 지연·비용을 숨긴다.
- Repair rate / repair success / amplification
- repair가 필요했던 비율, 제한 안에서 성공한 비율, repair로 늘어난 token·비용·지연을 함께 측정한다.repair 성공률만 높이면 무한 재호출에 가까운 비싼 시스템을 건강하다고 오판할 수 있다.
- TTFT distribution
- request 수신부터 consumer가 첫 유효 token을 받을 때까지의 p50·p95·p99를 queue·route·provider 단계로 분해한다.provider 첫 byte만 재면 gateway queue와 느린 client 전송을 숨긴다.
- Output tokens per second
- 첫 token 이후 유효 output token을 generation 시간으로 나눈 분포를 model·입력 크기·region별로 본다.tokenizer·usage 정의가 다른 공급자를 보정 없이 비교하거나 network 대기 시간을 generation 속도에 섞을 수 있다.
- End-to-end latency and deadline miss rate
- queue, attempt, repair, fallback, validation, flush를 포함한 전체 지연과 사용자 deadline 초과율을 route별로 측정한다.평균 지연은 retry·fallback과 느린 tenant의 tail을 숨기며 provider latency만으로 사용자 경험을 설명할 수 없다.
- Retry / fallback request amplification
- 사용자 요청 한 건이 만든 provider attempt, repair와 fallback 호출 수와 추가 token·비용의 배수다.최종 성공률이 유지돼도 증폭 상승은 quota 고갈, 비용 폭주와 공급자 장애를 숨긴다.
- Admission rejection / queue wait / concurrency saturation
- rate limit 거절, queue_full, queue 대기 분포와 provider·tenant별 concurrency 포화 시간을 분리한다.모든 429와 timeout을 합치면 내부 admission 문제와 외부 provider quota를 구분하지 못한다.
- Circuit-open and fallback quality delta
- circuit open 시간·원인, half-open 회복과 fallback route의 성공·품질·비용·지연 차이를 baseline과 비교한다.가용성만 유지하면 fallback이 더 약한 품질이나 다른 데이터 지역을 사용한 사실을 놓칠 수 있다.
- Cache hit / stale / isolation failure rate
- privacy class별 hit, stale 사용, 삭제 지연과 교차 tenant·권한 scope 노출 시도를 별도 측정한다.높은 hit rate는 잘못된 version key, 오래된 답과 개인정보 재사용을 가릴 수 있다.
- Cost per verified success
- pinned price card와 provider usage로 검증된 성공 하나당 모델 비용을 계산하고 route·slice별 분포를 본다.요청당 비용만 낮추면 실패와 repair가 많은 route가 싸 보일 수 있고 가격표 변경도 비교를 왜곡한다.
- Cancel propagation latency / post-cancel work
- 사용자 취소부터 queue·gateway·provider·consumer가 멈출 때까지의 시간과 취소 뒤 생성된 token·비용을 측정한다.UI만 닫고 upstream 실행이 계속되면 사용자는 취소됐다고 보지만 비용과 데이터 처리는 남는다.
- Secret or cross-tenant disclosure count
- prompt, provider payload, cache, 오류, trace와 stream에서 secret 또는 다른 tenant 데이터가 노출된 사건 수다.목표는 항상 0이며 한 건도 평균 품질에 섞지 않고 배포 중단·사고 조사 대상으로 둔다.