Aswin Biju
3640b2d785
Shops can now send handwritten slips as WhatsApp photos. srs-api reads them through OpenRouter in the background. The model only proposes text: the parser decides what is an entry, and nothing is saved without the user's Save.
216 lines
7.9 KiB
TypeScript
216 lines
7.9 KiB
TypeScript
import { describe, expect, it } from 'vitest';
|
|
import {
|
|
MAX_RETRY_WAIT_MS,
|
|
OPENROUTER_URL,
|
|
SLIP_PROMPT,
|
|
stripCodeFences,
|
|
transcribeImage,
|
|
type TranscribeDeps,
|
|
} from './transcribeImage';
|
|
|
|
const KEY = 'test-key-not-real-Zq7';
|
|
const IMAGE = Buffer.from('fake-image-bytes');
|
|
const MODEL_TEXT = '6379 100 MODELTEXT';
|
|
const ERROR_BODY = 'BODY-MAY-QUOTE-THE-INPUT';
|
|
|
|
type Reply = () => Response;
|
|
|
|
const json =
|
|
(body: unknown, status = 200, headers: Record<string, string> = {}): Reply =>
|
|
() =>
|
|
new Response(JSON.stringify(body), { status, headers });
|
|
const answer = (content: unknown, finish: unknown = 'stop'): Reply =>
|
|
json({ choices: [{ finish_reason: finish, message: { content } }] });
|
|
const fail = (status: number, headers: Record<string, string> = {}, error: unknown = { message: ERROR_BODY }): Reply =>
|
|
json({ error }, status, headers);
|
|
const networkError: Reply = () => {
|
|
throw new TypeError('fetch failed');
|
|
};
|
|
const htmlPage =
|
|
(status: number): Reply =>
|
|
() =>
|
|
new Response('<html>not json</html>', { status });
|
|
|
|
function setup(...replies: Reply[]) {
|
|
const calls: { url: string; init: RequestInit }[] = [];
|
|
const waits: number[] = [];
|
|
const logs: Record<string, unknown>[] = [];
|
|
let attempts = 0;
|
|
const deps: TranscribeDeps = {
|
|
apiKey: KEY,
|
|
model: 'test/model:free',
|
|
timeoutSec: 60,
|
|
fetchImpl: (async (url: string, init: RequestInit) => {
|
|
calls.push({ url, init });
|
|
const reply = replies[calls.length - 1];
|
|
if (!reply) throw new Error('more calls than expected');
|
|
return reply();
|
|
}) as unknown as typeof fetch,
|
|
log: (e) => logs.push(e),
|
|
sleep: async (ms) => {
|
|
waits.push(ms);
|
|
},
|
|
beforeAttempt: async () => {
|
|
attempts++;
|
|
},
|
|
};
|
|
return { deps, calls, waits, logs, attempts: () => attempts };
|
|
}
|
|
|
|
const run = (s: ReturnType<typeof setup>) => transcribeImage(IMAGE, 'image/jpeg', s.deps);
|
|
|
|
describe('the request', () => {
|
|
it('sends the prompt first, reasoning off, a data URL and the key in the header', async () => {
|
|
const s = setup(answer(MODEL_TEXT));
|
|
await run(s);
|
|
|
|
expect(s.calls).toHaveLength(1);
|
|
const { url, init } = s.calls[0]!;
|
|
expect(url).toBe(OPENROUTER_URL);
|
|
expect(init.method).toBe('POST');
|
|
expect(init.redirect).toBe('error');
|
|
expect((init.headers as Record<string, string>).authorization).toBe(`Bearer ${KEY}`);
|
|
|
|
const body = JSON.parse(init.body as string);
|
|
expect(body.model).toBe('test/model:free');
|
|
expect(body.temperature).toBe(0);
|
|
expect(body.max_tokens).toBe(1500);
|
|
expect(body.reasoning).toEqual({ effort: 'none' });
|
|
expect(body.messages[0].content).toEqual([
|
|
{ type: 'text', text: SLIP_PROMPT },
|
|
{ type: 'image_url', image_url: { url: `data:image/jpeg;base64,${IMAGE.toString('base64')}` } },
|
|
]);
|
|
expect(init.body).not.toContain(KEY);
|
|
});
|
|
});
|
|
|
|
describe('a good answer', () => {
|
|
it('returns the text', async () => {
|
|
expect(await run(setup(answer('6379 100\n9001 10')))).toEqual({ ok: true, text: '6379 100\n9001 10' });
|
|
});
|
|
|
|
it('strips a code fence but keeps commentary', async () => {
|
|
expect(await run(setup(answer('```text\n6379 100\n```')))).toEqual({ ok: true, text: '6379 100' });
|
|
expect(await run(setup(answer('Here it is: 6379 100')))).toEqual({ ok: true, text: 'Here it is: 6379 100' });
|
|
});
|
|
});
|
|
|
|
describe('failures that are retried once', () => {
|
|
const retried: [string, Reply][] = [
|
|
['per-minute 429', fail(429, { 'x-ratelimit-limit': '20' })],
|
|
['a 429 with no limit header', fail(429)],
|
|
["a provider's own 429", fail(429, { 'x-ratelimit-limit': '1000' }, { metadata: { provider_code: 'x' } })],
|
|
['408', fail(408)],
|
|
['502', fail(502)],
|
|
['503', fail(503)],
|
|
['a 503 error page that is not JSON', htmlPage(503)],
|
|
['a network error', networkError],
|
|
['a warm-up reply with no choices', json({ choices: [] })],
|
|
['a warm-up reply with no finish reason', answer('', null)],
|
|
['a 200 that is not JSON', htmlPage(200)],
|
|
['503 inside a 200', json({ error: { code: 503, message: ERROR_BODY } })],
|
|
];
|
|
|
|
it.each(retried)('%s: the second try succeeds', async (_name, reply) => {
|
|
const s = setup(reply, answer(MODEL_TEXT));
|
|
expect(await run(s)).toEqual({ ok: true, text: MODEL_TEXT });
|
|
expect(s.calls).toHaveLength(2);
|
|
});
|
|
|
|
it.each(retried)('%s: fails twice, no third try', async (_name, reply) => {
|
|
const s = setup(reply, reply);
|
|
expect(await run(s)).toEqual({ ok: false, reason: 'unavailable' });
|
|
expect(s.calls).toHaveLength(2);
|
|
});
|
|
|
|
it('asks the limiter before every attempt, retries included', async () => {
|
|
const s = setup(fail(503), answer(MODEL_TEXT));
|
|
await run(s);
|
|
expect(s.attempts()).toBe(2);
|
|
});
|
|
|
|
it('waits for Retry-After, capped, and 5 s without a usable one', async () => {
|
|
const given = setup(fail(503, { 'retry-after': '7' }), answer(MODEL_TEXT));
|
|
await run(given);
|
|
expect(given.waits).toEqual([7000]);
|
|
|
|
const capped = setup(fail(503, { 'retry-after': '500' }), answer(MODEL_TEXT));
|
|
await run(capped);
|
|
expect(capped.waits).toEqual([MAX_RETRY_WAIT_MS]);
|
|
|
|
const none = setup(fail(503), answer(MODEL_TEXT));
|
|
await run(none);
|
|
expect(none.waits).toEqual([5000]);
|
|
|
|
const date = setup(fail(503, { 'retry-after': 'Wed, 04 Nov 2026 10:00:00 GMT' }), answer(MODEL_TEXT));
|
|
await run(date);
|
|
expect(date.waits).toEqual([5000]);
|
|
});
|
|
});
|
|
|
|
describe('failures that are not retried', () => {
|
|
const once: [string, Reply, string][] = [
|
|
['daily 429 at the free limit', fail(429, { 'x-ratelimit-limit': '50' }), 'daily_limit'],
|
|
['daily 429 at the higher limit', fail(429, { 'x-ratelimit-limit': '1000' }), 'daily_limit'],
|
|
['daily 429 inside a 200', json({ error: { code: 429 } }, 200, { 'x-ratelimit-limit': '50' }), 'daily_limit'],
|
|
['400', fail(400), 'unreadable'],
|
|
['401', fail(401), 'unreadable'],
|
|
['402', fail(402), 'unreadable'],
|
|
['403', fail(403), 'unreadable'],
|
|
['422', fail(422), 'unreadable'],
|
|
['401 inside a 200', json({ error: { code: 401 } }), 'unreadable'],
|
|
['ran out of budget', answer('', 'length'), 'unreadable'],
|
|
['another finish reason, even with text', answer('6379 100', 'content_filter'), 'unreadable'],
|
|
['content that is not text', answer([{ type: 'text' }]), 'unreadable'],
|
|
['empty text', answer(''), 'no_writing'],
|
|
['null text', answer(null), 'no_writing'],
|
|
['only a fence', answer('```\n```'), 'no_writing'],
|
|
];
|
|
|
|
it.each(once)('%s', async (_name, reply, reason) => {
|
|
const s = setup(reply);
|
|
expect(await run(s)).toEqual({ ok: false, reason });
|
|
expect(s.calls).toHaveLength(1);
|
|
expect(s.waits).toEqual([]);
|
|
});
|
|
});
|
|
|
|
describe('logs', () => {
|
|
it('never hold the key, the image, the model text or an error body', async () => {
|
|
const replies: Reply[] = [
|
|
answer(MODEL_TEXT),
|
|
fail(401),
|
|
json({ error: { code: 401, message: ERROR_BODY } }),
|
|
answer(MODEL_TEXT, 'length'),
|
|
networkError,
|
|
];
|
|
for (const reply of replies) {
|
|
const s = setup(reply, reply);
|
|
await run(s);
|
|
expect(s.logs.length).toBeGreaterThan(0);
|
|
const logged = JSON.stringify(s.logs);
|
|
for (const secret of [KEY, MODEL_TEXT, ERROR_BODY, IMAGE.toString('base64'), 'fetch failed']) {
|
|
expect(logged).not.toContain(secret);
|
|
}
|
|
}
|
|
});
|
|
|
|
it('record status, finish reason, error type and duration', async () => {
|
|
const s = setup(answer(MODEL_TEXT));
|
|
await run(s);
|
|
expect(s.logs[0]).toMatchObject({ status: 200, finish_reason: 'stop' });
|
|
expect(s.logs[0]).toHaveProperty('duration_ms');
|
|
|
|
const bad = setup(fail(401));
|
|
await run(bad);
|
|
expect(bad.logs[0]).toMatchObject({ status: 401, error_type: 'http_error', level: 'warn' });
|
|
});
|
|
});
|
|
|
|
describe('stripCodeFences', () => {
|
|
it('leaves plain text alone and handles an unclosed fence', () => {
|
|
expect(stripCodeFences(' 6379 100 ')).toBe('6379 100');
|
|
expect(stripCodeFences('```\n6379 100')).toBe('6379 100');
|
|
});
|
|
});
|