feat(parser): built the order parser

This commit is contained in:
Aswin B. S 2026-11-03 12:53:26 +05:30
parent 6851d66b38
commit ed4d1280e1
18 changed files with 1345 additions and 51 deletions

30
src/parser/corpus.test.ts Normal file
View File

@ -0,0 +1,30 @@
import { readdirSync, readFileSync } from 'node:fs';
import { join } from 'node:path';
import { describe, expect, it } from 'vitest';
import { parseOrder } from './parseOrder';
const DIR = join(__dirname, '../../test/fixtures/orders');
const names = readdirSync(DIR)
.filter((f) => f.endsWith('.txt'))
.map((f) => f.slice(0, -4));
describe('regression corpus', () => {
it('has at least one fixture', () => {
expect(names.length).toBeGreaterThan(0);
});
for (const name of names) {
it(name, () => {
const r = parseOrder(readFileSync(join(DIR, `${name}.txt`), 'utf8'));
const actual = {
totalTickets: r.totalTickets,
lines: r.lines.map((l) => `${l.ending} x${l.qty}${l.fromRange ? ' range' : ''} @${l.line}`),
repeated: r.repeated,
unreadable: r.unreadable,
ignored: r.ignored,
};
const expected = JSON.parse(readFileSync(join(DIR, `${name}.expected.json`), 'utf8'));
expect(actual).toEqual(expected);
});
}
});

View File

@ -0,0 +1,277 @@
import { describe, expect, it } from 'vitest';
import { read } from '../testing/parser';
import { parseOrder } from './parseOrder';
describe('6.1 input hygiene', () => {
it('EC-02 CRLF, CR and mixed line endings read like LF', () => {
const lf = parseOrder('0888\n0057x2\n0003');
for (const t of ['0888\r\n0057x2\r\n0003', '0888\r0057x2\r0003', '0888\r\n0057x2\r0003']) {
expect(parseOrder(t)).toEqual(lf);
}
});
it('EC-03 invisible marks are removed, line numbers unchanged', () => {
const r = parseOrder('‎0888\n08​88 0057');
expect(r.lines.map((l) => [l.ending, l.line])).toEqual([['0888', 1], ['0888', 2], ['0057', 2]]);
});
it('EC-04 tabs and runs of spaces are noise', () => {
expect(read('0888\t\t0057 0003').lines).toEqual(['0888x1', '0057x1', '0003x1']);
});
it('EC-05 full-width digits become ASCII', () => {
expect(read('0888').lines).toEqual(['0888x1']);
});
it('EC-06 Malayalam or Arabic-Indic digits are unreadable', () => {
expect(read('൦൮൮൮ ٠٨٨٨').unreadable).toEqual(['൦൮൮൮:unsupported_digits', '٠٨٨٨:unsupported_digits']);
});
it('EC-07 input above the limit is reported, not read', () => {
const r = parseOrder('0888 '.repeat(30_000));
expect(r.lines).toEqual([]);
expect(r.unreadable[0]?.reason).toBe('input_too_long');
});
});
describe('6.2 prefixes', () => {
it('EC-10 app copy prefix', () => {
expect(read('[12/10, 14:35] Ravi: 0888').lines).toEqual(['0888x1']);
});
it('EC-11 exported prefix', () => {
expect(read('12/10/2026, 2:35 pm - Ravi: 0888').lines).toEqual(['0888x1']);
expect(read('12/10/26, 14:35 - Ravi: 0057x2').lines).toEqual(['0057x2']);
});
it('EC-12 a name with digits is removed completely', () => {
expect(read('[12/10, 14:35] Shop 2 Main: 0888')).toMatchObject({ lines: ['0888x1'], unreadable: [], ignored: [] });
});
it('EC-13 a prefix on every line of a merged forward', () => {
expect(read('[12/10, 14:35] A: 0888\n[12/10, 14:36] B: 0057x2').lines).toEqual(['0888x1', '0057x2']);
});
it('EC-14 a line that only looks like a prefix is not stripped', () => {
expect(read('12/10 0888')).toMatchObject({ lines: ['0888x1'], unreadable: ['12/10:date_or_time'] });
});
it('EC-15 prefix only', () => {
expect(read('[12/10, 14:35] Ravi:')).toEqual({ lines: [], unreadable: [], ignored: [], tickets: 0 });
});
it('EC-16 seconds and am/pm in the time are stripped, never read as 1435', () => {
expect(read('[12/10/26, 2:35:20 pm] Ravi: 0888').lines).toEqual(['0888x1']);
expect(read('[12/10, 14:35:20] Ravi: 0888').lines).toEqual(['0888x1']);
});
});
describe('6.3 endings', () => {
it('EC-21 leading zeros survive', () => {
expect(parseOrder('0007 0000').lines.map((l) => l.ending)).toEqual(['0007', '0000']);
});
it('EC-22 1-3 digits on an entry line are unreadable, never padded', () => {
expect(read('0888 2 888 0003').unreadable).toEqual(['888:stray_number']);
expect(read('7 0888').unreadable).toEqual(['7:stray_number']);
});
it('EC-23 5+ digits are unreadable, never truncated', () => {
expect(read('12345 404595').unreadable).toEqual(['12345:bad_ending_length', '404595:bad_ending_length']);
});
it('EC-24 letters mixed in are unreadable', () => {
expect(read('O888 08S8 0888').unreadable).toEqual(['O888:mixed_characters', '08S8:mixed_characters']);
// 0 right after an ending is a (zero) quantity by the space rule; 888 is still reported, not ignored.
expect(read('0888 0 888').unreadable).toEqual(['0888 0:zero_quantity', '888:stray_number']);
expect(read('0 888')).toMatchObject({ unreadable: [], ignored: ['0', '888'] }); // known limit: alone on a line
});
it('EC-25 ? or x as an unclear digit', () => {
expect(read('08?8 0xx8 0888?').unreadable).toEqual(['08?8:unclear_digit', '0xx8:unclear_digit', '0888?:unclear_digit']);
});
it('EC-26 endings glued together are not split', () => {
expect(read('08880003').unreadable).toEqual(['08880003:bad_ending_length']);
});
it('EC-28 , ; . | are noise; / is not', () => {
expect(read('0888;0057.0003|3472').lines).toEqual(['0888x1', '0057x1', '0003x1', '3472x1']);
expect(read('0888/0057').unreadable).toEqual(['0888/0057:date_or_time']);
});
});
describe('6.4 quantities', () => {
it('EC-30 x and * in every spacing', () => {
expect(read('0057x2 0057 x 2 0057X2 0057×2 0057*2 0057 * 2').lines).toEqual(Array(6).fill('0057x2'));
});
it('EC-32 bracket with spaces', () => {
expect(read('0888 (2) 0888( 2 )').lines).toEqual(['0888x2', '0888x2']);
});
it('EC-33 dash quantity of 1 to 3 digits', () => {
expect(read('0003-2 0003-12 0003-123').lines).toEqual(['0003x2', '0003x12', '0003x123']);
});
it('EC-34 zero quantity in every form', () => {
expect(read('0003-0 0057x0 0888(0) 8910=0').unreadable.map((u) => u.split(':')[1])).toEqual(Array(4).fill('zero_quantity'));
});
it('EC-35 quantity above the limit', () => {
expect(read('0057x5000 0057*1000').unreadable).toEqual(['0057x5000:quantity_too_large', '0057*1000:quantity_too_large']);
});
it('EC-36 a quantity with no ending: unreadable on an entry line, ignored alone', () => {
expect(read('0888 , x2').unreadable).toEqual(['x2:orphan_quantity']);
expect(read('0888\n(3)')).toMatchObject({ lines: ['0888x1'], unreadable: [], ignored: ['(3)'] });
});
it('EC-37 two quantities are not multiplied', () => {
expect(read('0057x2x3 0057x2(3)').unreadable).toEqual(['0057x2x3:double_quantity', '0057x2(3):double_quantity']);
});
it('EC-38 decimal or negative quantities', () => {
expect(read('0057x2.5 0057x-1').unreadable).toEqual(['0057x2.5:mixed_characters', '0057x-1:mixed_characters']);
});
it('EC-39 A,B,C-2 binds the quantity to C only (provisional, Q3)', () => {
expect(read('0888,0057,0003-2').lines).toEqual(['0888x1', '0057x1', '0003x2']);
});
it('EC-39c space quantity is 1 to 3 digits; 4 digits is the next ending', () => {
expect(read('7784 1 7764 12 3367 123').lines).toEqual(['7784x1', '7764x12', '3367x123']);
expect(read('7784 1234').lines).toEqual(['7784x1', '1234x1']);
});
it('EC-39e a quantity on the next line is not a quantity', () => {
expect(read('7784\n1')).toMatchObject({ lines: ['7784x1'], ignored: ['1'] });
});
it('EC-39f = and _ in every spacing', () => {
expect(read('8910=3 8910 = 3 0159_1 0159 _ 1').lines).toEqual(['8910x3', '8910x3', '0159x1', '0159x1']);
});
it('EC-39h 4 digits after = or _ is never a quantity or an ending', () => {
expect(read('8910=1234 0159_1234').unreadable).toEqual(['8910=1234:quantity_too_large', '0159_1234:quantity_too_large']);
});
it('EC-39i separators mixed in one order', () => {
expect(read('3175-1\n0702=1\n0159_1\n7740(2)').lines).toEqual(['3175x1', '0702x1', '0159x1', '7740x2']);
});
});
describe('6.5 ranges', () => {
it('EC-41 single-ending range', () => {
expect(read('5104-5104').lines).toEqual(['5104x1']);
});
it('EC-43 no wrap-around', () => {
expect(read('9998-0001').unreadable).toEqual(['9998-0001:bad_range']);
});
it('EC-44 range too large', () => {
expect(read('0000-9999').unreadable).toEqual(['0000-9999:range_too_large']);
expect(read('0000-0099').lines).toHaveLength(100);
expect(read('0000-0100').unreadable).toEqual(['0000-0100:range_too_large']);
});
it('EC-45 crossing a hundred keeps zero padding', () => {
expect(read('0099-0101').lines).toEqual(['0099x1', '0100x1', '0101x1']);
});
it('EC-46 a range with a quantity is unreadable (provisional, Q5)', () => {
expect(read('5104-5106x2').unreadable).toEqual(['5104-5106x2:bad_range']);
});
it('EC-47 a spaced dash between numbers is unreadable, so 5105 is never lost', () => {
expect(read('5104 - 5106').unreadable).toEqual(['5104 - 5106:bad_range']);
expect(read('5104 -5106').unreadable).toEqual(['5104 -5106:bad_range']);
});
it('EC-48 dash quantity followed by a range', () => {
expect(read('0003-2-5104').unreadable).toEqual(['0003-2-5104:double_quantity']);
});
});
describe('6.6 totals and ignored text', () => {
it('EC-50 every total form is ignored', () => {
for (const t of ['Totel 54 number', 'Total 54', 'total: 54', '54 total', 'Tot-54', 'Total..50', 'Total =20', 'മൊത്തം 29']) {
expect(read(t), t).toMatchObject({ lines: [], unreadable: [] });
}
});
it('EC-51 bare numbers on their own line are ignored', () => {
expect(read('0888\n250')).toMatchObject({ lines: ['0888x1'], unreadable: [], ignored: ['250'] });
});
it('EC-63 a keyword with no number', () => {
expect(read('Total')).toMatchObject({ unreadable: [], ignored: ['Total'] });
});
it('EC-64 split digits are never joined', () => {
expect(read('Total 5 4')).toMatchObject({ unreadable: [], ignored: ['Total', '5', '4'] });
});
it('EC-65 known limit: a 4-digit total is read as an ending (Q8)', () => {
expect(read('Total 1200').lines).toEqual(['1200x1']);
});
it('EC-66 a total on an entry line is unreadable (Q9)', () => {
expect(read('0888 0057 Total 2')).toMatchObject({ lines: ['0888x1', '0057x1'], unreadable: ['2:stray_number'] });
});
});
describe('6.7 repeats', () => {
it('EC-70 two occurrences are both counted', () => {
const r = parseOrder('0888\n0057\n0888');
expect(r.totalTickets).toBe(3);
expect(r.repeated).toEqual([{ ending: '0888', occurrences: 2, totalQty: 2 }]);
});
it('EC-72 a repeat inside a range', () => {
expect(parseOrder('0888 0887-0889').repeated).toEqual([{ ending: '0888', occurrences: 2, totalQty: 2 }]);
});
});
describe('6.8 mixed, merged and noisy input', () => {
it('EC-81 other-script words are ignored', () => {
expect(read('ഇന്നത്തെ 0888')).toMatchObject({ lines: ['0888x1'], unreadable: [] });
});
it('EC-82 emoji are ignored', () => {
expect(read('0888 ✅ 👍')).toMatchObject({ lines: ['0888x1'], unreadable: [], ignored: ['✅', '👍'] });
});
it('EC-83 the same order twice shows as repeats', () => {
expect(parseOrder('0888 0057x2\n0888 0057x2').repeated.map((r) => r.ending)).toEqual(['0888', '0057']);
});
it('EC-84 merged messages with prefixes and totals', () => {
const t = '[12/10, 14:35] A: 0888 0057x2\nTotal 3\n\n[12/10, 14:40] A: 0003-2\n250';
expect(read(t)).toMatchObject({ lines: ['0888x1', '0057x2', '0003x2'], unreadable: [] });
});
it('EC-85 only a total', () => {
expect(read('250')).toEqual({ lines: [], unreadable: [], ignored: ['250'], tickets: 0 });
});
it('EC-86 digits inside words', () => {
expect(read('ab0888cd 0888a').unreadable).toEqual(['ab0888cd:mixed_characters', '0888a:mixed_characters']);
});
it('EC-87 decimals and thousands separators', () => {
expect(read('0888.5 0057').unreadable).toEqual(['0888.5:mixed_characters']);
// 1 right after an ending is its quantity (space rule); 200 is reported, never joined into 1200.
expect(read('0057 1,200')).toMatchObject({ lines: ['0057x1'], unreadable: ['200:stray_number'] });
expect(read('1,200')).toMatchObject({ lines: [], ignored: ['1', '200'] });
});
it('EC-88 dates and times inside the body', () => {
expect(read('0888 12/10/2026 14:35').unreadable).toEqual(['12/10/2026:date_or_time', '14:35:date_or_time']);
});
it('a word right after a bracket quantity is separate text', () => {
expect(read('0888(1)Ravi')).toMatchObject({ lines: ['0888x1'], unreadable: [], ignored: ['Ravi'] });
});
});

View File

@ -0,0 +1,58 @@
import { describe, expect, it } from 'vitest';
import { rng } from '../testing/parser';
import { parseOrder } from './parseOrder';
type Line = [ending: string, qty: number];
const pad4 = (n: number): string => String(n).padStart(4, '0');
const SEPARATORS = [' ', ', ', ',', '\n', ' ; ', '\t', ',,, ', '\n\n'];
function generate(seed: number): { text: string; expected: Line[] } {
const r = rng(seed);
const expected: Line[] = [];
const parts: string[] = [];
const count = 1 + r.int(40);
for (let k = 0; k < count; k++) {
if (r.int(10) === 0) {
// range of 1-5 endings, qty 1 each
const from = r.int(9990);
const to = from + r.int(5);
for (let e = from; e <= to; e++) expected.push([pad4(e), 1]);
parts.push(`${pad4(from)}-${pad4(to)}`);
continue;
}
const ending = pad4(r.int(10000));
const qty = r.int(3) === 0 ? 1 : 1 + r.int(150);
const form = r.pick(['bare', 'x', ' x ', '*', '=', ' = ', '_', '-', '()', ' ()', 'space'] as const);
const q = String(qty);
const rendered =
form === 'bare' ? (qty === 1 ? ending : `${ending}x${q}`)
: form === '()' ? `${ending}(${q})`
: form === ' ()' ? `${ending} (${q})`
: form === 'space' ? `${ending} ${q}`
: `${ending}${form}${q}`;
expected.push([ending, qty]);
parts.push(rendered);
}
let text = r.int(3) === 0 ? '[12/10, 14:35] Shop 2: ' : '';
parts.forEach((p, k) => {
text += p;
if (k < parts.length - 1) text += r.pick(SEPARATORS);
if (r.int(12) === 0) text += `\n${r.pick(['Total..', 'Total =', 'Totel ', 'മൊത്തം ', ''])}${r.int(1000)}\n`;
});
if (r.int(2) === 0) text += `\nTotal ${r.int(1000)}`;
return { text, expected };
}
describe('generated orders round-trip', () => {
const seeds = process.env.SWEEP ? 20_000 : 400;
it(`reads back ${seeds} generated orders exactly`, () => {
for (let seed = 1; seed <= seeds; seed++) {
const { text, expected } = generate(seed);
const r = parseOrder(text);
expect(r.unreadable, `seed ${seed}: ${JSON.stringify(text)}`).toEqual([]);
expect(r.lines.map((l): Line => [l.ending, l.qty]), `seed ${seed}`).toEqual(expected);
}
});
});

View File

@ -0,0 +1,84 @@
import { readdirSync, readFileSync } from 'node:fs';
import { join } from 'node:path';
import { describe, expect, it } from 'vitest';
import { rng } from '../testing/parser';
import { MAX_QTY } from './limits';
import { parseOrder, scanOrder } from './parseOrder';
const ALPHABET = [
...'0123456789012345678901234567890123456789',
...'xX*=_-()/:.,;| \n\n\t?Oa',
'മൊ', '٠', '൮', '✅', '​', ' ', '×', '—', '[12/10, 14:35] A: ', 'Total', '0888', '5104-5106',
];
function garbage(seed: number): string {
const r = rng(seed);
let s = '';
const len = r.int(200);
for (let k = 0; k < len; k++) s += r.pick(ALPHABET);
return s;
}
const FIXTURES = join(__dirname, '../../test/fixtures/orders');
const fixtureTexts = readdirSync(FIXTURES)
.filter((f) => f.endsWith('.txt'))
.map((f) => readFileSync(join(FIXTURES, f), 'utf8'));
const named = [
'7784 1 7764 1 3367 2', '0888(1)0003(1)9077(1)', '5104-5106 0099-0101', '888 12345 O888 08?8 0888',
'Totel 54 number 250', '[12/10, 14:35] Shop 2: 0888', '0057x2x3 0003-2-5104 5104 - 5106', '12/10/2026 14:35 0888.5',
];
const inputs = [...named, ...fixtureTexts, ...Array.from({ length: 500 }, (_, s) => garbage(s + 1))];
const digitsOf = (s: string): string => s.replace(/[^0-9]/g, '');
describe('parser invariants', () => {
it('I1 digit conservation: every digit lands in exactly one item, in order', () => {
for (const t of inputs) {
const { bodies, items } = scanOrder(t);
expect(digitsOf(items.map((i) => i.text).join('')), JSON.stringify(t)).toBe(digitsOf(bodies.join('\n')));
for (const i of items) if (i.kind === 'ignored' && i.reason === 'text') expect(digitsOf(i.text)).toBe('');
}
});
it('I2 never throws, for any string', () => {
for (const t of inputs) expect(() => parseOrder(t)).not.toThrow();
});
it('I3 deterministic', () => {
for (const t of inputs) expect(parseOrder(t)).toEqual(parseOrder(t));
});
it('I4 re-rendering the lines and parsing again gives the same lines', () => {
for (const t of inputs) {
const once = parseOrder(t).lines.map((l) => [l.ending, l.qty]);
const again = parseOrder(once.map(([e, q]) => `${e}x${q}`).join('\n')).lines.map((l) => [l.ending, l.qty]);
expect(again).toEqual(once);
}
});
it('I6 quantities are 1..MAX_QTY and endings are exactly 4 digits', () => {
for (const t of inputs) {
for (const l of parseOrder(t).lines) {
expect(Number.isInteger(l.qty) && l.qty >= 1 && l.qty <= MAX_QTY).toBe(true);
expect(l.ending).toMatch(/^\d{4}$/);
}
}
});
it('I7 totalTickets is the sum of qty', () => {
for (const t of inputs) {
const r = parseOrder(t);
expect(r.totalTickets).toBe(r.lines.reduce((s, l) => s + l.qty, 0));
}
});
it('I8 no invention: every non-range ending is written on its own line', () => {
for (const t of inputs) {
const { bodies } = scanOrder(t);
for (const l of parseOrder(t).lines) {
if (!l.fromRange) expect(bodies[l.line - 1]).toContain(l.ending);
}
}
});
});

62
src/parser/lexer.ts Normal file
View File

@ -0,0 +1,62 @@
export type AtomKind =
| 'num' // run of ASCII digits
| 'baddigit' // run of non-ASCII digits (Malayalam, Arabic-Indic, Devanagari)
| 'word' // run of anything else that is not an operator, separator or space (letters, emoji, ?, ...)
| 'op' // one of x * = _ - ( ) / : . (x only when it stands alone as a word)
| 'sep'; // , ; |
export interface Atom {
kind: AtomKind;
text: string;
start: number;
end: number;
glued: boolean;
}
const OPS = new Set(['*', '=', '_', '-', '(', ')', '/', ':', '.']);
const SEPS = new Set([',', ';', '|']);
function isAsciiDigit(c: number): boolean {
return c >= 48 && c <= 57;
}
function isOtherDigit(c: number): boolean {
return (
(c >= 0x0d66 && c <= 0x0d6f) || // Malayalam
(c >= 0x0660 && c <= 0x0669) || // Arabic-Indic
(c >= 0x06f0 && c <= 0x06f9) || // Extended Arabic-Indic
(c >= 0x0966 && c <= 0x096f) // Devanagari
);
}
function classOf(ch: string, code: number): AtomKind | 'space' {
if (ch === ' ') return 'space';
if (isAsciiDigit(code)) return 'num';
if (isOtherDigit(code)) return 'baddigit';
if (OPS.has(ch)) return 'op';
if (SEPS.has(ch)) return 'sep';
return 'word';
}
export function lex(body: string): Atom[] {
const atoms: Atom[] = [];
let i = 0;
while (i < body.length) {
const code = body.charCodeAt(i);
const kind = classOf(body[i] as string, code);
if (kind === 'space') {
i++;
continue;
}
const glued = i > 0 && body[i - 1] !== ' ';
let j = i + 1;
if (kind === 'num' || kind === 'baddigit' || kind === 'word') {
while (j < body.length && classOf(body[j] as string, body.charCodeAt(j)) === kind) j++;
}
const text = body.slice(i, j);
const isTimes = kind === 'word' && (text === 'x' || text === 'X');
atoms.push({ kind: isTimes ? 'op' : kind, text: isTimes ? 'x' : text, start: i, end: j, glued });
i = j;
}
return atoms;
}

46
src/parser/limits.test.ts Normal file
View File

@ -0,0 +1,46 @@
import { describe, expect, it } from 'vitest';
import { MAX_INPUT_CHARS } from './limits';
import { parseOrder } from './parseOrder';
function timed(label: string, text: string): number {
const t0 = performance.now();
parseOrder(text);
const ms = performance.now() - t0;
console.log(`${label}: ${ms.toFixed(1)} ms (${text.length} chars)`);
return ms;
}
describe('parser limits', () => {
it('5,000 realistic lines parse well within the bound', () => {
const forms = ['8910=3', '0057x2', '3175-1', '7740(2)', '0159_1', '7784 1'];
const text = Array.from({ length: 5000 }, (_, k) => forms[k % forms.length]).join('\n');
expect(text.length).toBeLessThanOrEqual(MAX_INPUT_CHARS);
const r = parseOrder(text);
expect(r.lines).toHaveLength(5000);
expect(timed('5000 lines', text)).toBeLessThan(1000);
});
it('pathological inputs at the size limit stay bounded', () => {
const size = MAX_INPUT_CHARS - 10;
const cases: [string, string][] = [
['commas', ','.repeat(size)],
['open brackets', '('.repeat(size)],
['dangling dashes', '0888-'.repeat(Math.floor(size / 5))],
['glued range chain', '0888-0889'.repeat(Math.floor(size / 9))],
['word/op alternation', `${'a-'.repeat(Math.floor(size / 2) - 1)}0`],
['one long digit run', '7'.repeat(size)],
['digits and letters', 'a0'.repeat(Math.floor(size / 2))],
];
for (const [label, text] of cases) expect(timed(label, text), label).toBeLessThan(1000);
});
it('input above the limit returns immediately', () => {
const text = '0888 '.repeat(250_000); // ~1.25 MB
expect(timed('over limit', text)).toBeLessThan(50);
expect(parseOrder(text).unreadable[0]?.reason).toBe('input_too_long');
});
it('the largest allowed range expands without trouble', () => {
expect(parseOrder('0000-0099').lines).toHaveLength(100);
});
});

8
src/parser/limits.ts Normal file
View File

@ -0,0 +1,8 @@
/** Largest quantity per entry. Quantities are always 1 to 3 digits.*/
export const MAX_QTY = 999;
/** Largest number of endings one range may expand to. Provisional (plan Q4). */
export const MAX_RANGE = 100;
/** Longest input accepted; matches the ingest route's text cap. */
export const MAX_INPUT_CHARS = 100_000;

17
src/parser/normalize.ts Normal file
View File

@ -0,0 +1,17 @@
const INVISIBLE = /[­​-‏‪-‮⁠-⁩︎️]/g;
// Tabs and Unicode spaces (NBSP, narrow NBSP used by WhatsApp before am/pm, en/em spaces...).
const SPACES = /[\t   -    ]/g;
const TIMES = /[×✕✖]/g; // × ✕ ✖
const DASHES = /[‐-―−﹣-]/g; // hyphens, en/em dash, minus, full-width hyphen
const FULLWIDTH_DIGIT = /[0-9]/g;
export function normalize(text: string): string[] {
return text
.replace(/\r\n?/g, '\n')
.replace(INVISIBLE, '')
.replace(SPACES, ' ')
.replace(TIMES, 'x')
.replace(DASHES, '-')
.replace(FULLWIDTH_DIGIT, (d) => String.fromCharCode(d.charCodeAt(0) - 0xff10 + 48))
.split('\n');
}

View File

@ -1,42 +1,127 @@
import { existsSync, readFileSync } from 'node:fs';
import { join } from 'node:path';
import { describe, expect, it } from 'vitest';
import { parseOrder } from './parseOrder';
import { read } from '../testing/parser';
import { mergeLines, parseOrder } from './parseOrder';
const samplePath = join(__dirname, '../../test/fixtures/sample-order.txt');
describe('parseOrder: notation', () => {
it.todo('a bare 4-digit ending is 1 ticket');
it.todo('0057x2 and 9155*40 set the quantity');
it.todo('0003-2 (dash plus 1-3 digits) is a quantity');
it.todo('5104-5106 (dash plus 4 digits) is a range');
it.todo('0888(1)0003(1)9077(1) with no separators reads as three endings');
it.todo('strips the WhatsApp copy prefix [dd/mm, hh:mm] Name:');
it.todo('"Totel 54 number" is checked by walking back until the ticket sum or line count matches');
it.todo('a bare number on its own line (250) is a total');
it.todo('never guesses: unreadable text goes to `unreadable`');
describe('parseOrder: core cases', () => {
it('1: empty input gives nothing and does not throw', () => {
for (const t of ['', ' ', '\n\n\n']) {
expect(read(t)).toEqual({ lines: [], unreadable: [], ignored: [], tickets: 0 });
}
});
describe.skipIf(!existsSync(samplePath))('parseOrder: the real 90-line sample', () => {
const result = () => parseOrder(readFileSync(samplePath, 'utf8'));
it('2: a bare 4-digit ending is 1 ticket', () => {
expect(read('0888').lines).toEqual(['0888x1']);
});
it('reads 338 tickets and leaves nothing unreadable', () => {
const r = result();
expect(r.totalTickets).toBe(338);
it('3: leading zeros are kept as strings', () => {
const r = parseOrder('0057 0000');
expect(r.lines.map((l) => l.ending)).toEqual(['0057', '0000']);
});
it('4: bare endings separated by spaces', () => {
expect(read('7598 7896 7986 7877').lines).toEqual(['7598x1', '7896x1', '7986x1', '7877x1']);
});
it('5: runs of commas are noise', () => {
expect(read('3472,,,3484,,3422, 0887').lines).toEqual(['3472x1', '3484x1', '3422x1', '0887x1']);
});
it('6: ending, space, quantity', () => {
expect(read('7784 1 7764 1 3367 2').lines).toEqual(['7784x1', '7764x1', '3367x2']);
});
it('7: x quantity in every form', () => {
expect(read('0057x2 0057 x 2 0057X2 0057×2').lines).toEqual(['0057x2', '0057x2', '0057x2', '0057x2']);
});
it('8: * quantity', () => {
expect(read('9155*40\n8203*100').lines).toEqual(['9155x40', '8203x100']);
});
it('9: bracket quantities with no separators', () => {
expect(read('0888(1)0003(1)9077(1)').lines).toEqual(['0888x1', '0003x1', '9077x1']);
});
it('10: dash plus 1-3 digits is a quantity', () => {
expect(read('0003-2').lines).toEqual(['0003x2']);
});
it('10a: = and _ quantities', () => {
expect(read('8910=3 0159_1').lines).toEqual(['8910x3', '0159x1']);
});
it('11: dash plus 4 digits is a range, qty 1 per ending', () => {
const r = parseOrder('5104-5106');
expect(r.lines).toEqual([
{ ending: '5104', qty: 1, line: 1, fromRange: true },
{ ending: '5105', qty: 1, line: 1, fromRange: true },
{ ending: '5106', qty: 1, line: 1, fromRange: true },
]);
});
it('12: a backwards range is unreadable', () => {
expect(read('5106-5104')).toMatchObject({ lines: [], unreadable: ['5106-5104:bad_range'] });
});
it('13: broken endings are never padded, truncated or corrected', () => {
expect(read('888 12345 O888 08?8 0888')).toMatchObject({
lines: ['0888x1'],
unreadable: ['888:stray_number', '12345:bad_ending_length', 'O888:mixed_characters', '08?8:unclear_digit'],
});
});
it('14: zero and oversized quantities are unreadable', () => {
expect(read('0057x0 0057x5000')).toMatchObject({
lines: [],
unreadable: ['0057x0:zero_quantity', '0057x5000:quantity_too_large'],
});
});
it('15: repeated endings are added up and listed', () => {
const r = parseOrder('0888 0888x3');
expect(r.lines).toHaveLength(2);
expect(r.totalTickets).toBe(4);
expect(r.repeated).toEqual([{ ending: '0888', occurrences: 2, totalQty: 4 }]);
});
it('16: totals are ignored, with no entry and no question', () => {
const r = read('Totel 54 number\nTotal..50\nTotal =20\nമൊത്തം 29\n250');
expect(r.lines).toEqual([]);
expect(r.unreadable).toEqual([]);
expect(r.ignored).toEqual(['Totel', '54', 'number', 'Total', '50', 'Total', '=20', 'മൊത്തം', '29', '250']);
});
it('matches the stated totals 54 and 250', () => {
const r = result();
expect(r.totals.find((t) => t.stated === 54)?.matched).toBe(true);
expect(r.totals.find((t) => t.stated === 250)?.matched).toBe(true);
it('17: WhatsApp prefix is stripped, name and time digits are not read', () => {
expect(read('[12/10, 14:35] Shop 2: 0888')).toEqual({ lines: ['0888x1'], unreadable: [], ignored: [], tickets: 1 });
});
it('finds the repeated endings', () => {
expect([...result().repeated].sort()).toEqual(['0003', '0057', '0888', '6678']);
it('18: greeting words are ignored, numbers still read', () => {
expect(read('Hi sir, order: 0888 0057x2')).toEqual({
lines: ['0888x1', '0057x2'],
unreadable: [],
ignored: ['Hi', 'sir', 'order:'],
tickets: 3,
});
});
it('reports 27 endings not covered by any total', () => {
expect(result().uncovered).toHaveLength(27);
it('20: a number after a quantity on an entry line is unreadable', () => {
expect(read('3367 2 250')).toMatchObject({ lines: ['3367x2'], unreadable: ['250:stray_number'] });
});
});
describe('parseOrder: output details', () => {
it('records the 1-based line of every entry', () => {
expect(parseOrder('0888\n\n0057x2').lines.map((l) => l.line)).toEqual([1, 3]);
});
it('mergeLines sums per ending in first-seen order', () => {
expect(mergeLines(parseOrder('0888 0057x2 0888x3').lines)).toEqual([
{ ending: '0888', qty: 4 },
{ ending: '0057', qty: 2 },
]);
});
it('treats a non-string as empty instead of throwing', () => {
expect(parseOrder(undefined as unknown as string).lines).toEqual([]);
});
});

View File

@ -1,5 +1,87 @@
import type { ParseResult } from './types';
import { lex } from './lexer';
import { MAX_INPUT_CHARS } from './limits';
import { normalize } from './normalize';
import { stripPrefix } from './prefix';
import { scanLine, type ScanItem } from './scan';
import type { Ignored, OrderLine, ParseResult, RepeatedEnding, Unreadable } from './types';
export function parseOrder(_text: string): ParseResult {
throw new Error('parseOrder is not implemented yet: port the tested parser into src/parser/parseOrder.ts');
export interface ScanResult {
bodies: string[];
items: ScanItem[];
}
export function scanOrder(text: string): ScanResult {
const bodies: string[] = [];
const items: ScanItem[] = [];
normalize(text).forEach((raw, index) => {
const { body } = stripPrefix(raw);
bodies.push(body);
items.push(...scanLine(body, lex(body), index + 1));
});
return { bodies, items };
}
const pad4 = (n: number): string => String(n).padStart(4, '0');
export function parseOrder(input: string): ParseResult {
const text = typeof input === 'string' ? input : '';
if (text.length > MAX_INPUT_CHARS) {
return {
lines: [],
totalTickets: 0,
ignored: [],
repeated: [],
unreadable: [{ text: `${text.slice(0, 40)}…`, line: 1, reason: 'input_too_long' }],
};
}
const lines: OrderLine[] = [];
const ignored: Ignored[] = [];
const unreadable: Unreadable[] = [];
for (const item of scanOrder(text).items) {
switch (item.kind) {
case 'entry':
lines.push({ ending: item.ending, qty: item.qty, line: item.line, fromRange: false });
break;
case 'range':
for (let e = Number(item.from); e <= Number(item.to); e++) {
lines.push({ ending: pad4(e), qty: 1, line: item.line, fromRange: true });
}
break;
case 'ignored':
ignored.push({ text: item.text, line: item.line, reason: item.reason });
break;
case 'unreadable':
unreadable.push({ text: item.text, line: item.line, reason: item.reason });
break;
}
}
return {
lines,
totalTickets: lines.reduce((sum, l) => sum + l.qty, 0),
ignored,
repeated: findRepeated(lines),
unreadable,
};
}
function findRepeated(lines: OrderLine[]): RepeatedEnding[] {
const byEnding = new Map<string, RepeatedEnding>();
for (const l of lines) {
const r = byEnding.get(l.ending);
if (r) {
r.occurrences++;
r.totalQty += l.qty;
} else {
byEnding.set(l.ending, { ending: l.ending, occurrences: 1, totalQty: l.qty });
}
}
return [...byEnding.values()].filter((r) => r.occurrences > 1);
}
export function mergeLines(lines: OrderLine[]): { ending: string; qty: number }[] {
const merged = new Map<string, number>();
for (const l of lines) merged.set(l.ending, (merged.get(l.ending) ?? 0) + l.qty);
return [...merged].map(([ending, qty]) => ({ ending, qty }));
}

17
src/parser/prefix.ts Normal file
View File

@ -0,0 +1,17 @@
const DATE = String.raw`\d{1,2}[/.-]\d{1,2}`;
const TIME = String.raw`\d{1,2}[:.]\d{2}(?:[:.]\d{2})?(?: ?[ap]\.?m\.?)?`;
const NAME = String.raw`[^:\n]{1,60}:`;
const APP_COPY = new RegExp(String.raw`^\[${DATE}(?:[/.-]\d{2,4})?,? ${TIME}\] ?${NAME} ?`, 'i');
const EXPORT = new RegExp(String.raw`^${DATE}[/.-]\d{2,4},? ${TIME} - ${NAME} ?`, 'i');
export interface Stripped {
body: string;
removed: number;
}
export function stripPrefix(line: string): Stripped {
const m = APP_COPY.exec(line) ?? EXPORT.exec(line);
if (!m) return { body: line, removed: 0 };
return { body: line.slice(m[0].length), removed: m[0].length };
}

221
src/parser/scan.ts Normal file
View File

@ -0,0 +1,221 @@
import { MAX_QTY, MAX_RANGE } from './limits';
import type { Atom } from './lexer';
import type { UnreadableReason } from './types';
export type ScanItem =
| { kind: 'entry'; ending: string; qty: number; text: string; line: number }
| { kind: 'range'; from: string; to: string; text: string; line: number }
| { kind: 'ignored'; reason: 'text' | 'lone_number'; text: string; line: number }
| { kind: 'unreadable'; reason: UnreadableReason; text: string; line: number };
interface Loose {
kind: 'loose';
reason: 'stray_number' | 'orphan_quantity';
text: string;
line: number;
}
type Item = ScanItem | Loose;
const QTY_OPS = new Set(['x', '*', '=', '_']);
const CONTINUATION_OPS = new Set(['x', '*', '=', '_', '(', '-']);
type Step = { ok: true; item: ScanItem; next: number } | { ok: false; reason: UnreadableReason; next: number };
export function scanLine(body: string, atoms: Atom[], line: number): ScanItem[] {
const n = atoms.length;
const at = (k: number): Atom | undefined => atoms[k];
const isOp = (a: Atom | undefined, ...ops: string[]): boolean =>
a !== undefined && a.kind === 'op' && ops.includes(a.text);
const isDigits = (a: Atom | undefined): boolean => a !== undefined && (a.kind === 'num' || a.kind === 'baddigit');
const slice = (from: number, toExclusive: number): string =>
body.slice((atoms[from] as Atom).start, (atoms[toExclusive - 1] as Atom).end);
const isNoiseDot = (k: number): boolean => {
const a = at(k);
if (!isOp(a, '.')) return false;
const next = at(k + 1);
const decimal =
isDigits(at(k - 1)) && (a as Atom).glued && isDigits(next) && (next as Atom).glued && (next as Atom).text.length <= 3;
return !decimal;
};
const clusterEnd = (k: number): number => {
let j = k + 1;
while (j < n && (atoms[j] as Atom).glued && (atoms[j] as Atom).kind !== 'sep' && !isNoiseDot(j)) j++;
return j;
};
const hasDigits = (from: number, to: number): boolean => {
for (let k = from; k < to; k++) if (isDigits(atoms[k])) return true;
return false;
};
const subclusterEnd = (k: number, end: number): number => {
let j = k + 1;
while (j < end) {
const a = atoms[j - 1] as Atom;
const b = atoms[j] as Atom;
if ((a.kind === 'word' && b.kind === 'op') || (a.kind === 'op' && b.kind === 'word')) break;
j++;
}
return j;
};
const quantity = (digits: string): { ok: true; qty: number } | { ok: false; reason: UnreadableReason } => {
if (digits.length > 3) return { ok: false, reason: 'quantity_too_large' };
const qty = Number(digits);
if (qty === 0) return { ok: false, reason: 'zero_quantity' };
if (qty > MAX_QTY) return { ok: false, reason: 'quantity_too_large' };
return { ok: true, qty };
};
const withQty = (i: number, digits: string, next: number): Step => {
const q = quantity(digits);
if (!q.ok) return { ok: false, reason: q.reason, next };
const ending = (atoms[i] as Atom).text;
return { ok: true, item: { kind: 'entry', ending, qty: q.qty, text: slice(i, next), line }, next };
};
const single = (i: number): Step => ({
ok: true,
item: { kind: 'entry', ending: (atoms[i] as Atom).text, qty: 1, text: slice(i, i + 1), line },
next: i + 1,
});
const parseEntry = (i: number): Step => {
const k = i + 1;
const a = at(k);
const b = at(k + 1);
if (isOp(a, '-')) {
const gluedBoth = (a as Atom).glued && b !== undefined && b.glued;
if (b !== undefined && b.kind === 'num') {
if (!gluedBoth) return { ok: false, reason: 'bad_range', next: k + 2 };
if (b.text.length === 4) {
const from = Number((atoms[i] as Atom).text);
const to = Number(b.text);
if (to < from) return { ok: false, reason: 'bad_range', next: k + 2 };
if (to - from + 1 > MAX_RANGE) return { ok: false, reason: 'range_too_large', next: k + 2 };
const item: ScanItem = { kind: 'range', from: (atoms[i] as Atom).text, to: b.text, text: slice(i, k + 2), line };
return { ok: true, item, next: k + 2 };
}
if (b.text.length <= 3) return withQty(i, b.text, k + 2);
return { ok: false, reason: 'bad_ending_length', next: k + 2 };
}
if ((a as Atom).glued) return { ok: false, reason: 'mixed_characters', next: k + 1 };
return single(i);
}
if (a !== undefined && a.kind === 'op' && QTY_OPS.has(a.text)) {
if (b !== undefined && b.kind === 'num') return withQty(i, b.text, k + 2);
if (a.glued) return { ok: false, reason: 'mixed_characters', next: k + 1 };
return single(i);
}
if (isOp(a, '(')) {
if (b !== undefined && b.kind === 'num' && isOp(at(k + 2), ')')) return withQty(i, b.text, k + 3);
if ((a as Atom).glued) return { ok: false, reason: 'mixed_characters', next: k + 1 };
return single(i);
}
if (a !== undefined && a.kind === 'num' && !a.glued && a.text.length <= 3) {
const after = at(k + 1);
const free = after === undefined || !after.glued || after.kind === 'sep' || isNoiseDot(k + 1);
if (free) return withQty(i, a.text, k + 1);
}
return single(i);
};
const continuationReason = (item: ScanItem, next: Atom): UnreadableReason => {
if (next.kind === 'op' && CONTINUATION_OPS.has(next.text)) {
return item.kind === 'range' ? 'bad_range' : 'double_quantity';
}
if (next.kind === 'op' && (next.text === '/' || next.text === ':')) return 'date_or_time';
if (next.kind === 'baddigit') return 'unsupported_digits';
if (next.kind === 'word' && /^[?xX]+$/.test(next.text)) return 'unclear_digit';
return 'mixed_characters';
};
const classify = (from: number, to: number): Item => {
const text = slice(from, to);
const part = atoms.slice(from, to);
if (part.some((a) => a.kind === 'baddigit')) return { kind: 'unreadable', reason: 'unsupported_digits', text, line };
const words = part.filter((a) => a.kind === 'word');
if (words.length > 0) {
const unclear = words.every((w) => /^[?xX]+$/.test(w.text));
return { kind: 'unreadable', reason: unclear ? 'unclear_digit' : 'mixed_characters', text, line };
}
if (part.some((a) => a.kind === 'op' && (a.text === '/' || a.text === ':'))) {
return { kind: 'unreadable', reason: 'date_or_time', text, line };
}
const nums = part.filter((a) => a.kind === 'num');
if (nums.some((a) => a.text.length >= 5)) return { kind: 'unreadable', reason: 'bad_ending_length', text, line };
if (nums.length === 1) {
const num = nums[0] as Atom;
if (part.length === 1) {
return num.text.length <= 3 ? { kind: 'loose', reason: 'stray_number', text, line } : { kind: 'unreadable', reason: 'mixed_characters', text, line };
}
const [first, second, third] = part;
const opThenNum =
part.length === 2 && first !== undefined && first.kind === 'op' && (QTY_OPS.has(first.text) || first.text === '-') && second === num;
const bracketed = part.length === 3 && isOp(first, '(') && second === num && isOp(third, ')');
if (num.text.length <= 3 && (opThenNum || bracketed)) {
return { kind: 'loose', reason: 'orphan_quantity', text, line };
}
}
return { kind: 'unreadable', reason: 'mixed_characters', text, line };
};
const items: Item[] = [];
let entryLine = false;
let i = 0;
while (i < n) {
const a = atoms[i] as Atom;
if (a.kind === 'sep' || isNoiseDot(i)) {
i++;
continue;
}
if (a.kind === 'num' && a.text.length === 4) {
entryLine = true;
const step = parseEntry(i);
const next = at(step.next);
const prev = at(step.next - 1);
const glued = next !== undefined && next.glued && next.kind !== 'sep' && !isNoiseDot(step.next);
const chain = glued && isOp(prev, ')') && next.kind === 'num' && next.text.length === 4;
const wordAfterOp = glued && next.kind === 'word' && prev !== undefined && prev.kind === 'op';
if (step.ok && (!glued || chain || wordAfterOp)) {
items.push(step.item);
i = step.next;
continue;
}
const reason = step.ok ? continuationReason(step.item, next as Atom) : step.reason;
const end = step.next < n && glued ? clusterEnd(step.next) : Math.max(step.next, i + 1);
items.push({ kind: 'unreadable', reason, text: slice(i, end), line });
i = end;
continue;
}
const end = clusterEnd(i);
if (!hasDigits(i, end)) {
items.push({ kind: 'ignored', reason: 'text', text: slice(i, end), line });
i = end;
continue;
}
for (let s = i; s < end; ) {
const sub = subclusterEnd(s, end);
if (!hasDigits(s, sub)) items.push({ kind: 'ignored', reason: 'text', text: slice(s, sub), line });
else items.push(classify(s, sub));
s = sub;
}
i = end;
}
return items.map((it): ScanItem => {
if (it.kind !== 'loose') return it;
if (entryLine) return { kind: 'unreadable', reason: it.reason, text: it.text, line: it.line };
return { kind: 'ignored', reason: 'lone_number', text: it.text, line: it.line };
});
}

View File

@ -1,28 +1,60 @@
export interface OrderLine {
/** The last 4 digits, as a string so leading zeros survive ("0057"). */
ending: string;
/** How many tickets. Always at least 1. */
/** How many tickets. Always 1 to MAX_QTY. */
qty: number;
/** 1-based input line it came from. */
line: number;
/** True when expanded from a range such as 5104-5106. */
fromRange: boolean;
}
/** A total the sender wrote, such as "Totel 54 number" or a bare "250" on its own line. */
export interface TotalCheck {
/** The number the sender wrote. */
stated: number;
/** True if the tickets before that total add up to it (by ticket count or by line count). */
matched: boolean;
export interface Ignored {
/** The exact characters skipped. */
text: string;
line: number;
/** `text`: no digits at all. `lone_number`: a 1-3 digit number on a line with no entry (usually a total). */
reason: 'text' | 'lone_number';
}
export type UnreadableReason =
| 'bad_ending_length' // 5 or more digits (could be a full ticket number)
| 'stray_number' // 1-3 digits on a line with entries, not a quantity
| 'mixed_characters' // digits glued to letters or symbols (O888, 0888a, 0888.5)
| 'unclear_digit' // digits glued to ? or x (08?8, 0xx8)
| 'date_or_time' // 12/10/2026, 14:35
| 'zero_quantity'
| 'quantity_too_large'
| 'double_quantity' // 0057x2x3
| 'orphan_quantity' // x2 or (3) with no ending before it, on a line with entries
| 'bad_range' // backwards range, spaced dash, range with a quantity
| 'range_too_large'
| 'unsupported_digits' // Malayalam or other non-ASCII digits
| 'input_too_long';
/** Something containing digits that the parser will not guess about. The bot asks the owner. */
export interface Unreadable {
/** The exact characters, for the bot to show. */
text: string;
line: number;
reason: UnreadableReason;
}
export interface RepeatedEnding {
ending: string;
occurrences: number;
/** Sum of qty over all occurrences (repeats are added up). */
totalQty: number;
}
export interface ParseResult {
/** Every occurrence, in input order. */
lines: OrderLine[];
/** Sum of qty over all lines. */
totalTickets: number;
totals: TotalCheck[];
/** Endings that no stated total covers. The bot lists them and asks. */
uncovered: string[];
/** Endings that appear more than once. The bot asks whether to add them up. */
repeated: string[];
/** Text the parser could not read. It is never guessed: the bot asks. */
unreadable: string[];
/** Words and lone numbers skipped. Informational only. */
ignored: Ignored[];
/** Endings that occur more than once, in order of first occurrence. Shown, not asked about. */
repeated: RepeatedEnding[];
unreadable: Unreadable[];
}

View File

@ -1,13 +1,21 @@
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import { describe, expect, it } from 'vitest';
import { makeDb } from '../testing/helpers';
import { hasPdftotext, makeDb } from '../testing/helpers';
import { extractPdfText } from './pdfText';
import { processPdf, runResultsJob } from './job';
import { parseResult, validateDraw } from './parseResult';
import { parsePortalListing, selectRecent } from './portal';
import { storeDraw } from './store';
if (!hasPdftotext) {
console.warn(
'[job.test] pdftotext is not installed: tests that read the real PDF are SKIPPED. ' +
'Install poppler (conda install -c conda-forge poppler, or sudo apt install poppler-utils) and re-run.',
);
}
const withPdftotext = it.skipIf(!hasPdftotext);
const FX = join(__dirname, '../../test/fixtures/results');
const layout = (n: string) => readFileSync(join(FX, n), 'utf8');
const sk73Pdf = readFileSync(join(FX, 'sk73_2026-10-16.pdf'));
@ -76,19 +84,23 @@ describe('storeDraw', () => {
});
describe('processPdf (real pdftotext on the real SK-73 PDF)', () => {
it('parses and validates clean', async () => {
withPdftotext('parses and validates clean', async () => {
const { draw, issues } = await processPdf(sk73Pdf);
expect(draw?.schemeCode).toBe('SK');
expect(issues).toEqual([]);
});
it('catches a PDF that is not the draw the listing claims', async () => {
withPdftotext('catches a PDF that is not the draw the listing claims', async () => {
const entry = { name: 'x', code: 'KN', drawNo: 645, date: '2026-10-15', url: 'u#1' };
const { issues } = await processPdf(sk73Pdf, entry);
expect(issues.some((i) => i.message.includes('listing says'))).toBe(true);
});
it('reports an unreadable PDF instead of throwing', async () => {
it('says how to fix a missing pdftotext instead of failing silently', async () => {
await expect(extractPdfText(sk73Pdf, 'pdftotext-not-installed-xyz')).rejects.toThrow(/install poppler-utils/);
});
withPdftotext('reports an unreadable PDF instead of throwing', async () => {
const { draw, issues } = await processPdf(Buffer.from('not a pdf'), undefined, extractPdfText);
expect(draw).toBeNull();
expect(issues[0]?.level).toBe('ERROR');
@ -99,7 +111,7 @@ describe('runResultsJob', () => {
const fetchImpl = (async () => new Response(LISTING)) as typeof fetch;
const base = { today: '2026-10-16', days: 3, fetchImpl };
it('stores a good draw, holds a mismatched one, reports a failed fetch, skips ready ones on rerun', async () => {
withPdftotext('stores a good draw, holds a mismatched one, reports a failed fetch, skips ready ones on rerun', async () => {
const db = makeDb();
const getPdf = async (e: { code: string }) => {
if (e.code === 'SK') return sk73Pdf;

View File

@ -1,3 +1,4 @@
import { spawnSync } from 'node:child_process';
import type { Db } from '../db';
import { openDb } from '../db';
import { runMigrations } from '../db/migrate';
@ -40,3 +41,8 @@ export function seedFixture(db: Db): Fixture {
return { shops, admin, admin2, shopUser, disabledUser };
}
export const hasPdftotext: boolean = (() => {
const r = spawnSync('pdftotext', ['-v'], { encoding: 'utf8' });
return !r.error;
})();

33
src/testing/parser.ts Normal file
View File

@ -0,0 +1,33 @@
import { parseOrder } from '../parser/parseOrder';
import type { ParseResult } from '../parser/types';
export interface Read {
lines: string[];
unreadable: string[];
ignored: string[];
tickets: number;
}
export function compact(r: ParseResult): Read {
return {
lines: r.lines.map((l) => `${l.ending}x${l.qty}`),
unreadable: r.unreadable.map((u) => `${u.text}:${u.reason}`),
ignored: r.ignored.map((i) => i.text),
tickets: r.totalTickets,
};
}
export function read(text: string): Read {
return compact(parseOrder(text));
}
/** Small deterministic PRNG (LCG) so generated tests reproduce from a seed. */
export function rng(seed: number): { int: (maxExclusive: number) => number; pick: <T>(xs: readonly T[]) => T } {
let s = seed >>> 0 || 1;
const next = (): number => {
s = (Math.imul(s, 1664525) + 1013904223) >>> 0;
return s / 0x100000000;
};
const int = (maxExclusive: number): number => Math.floor(next() * maxExclusive);
return { int, pick: <T>(xs: readonly T[]): T => xs[int(xs.length)] as T };
}

View File

@ -0,0 +1,134 @@
{
"totalTickets": 115,
"lines": [
"8910 x3 @1",
"7810 x1 @2",
"0129 x1 @3",
"0189 x1 @4",
"7086 x1 @5",
"1155 x2 @6",
"7569 x1 @7",
"1825 x2 @8",
"9081 x1 @9",
"9760 x1 @10",
"8891 x1 @11",
"8069 x2 @12",
"7081 x2 @13",
"2130 x2 @14",
"3195 x1 @15",
"1739 x1 @16",
"8960 x2 @17",
"7890 x2 @18",
"9310 x2 @19",
"1874 x1 @20",
"3417 x1 @21",
"5584 x1 @22",
"0019 x1 @23",
"3591 x1 @24",
"1239 x1 @25",
"1098 x2 @26",
"7815 x1 @27",
"0719 x1 @28",
"7315 x1 @29",
"9724 x1 @30",
"7310 x1 @31",
"6250 x1 @32",
"0102 x1 @33",
"1598 x1 @34",
"9815 x1 @35",
"8951 x1 @36",
"2055 x1 @37",
"8491 x1 @38",
"5173 x1 @39",
"3175 x1 @42",
"0072 x2 @43",
"0207 x1 @44",
"0702 x1 @45",
"0754 x1 @46",
"7145 x1 @47",
"1657 x1 @48",
"0848 x1 @49",
"5393 x1 @50",
"0159 x1 @53",
"9015 x1 @54",
"7502 x1 @55",
"1348 x1 @56",
"8670 x1 @57",
"2755 x1 @58",
"4406 x1 @61",
"6044 x1 @61",
"4460 x1 @61",
"1113 x1 @61",
"4604 x1 @62",
"6440 x1 @62",
"0464 x1 @62",
"0446 x1 @62",
"0644 x1 @63",
"4046 x1 @63",
"4640 x1 @63",
"4064 x1 @63",
"1775 x1 @64",
"3127 x1 @64",
"2070 x1 @64",
"9415 x1 @64",
"9516 x1 @65",
"1131 x1 @65",
"1311 x1 @65",
"3111 x1 @65",
"7740 x2 @69",
"7741 x2 @70",
"7742 x1 @71",
"7743 x1 @72",
"7744 x1 @73",
"7745 x1 @74",
"7746 x1 @75",
"7747 x1 @76",
"7748 x1 @77",
"7749 x2 @78",
"0810 x1 @79",
"0811 x2 @80",
"0812 x2 @81",
"0813 x1 @82",
"0814 x1 @83",
"0815 x2 @84",
"0816 x1 @85",
"0817 x2 @86",
"0818 x1 @87",
"0819 x2 @88",
"9086 x1 @89"
],
"repeated": [],
"unreadable": [],
"ignored": [
{
"text": "Total",
"line": 40,
"reason": "text"
},
{
"text": "50",
"line": 40,
"reason": "lone_number"
},
{
"text": "Total",
"line": 66,
"reason": "text"
},
{
"text": "=20",
"line": 66,
"reason": "lone_number"
},
{
"text": "മൊത്തം",
"line": 90,
"reason": "text"
},
{
"text": "29",
"line": 90,
"reason": "lone_number"
}
]
}

View File

@ -0,0 +1,90 @@
8910=3
7810=1
0129=1
0189=1
7086=1
1155=2
7569=1
1825=2
9081=1
9760=1
8891=1
8069=2
7081=2
2130=2
3195=1
1739=1
8960=2
7890=2
9310=2
1874=1
3417=1
5584=1
0019=1
3591=1
1239=1
1098=2
7815=1
0719=1
7315=1
9724=1
7310=1
6250=1
0102=1
1598=1
9815=1
8951=1
2055=1
8491=1
5173=1
Total..50
3175-1
0072-2
0207-1
0702=1
0754-1
7145-1
1657-1
0848-1
5393-1
0159_1
9015_1
7502_1
1348_1
8670_1
2755_1
4406,6044,4460,1113
4604,6440,0464,0446
0644,4046,4640,4064
1775,3127,2070,9415
9516,1131,1311,3111
Total =20
7740(2)
7741(2)
7742(1)
7743(1)
7744(1)
7745(1)
7746(1)
7747(1)
7748(1)
7749(2)
0810(1)
0811(2)
0812(2)
0813(1)
0814(1)
0815(2)
0816(1)
0817(2)
0818(1)
0819(2)
9086(1)
മൊത്തം 29