From ed4d1280e1180c987b0cdf4a5acf76a82f295688 Mon Sep 17 00:00:00 2001 From: Aswin Biju Date: Tue, 3 Nov 2026 12:53:26 +0530 Subject: [PATCH] feat(parser): built the order parser --- src/parser/corpus.test.ts | 30 ++ src/parser/edgeCases.test.ts | 277 ++++++++++++++++++ src/parser/generated.test.ts | 58 ++++ src/parser/invariants.test.ts | 84 ++++++ src/parser/lexer.ts | 62 ++++ src/parser/limits.test.ts | 46 +++ src/parser/limits.ts | 8 + src/parser/normalize.ts | 17 ++ src/parser/parseOrder.test.ts | 141 +++++++-- src/parser/parseOrder.ts | 88 +++++- src/parser/prefix.ts | 17 ++ src/parser/scan.ts | 221 ++++++++++++++ src/parser/types.ts | 62 +++- src/results/job.test.ts | 22 +- src/testing/helpers.ts | 6 + src/testing/parser.ts | 33 +++ .../orders/owner-2026-11-03.expected.json | 134 +++++++++ test/fixtures/orders/owner-2026-11-03.txt | 90 ++++++ 18 files changed, 1345 insertions(+), 51 deletions(-) create mode 100644 src/parser/corpus.test.ts create mode 100644 src/parser/edgeCases.test.ts create mode 100644 src/parser/generated.test.ts create mode 100644 src/parser/invariants.test.ts create mode 100644 src/parser/lexer.ts create mode 100644 src/parser/limits.test.ts create mode 100644 src/parser/limits.ts create mode 100644 src/parser/normalize.ts create mode 100644 src/parser/prefix.ts create mode 100644 src/parser/scan.ts create mode 100644 src/testing/parser.ts create mode 100644 test/fixtures/orders/owner-2026-11-03.expected.json create mode 100644 test/fixtures/orders/owner-2026-11-03.txt diff --git a/src/parser/corpus.test.ts b/src/parser/corpus.test.ts new file mode 100644 index 0000000..3776978 --- /dev/null +++ b/src/parser/corpus.test.ts @@ -0,0 +1,30 @@ +import { readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { parseOrder } from './parseOrder'; + +const DIR = join(__dirname, '../../test/fixtures/orders'); +const names = readdirSync(DIR) + .filter((f) => f.endsWith('.txt')) + .map((f) => f.slice(0, -4)); + +describe('regression corpus', () => { + it('has at least one fixture', () => { + expect(names.length).toBeGreaterThan(0); + }); + + for (const name of names) { + it(name, () => { + const r = parseOrder(readFileSync(join(DIR, `${name}.txt`), 'utf8')); + const actual = { + totalTickets: r.totalTickets, + lines: r.lines.map((l) => `${l.ending} x${l.qty}${l.fromRange ? ' range' : ''} @${l.line}`), + repeated: r.repeated, + unreadable: r.unreadable, + ignored: r.ignored, + }; + const expected = JSON.parse(readFileSync(join(DIR, `${name}.expected.json`), 'utf8')); + expect(actual).toEqual(expected); + }); + } +}); diff --git a/src/parser/edgeCases.test.ts b/src/parser/edgeCases.test.ts new file mode 100644 index 0000000..a72288f --- /dev/null +++ b/src/parser/edgeCases.test.ts @@ -0,0 +1,277 @@ +import { describe, expect, it } from 'vitest'; +import { read } from '../testing/parser'; +import { parseOrder } from './parseOrder'; + +describe('6.1 input hygiene', () => { + it('EC-02 CRLF, CR and mixed line endings read like LF', () => { + const lf = parseOrder('0888\n0057x2\n0003'); + for (const t of ['0888\r\n0057x2\r\n0003', '0888\r0057x2\r0003', '0888\r\n0057x2\r0003']) { + expect(parseOrder(t)).toEqual(lf); + } + }); + + it('EC-03 invisible marks are removed, line numbers unchanged', () => { + const r = parseOrder('‎0888\n08​88 0057'); + expect(r.lines.map((l) => [l.ending, l.line])).toEqual([['0888', 1], ['0888', 2], ['0057', 2]]); + }); + + it('EC-04 tabs and runs of spaces are noise', () => { + expect(read('0888\t\t0057 0003').lines).toEqual(['0888x1', '0057x1', '0003x1']); + }); + + it('EC-05 full-width digits become ASCII', () => { + expect(read('0888').lines).toEqual(['0888x1']); + }); + + it('EC-06 Malayalam or Arabic-Indic digits are unreadable', () => { + expect(read('൦൮൮൮ ٠٨٨٨').unreadable).toEqual(['൦൮൮൮:unsupported_digits', '٠٨٨٨:unsupported_digits']); + }); + + it('EC-07 input above the limit is reported, not read', () => { + const r = parseOrder('0888 '.repeat(30_000)); + expect(r.lines).toEqual([]); + expect(r.unreadable[0]?.reason).toBe('input_too_long'); + }); +}); + +describe('6.2 prefixes', () => { + it('EC-10 app copy prefix', () => { + expect(read('[12/10, 14:35] Ravi: 0888').lines).toEqual(['0888x1']); + }); + + it('EC-11 exported prefix', () => { + expect(read('12/10/2026, 2:35 pm - Ravi: 0888').lines).toEqual(['0888x1']); + expect(read('12/10/26, 14:35 - Ravi: 0057x2').lines).toEqual(['0057x2']); + }); + + it('EC-12 a name with digits is removed completely', () => { + expect(read('[12/10, 14:35] Shop 2 Main: 0888')).toMatchObject({ lines: ['0888x1'], unreadable: [], ignored: [] }); + }); + + it('EC-13 a prefix on every line of a merged forward', () => { + expect(read('[12/10, 14:35] A: 0888\n[12/10, 14:36] B: 0057x2').lines).toEqual(['0888x1', '0057x2']); + }); + + it('EC-14 a line that only looks like a prefix is not stripped', () => { + expect(read('12/10 0888')).toMatchObject({ lines: ['0888x1'], unreadable: ['12/10:date_or_time'] }); + }); + + it('EC-15 prefix only', () => { + expect(read('[12/10, 14:35] Ravi:')).toEqual({ lines: [], unreadable: [], ignored: [], tickets: 0 }); + }); + + it('EC-16 seconds and am/pm in the time are stripped, never read as 1435', () => { + expect(read('[12/10/26, 2:35:20 pm] Ravi: 0888').lines).toEqual(['0888x1']); + expect(read('[12/10, 14:35:20] Ravi: 0888').lines).toEqual(['0888x1']); + }); +}); + +describe('6.3 endings', () => { + it('EC-21 leading zeros survive', () => { + expect(parseOrder('0007 0000').lines.map((l) => l.ending)).toEqual(['0007', '0000']); + }); + + it('EC-22 1-3 digits on an entry line are unreadable, never padded', () => { + expect(read('0888 2 888 0003').unreadable).toEqual(['888:stray_number']); + expect(read('7 0888').unreadable).toEqual(['7:stray_number']); + }); + + it('EC-23 5+ digits are unreadable, never truncated', () => { + expect(read('12345 404595').unreadable).toEqual(['12345:bad_ending_length', '404595:bad_ending_length']); + }); + + it('EC-24 letters mixed in are unreadable', () => { + expect(read('O888 08S8 0888').unreadable).toEqual(['O888:mixed_characters', '08S8:mixed_characters']); + // 0 right after an ending is a (zero) quantity by the space rule; 888 is still reported, not ignored. + expect(read('0888 0 888').unreadable).toEqual(['0888 0:zero_quantity', '888:stray_number']); + expect(read('0 888')).toMatchObject({ unreadable: [], ignored: ['0', '888'] }); // known limit: alone on a line + }); + + it('EC-25 ? or x as an unclear digit', () => { + expect(read('08?8 0xx8 0888?').unreadable).toEqual(['08?8:unclear_digit', '0xx8:unclear_digit', '0888?:unclear_digit']); + }); + + it('EC-26 endings glued together are not split', () => { + expect(read('08880003').unreadable).toEqual(['08880003:bad_ending_length']); + }); + + it('EC-28 , ; . | are noise; / is not', () => { + expect(read('0888;0057.0003|3472').lines).toEqual(['0888x1', '0057x1', '0003x1', '3472x1']); + expect(read('0888/0057').unreadable).toEqual(['0888/0057:date_or_time']); + }); +}); + +describe('6.4 quantities', () => { + it('EC-30 x and * in every spacing', () => { + expect(read('0057x2 0057 x 2 0057X2 0057×2 0057*2 0057 * 2').lines).toEqual(Array(6).fill('0057x2')); + }); + + it('EC-32 bracket with spaces', () => { + expect(read('0888 (2) 0888( 2 )').lines).toEqual(['0888x2', '0888x2']); + }); + + it('EC-33 dash quantity of 1 to 3 digits', () => { + expect(read('0003-2 0003-12 0003-123').lines).toEqual(['0003x2', '0003x12', '0003x123']); + }); + + it('EC-34 zero quantity in every form', () => { + expect(read('0003-0 0057x0 0888(0) 8910=0').unreadable.map((u) => u.split(':')[1])).toEqual(Array(4).fill('zero_quantity')); + }); + + it('EC-35 quantity above the limit', () => { + expect(read('0057x5000 0057*1000').unreadable).toEqual(['0057x5000:quantity_too_large', '0057*1000:quantity_too_large']); + }); + + it('EC-36 a quantity with no ending: unreadable on an entry line, ignored alone', () => { + expect(read('0888 , x2').unreadable).toEqual(['x2:orphan_quantity']); + expect(read('0888\n(3)')).toMatchObject({ lines: ['0888x1'], unreadable: [], ignored: ['(3)'] }); + }); + + it('EC-37 two quantities are not multiplied', () => { + expect(read('0057x2x3 0057x2(3)').unreadable).toEqual(['0057x2x3:double_quantity', '0057x2(3):double_quantity']); + }); + + it('EC-38 decimal or negative quantities', () => { + expect(read('0057x2.5 0057x-1').unreadable).toEqual(['0057x2.5:mixed_characters', '0057x-1:mixed_characters']); + }); + + it('EC-39 A,B,C-2 binds the quantity to C only (provisional, Q3)', () => { + expect(read('0888,0057,0003-2').lines).toEqual(['0888x1', '0057x1', '0003x2']); + }); + + it('EC-39c space quantity is 1 to 3 digits; 4 digits is the next ending', () => { + expect(read('7784 1 7764 12 3367 123').lines).toEqual(['7784x1', '7764x12', '3367x123']); + expect(read('7784 1234').lines).toEqual(['7784x1', '1234x1']); + }); + + it('EC-39e a quantity on the next line is not a quantity', () => { + expect(read('7784\n1')).toMatchObject({ lines: ['7784x1'], ignored: ['1'] }); + }); + + it('EC-39f = and _ in every spacing', () => { + expect(read('8910=3 8910 = 3 0159_1 0159 _ 1').lines).toEqual(['8910x3', '8910x3', '0159x1', '0159x1']); + }); + + it('EC-39h 4 digits after = or _ is never a quantity or an ending', () => { + expect(read('8910=1234 0159_1234').unreadable).toEqual(['8910=1234:quantity_too_large', '0159_1234:quantity_too_large']); + }); + + it('EC-39i separators mixed in one order', () => { + expect(read('3175-1\n0702=1\n0159_1\n7740(2)').lines).toEqual(['3175x1', '0702x1', '0159x1', '7740x2']); + }); +}); + +describe('6.5 ranges', () => { + it('EC-41 single-ending range', () => { + expect(read('5104-5104').lines).toEqual(['5104x1']); + }); + + it('EC-43 no wrap-around', () => { + expect(read('9998-0001').unreadable).toEqual(['9998-0001:bad_range']); + }); + + it('EC-44 range too large', () => { + expect(read('0000-9999').unreadable).toEqual(['0000-9999:range_too_large']); + expect(read('0000-0099').lines).toHaveLength(100); + expect(read('0000-0100').unreadable).toEqual(['0000-0100:range_too_large']); + }); + + it('EC-45 crossing a hundred keeps zero padding', () => { + expect(read('0099-0101').lines).toEqual(['0099x1', '0100x1', '0101x1']); + }); + + it('EC-46 a range with a quantity is unreadable (provisional, Q5)', () => { + expect(read('5104-5106x2').unreadable).toEqual(['5104-5106x2:bad_range']); + }); + + it('EC-47 a spaced dash between numbers is unreadable, so 5105 is never lost', () => { + expect(read('5104 - 5106').unreadable).toEqual(['5104 - 5106:bad_range']); + expect(read('5104 -5106').unreadable).toEqual(['5104 -5106:bad_range']); + }); + + it('EC-48 dash quantity followed by a range', () => { + expect(read('0003-2-5104').unreadable).toEqual(['0003-2-5104:double_quantity']); + }); +}); + +describe('6.6 totals and ignored text', () => { + it('EC-50 every total form is ignored', () => { + for (const t of ['Totel 54 number', 'Total 54', 'total: 54', '54 total', 'Tot-54', 'Total..50', 'Total =20', 'മൊത്തം 29']) { + expect(read(t), t).toMatchObject({ lines: [], unreadable: [] }); + } + }); + + it('EC-51 bare numbers on their own line are ignored', () => { + expect(read('0888\n250')).toMatchObject({ lines: ['0888x1'], unreadable: [], ignored: ['250'] }); + }); + + it('EC-63 a keyword with no number', () => { + expect(read('Total')).toMatchObject({ unreadable: [], ignored: ['Total'] }); + }); + + it('EC-64 split digits are never joined', () => { + expect(read('Total 5 4')).toMatchObject({ unreadable: [], ignored: ['Total', '5', '4'] }); + }); + + it('EC-65 known limit: a 4-digit total is read as an ending (Q8)', () => { + expect(read('Total 1200').lines).toEqual(['1200x1']); + }); + + it('EC-66 a total on an entry line is unreadable (Q9)', () => { + expect(read('0888 0057 Total 2')).toMatchObject({ lines: ['0888x1', '0057x1'], unreadable: ['2:stray_number'] }); + }); +}); + +describe('6.7 repeats', () => { + it('EC-70 two occurrences are both counted', () => { + const r = parseOrder('0888\n0057\n0888'); + expect(r.totalTickets).toBe(3); + expect(r.repeated).toEqual([{ ending: '0888', occurrences: 2, totalQty: 2 }]); + }); + + it('EC-72 a repeat inside a range', () => { + expect(parseOrder('0888 0887-0889').repeated).toEqual([{ ending: '0888', occurrences: 2, totalQty: 2 }]); + }); +}); + +describe('6.8 mixed, merged and noisy input', () => { + it('EC-81 other-script words are ignored', () => { + expect(read('ഇന്നത്തെ 0888')).toMatchObject({ lines: ['0888x1'], unreadable: [] }); + }); + + it('EC-82 emoji are ignored', () => { + expect(read('0888 ✅ 👍')).toMatchObject({ lines: ['0888x1'], unreadable: [], ignored: ['✅', '👍'] }); + }); + + it('EC-83 the same order twice shows as repeats', () => { + expect(parseOrder('0888 0057x2\n0888 0057x2').repeated.map((r) => r.ending)).toEqual(['0888', '0057']); + }); + + it('EC-84 merged messages with prefixes and totals', () => { + const t = '[12/10, 14:35] A: 0888 0057x2\nTotal 3\n\n[12/10, 14:40] A: 0003-2\n250'; + expect(read(t)).toMatchObject({ lines: ['0888x1', '0057x2', '0003x2'], unreadable: [] }); + }); + + it('EC-85 only a total', () => { + expect(read('250')).toEqual({ lines: [], unreadable: [], ignored: ['250'], tickets: 0 }); + }); + + it('EC-86 digits inside words', () => { + expect(read('ab0888cd 0888a').unreadable).toEqual(['ab0888cd:mixed_characters', '0888a:mixed_characters']); + }); + + it('EC-87 decimals and thousands separators', () => { + expect(read('0888.5 0057').unreadable).toEqual(['0888.5:mixed_characters']); + // 1 right after an ending is its quantity (space rule); 200 is reported, never joined into 1200. + expect(read('0057 1,200')).toMatchObject({ lines: ['0057x1'], unreadable: ['200:stray_number'] }); + expect(read('1,200')).toMatchObject({ lines: [], ignored: ['1', '200'] }); + }); + + it('EC-88 dates and times inside the body', () => { + expect(read('0888 12/10/2026 14:35').unreadable).toEqual(['12/10/2026:date_or_time', '14:35:date_or_time']); + }); + + it('a word right after a bracket quantity is separate text', () => { + expect(read('0888(1)Ravi')).toMatchObject({ lines: ['0888x1'], unreadable: [], ignored: ['Ravi'] }); + }); +}); diff --git a/src/parser/generated.test.ts b/src/parser/generated.test.ts new file mode 100644 index 0000000..d053b3b --- /dev/null +++ b/src/parser/generated.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from 'vitest'; +import { rng } from '../testing/parser'; +import { parseOrder } from './parseOrder'; + +type Line = [ending: string, qty: number]; + +const pad4 = (n: number): string => String(n).padStart(4, '0'); +const SEPARATORS = [' ', ', ', ',', '\n', ' ; ', '\t', ',,, ', '\n\n']; + +function generate(seed: number): { text: string; expected: Line[] } { + const r = rng(seed); + const expected: Line[] = []; + const parts: string[] = []; + const count = 1 + r.int(40); + for (let k = 0; k < count; k++) { + if (r.int(10) === 0) { + // range of 1-5 endings, qty 1 each + const from = r.int(9990); + const to = from + r.int(5); + for (let e = from; e <= to; e++) expected.push([pad4(e), 1]); + parts.push(`${pad4(from)}-${pad4(to)}`); + continue; + } + const ending = pad4(r.int(10000)); + const qty = r.int(3) === 0 ? 1 : 1 + r.int(150); + const form = r.pick(['bare', 'x', ' x ', '*', '=', ' = ', '_', '-', '()', ' ()', 'space'] as const); + const q = String(qty); + const rendered = + form === 'bare' ? (qty === 1 ? ending : `${ending}x${q}`) + : form === '()' ? `${ending}(${q})` + : form === ' ()' ? `${ending} (${q})` + : form === 'space' ? `${ending} ${q}` + : `${ending}${form}${q}`; + expected.push([ending, qty]); + parts.push(rendered); + } + + let text = r.int(3) === 0 ? '[12/10, 14:35] Shop 2: ' : ''; + parts.forEach((p, k) => { + text += p; + if (k < parts.length - 1) text += r.pick(SEPARATORS); + if (r.int(12) === 0) text += `\n${r.pick(['Total..', 'Total =', 'Totel ', 'മൊത്തം ', ''])}${r.int(1000)}\n`; + }); + if (r.int(2) === 0) text += `\nTotal ${r.int(1000)}`; + return { text, expected }; +} + +describe('generated orders round-trip', () => { + const seeds = process.env.SWEEP ? 20_000 : 400; + it(`reads back ${seeds} generated orders exactly`, () => { + for (let seed = 1; seed <= seeds; seed++) { + const { text, expected } = generate(seed); + const r = parseOrder(text); + expect(r.unreadable, `seed ${seed}: ${JSON.stringify(text)}`).toEqual([]); + expect(r.lines.map((l): Line => [l.ending, l.qty]), `seed ${seed}`).toEqual(expected); + } + }); +}); diff --git a/src/parser/invariants.test.ts b/src/parser/invariants.test.ts new file mode 100644 index 0000000..78f55df --- /dev/null +++ b/src/parser/invariants.test.ts @@ -0,0 +1,84 @@ +import { readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { rng } from '../testing/parser'; +import { MAX_QTY } from './limits'; +import { parseOrder, scanOrder } from './parseOrder'; + +const ALPHABET = [ + ...'0123456789012345678901234567890123456789', + ...'xX*=_-()/:.,;| \n\n\t?Oa', + 'മൊ', '٠', '൮', '✅', '​', ' ', '×', '—', '[12/10, 14:35] A: ', 'Total', '0888', '5104-5106', +]; + +function garbage(seed: number): string { + const r = rng(seed); + let s = ''; + const len = r.int(200); + for (let k = 0; k < len; k++) s += r.pick(ALPHABET); + return s; +} + +const FIXTURES = join(__dirname, '../../test/fixtures/orders'); +const fixtureTexts = readdirSync(FIXTURES) + .filter((f) => f.endsWith('.txt')) + .map((f) => readFileSync(join(FIXTURES, f), 'utf8')); + +const named = [ + '7784 1 7764 1 3367 2', '0888(1)0003(1)9077(1)', '5104-5106 0099-0101', '888 12345 O888 08?8 0888', + 'Totel 54 number 250', '[12/10, 14:35] Shop 2: 0888', '0057x2x3 0003-2-5104 5104 - 5106', '12/10/2026 14:35 0888.5', +]; +const inputs = [...named, ...fixtureTexts, ...Array.from({ length: 500 }, (_, s) => garbage(s + 1))]; + +const digitsOf = (s: string): string => s.replace(/[^0-9]/g, ''); + +describe('parser invariants', () => { + it('I1 digit conservation: every digit lands in exactly one item, in order', () => { + for (const t of inputs) { + const { bodies, items } = scanOrder(t); + expect(digitsOf(items.map((i) => i.text).join('')), JSON.stringify(t)).toBe(digitsOf(bodies.join('\n'))); + for (const i of items) if (i.kind === 'ignored' && i.reason === 'text') expect(digitsOf(i.text)).toBe(''); + } + }); + + it('I2 never throws, for any string', () => { + for (const t of inputs) expect(() => parseOrder(t)).not.toThrow(); + }); + + it('I3 deterministic', () => { + for (const t of inputs) expect(parseOrder(t)).toEqual(parseOrder(t)); + }); + + it('I4 re-rendering the lines and parsing again gives the same lines', () => { + for (const t of inputs) { + const once = parseOrder(t).lines.map((l) => [l.ending, l.qty]); + const again = parseOrder(once.map(([e, q]) => `${e}x${q}`).join('\n')).lines.map((l) => [l.ending, l.qty]); + expect(again).toEqual(once); + } + }); + + it('I6 quantities are 1..MAX_QTY and endings are exactly 4 digits', () => { + for (const t of inputs) { + for (const l of parseOrder(t).lines) { + expect(Number.isInteger(l.qty) && l.qty >= 1 && l.qty <= MAX_QTY).toBe(true); + expect(l.ending).toMatch(/^\d{4}$/); + } + } + }); + + it('I7 totalTickets is the sum of qty', () => { + for (const t of inputs) { + const r = parseOrder(t); + expect(r.totalTickets).toBe(r.lines.reduce((s, l) => s + l.qty, 0)); + } + }); + + it('I8 no invention: every non-range ending is written on its own line', () => { + for (const t of inputs) { + const { bodies } = scanOrder(t); + for (const l of parseOrder(t).lines) { + if (!l.fromRange) expect(bodies[l.line - 1]).toContain(l.ending); + } + } + }); +}); diff --git a/src/parser/lexer.ts b/src/parser/lexer.ts new file mode 100644 index 0000000..2ddf81a --- /dev/null +++ b/src/parser/lexer.ts @@ -0,0 +1,62 @@ +export type AtomKind = + | 'num' // run of ASCII digits + | 'baddigit' // run of non-ASCII digits (Malayalam, Arabic-Indic, Devanagari) + | 'word' // run of anything else that is not an operator, separator or space (letters, emoji, ?, ...) + | 'op' // one of x * = _ - ( ) / : . (x only when it stands alone as a word) + | 'sep'; // , ; | + +export interface Atom { + kind: AtomKind; + text: string; + start: number; + end: number; + glued: boolean; +} + +const OPS = new Set(['*', '=', '_', '-', '(', ')', '/', ':', '.']); +const SEPS = new Set([',', ';', '|']); + +function isAsciiDigit(c: number): boolean { + return c >= 48 && c <= 57; +} + +function isOtherDigit(c: number): boolean { + return ( + (c >= 0x0d66 && c <= 0x0d6f) || // Malayalam + (c >= 0x0660 && c <= 0x0669) || // Arabic-Indic + (c >= 0x06f0 && c <= 0x06f9) || // Extended Arabic-Indic + (c >= 0x0966 && c <= 0x096f) // Devanagari + ); +} + +function classOf(ch: string, code: number): AtomKind | 'space' { + if (ch === ' ') return 'space'; + if (isAsciiDigit(code)) return 'num'; + if (isOtherDigit(code)) return 'baddigit'; + if (OPS.has(ch)) return 'op'; + if (SEPS.has(ch)) return 'sep'; + return 'word'; +} + +export function lex(body: string): Atom[] { + const atoms: Atom[] = []; + let i = 0; + while (i < body.length) { + const code = body.charCodeAt(i); + const kind = classOf(body[i] as string, code); + if (kind === 'space') { + i++; + continue; + } + const glued = i > 0 && body[i - 1] !== ' '; + let j = i + 1; + if (kind === 'num' || kind === 'baddigit' || kind === 'word') { + while (j < body.length && classOf(body[j] as string, body.charCodeAt(j)) === kind) j++; + } + const text = body.slice(i, j); + const isTimes = kind === 'word' && (text === 'x' || text === 'X'); + atoms.push({ kind: isTimes ? 'op' : kind, text: isTimes ? 'x' : text, start: i, end: j, glued }); + i = j; + } + return atoms; +} diff --git a/src/parser/limits.test.ts b/src/parser/limits.test.ts new file mode 100644 index 0000000..fc0870f --- /dev/null +++ b/src/parser/limits.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from 'vitest'; +import { MAX_INPUT_CHARS } from './limits'; +import { parseOrder } from './parseOrder'; + +function timed(label: string, text: string): number { + const t0 = performance.now(); + parseOrder(text); + const ms = performance.now() - t0; + console.log(`${label}: ${ms.toFixed(1)} ms (${text.length} chars)`); + return ms; +} + +describe('parser limits', () => { + it('5,000 realistic lines parse well within the bound', () => { + const forms = ['8910=3', '0057x2', '3175-1', '7740(2)', '0159_1', '7784 1']; + const text = Array.from({ length: 5000 }, (_, k) => forms[k % forms.length]).join('\n'); + expect(text.length).toBeLessThanOrEqual(MAX_INPUT_CHARS); + const r = parseOrder(text); + expect(r.lines).toHaveLength(5000); + expect(timed('5000 lines', text)).toBeLessThan(1000); + }); + + it('pathological inputs at the size limit stay bounded', () => { + const size = MAX_INPUT_CHARS - 10; + const cases: [string, string][] = [ + ['commas', ','.repeat(size)], + ['open brackets', '('.repeat(size)], + ['dangling dashes', '0888-'.repeat(Math.floor(size / 5))], + ['glued range chain', '0888-0889'.repeat(Math.floor(size / 9))], + ['word/op alternation', `${'a-'.repeat(Math.floor(size / 2) - 1)}0`], + ['one long digit run', '7'.repeat(size)], + ['digits and letters', 'a0'.repeat(Math.floor(size / 2))], + ]; + for (const [label, text] of cases) expect(timed(label, text), label).toBeLessThan(1000); + }); + + it('input above the limit returns immediately', () => { + const text = '0888 '.repeat(250_000); // ~1.25 MB + expect(timed('over limit', text)).toBeLessThan(50); + expect(parseOrder(text).unreadable[0]?.reason).toBe('input_too_long'); + }); + + it('the largest allowed range expands without trouble', () => { + expect(parseOrder('0000-0099').lines).toHaveLength(100); + }); +}); diff --git a/src/parser/limits.ts b/src/parser/limits.ts new file mode 100644 index 0000000..40146a8 --- /dev/null +++ b/src/parser/limits.ts @@ -0,0 +1,8 @@ +/** Largest quantity per entry. Quantities are always 1 to 3 digits.*/ +export const MAX_QTY = 999; + +/** Largest number of endings one range may expand to. Provisional (plan Q4). */ +export const MAX_RANGE = 100; + +/** Longest input accepted; matches the ingest route's text cap. */ +export const MAX_INPUT_CHARS = 100_000; diff --git a/src/parser/normalize.ts b/src/parser/normalize.ts new file mode 100644 index 0000000..4750d2e --- /dev/null +++ b/src/parser/normalize.ts @@ -0,0 +1,17 @@ +const INVISIBLE = /[­​-‏‪-‮⁠-⁩︎️]/g; +// Tabs and Unicode spaces (NBSP, narrow NBSP used by WhatsApp before am/pm, en/em spaces...). +const SPACES = /[\t   -    ]/g; +const TIMES = /[×✕✖]/g; // × ✕ ✖ +const DASHES = /[‐-―−﹣-]/g; // hyphens, en/em dash, minus, full-width hyphen +const FULLWIDTH_DIGIT = /[0-9]/g; + +export function normalize(text: string): string[] { + return text + .replace(/\r\n?/g, '\n') + .replace(INVISIBLE, '') + .replace(SPACES, ' ') + .replace(TIMES, 'x') + .replace(DASHES, '-') + .replace(FULLWIDTH_DIGIT, (d) => String.fromCharCode(d.charCodeAt(0) - 0xff10 + 48)) + .split('\n'); +} diff --git a/src/parser/parseOrder.test.ts b/src/parser/parseOrder.test.ts index d286f02..879abdb 100644 --- a/src/parser/parseOrder.test.ts +++ b/src/parser/parseOrder.test.ts @@ -1,42 +1,127 @@ -import { existsSync, readFileSync } from 'node:fs'; -import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; -import { parseOrder } from './parseOrder'; +import { read } from '../testing/parser'; +import { mergeLines, parseOrder } from './parseOrder'; -const samplePath = join(__dirname, '../../test/fixtures/sample-order.txt'); +describe('parseOrder: core cases', () => { + it('1: empty input gives nothing and does not throw', () => { + for (const t of ['', ' ', '\n\n\n']) { + expect(read(t)).toEqual({ lines: [], unreadable: [], ignored: [], tickets: 0 }); + } + }); -describe('parseOrder: notation', () => { - it.todo('a bare 4-digit ending is 1 ticket'); - it.todo('0057x2 and 9155*40 set the quantity'); - it.todo('0003-2 (dash plus 1-3 digits) is a quantity'); - it.todo('5104-5106 (dash plus 4 digits) is a range'); - it.todo('0888(1)0003(1)9077(1) with no separators reads as three endings'); - it.todo('strips the WhatsApp copy prefix [dd/mm, hh:mm] Name:'); - it.todo('"Totel 54 number" is checked by walking back until the ticket sum or line count matches'); - it.todo('a bare number on its own line (250) is a total'); - it.todo('never guesses: unreadable text goes to `unreadable`'); -}); + it('2: a bare 4-digit ending is 1 ticket', () => { + expect(read('0888').lines).toEqual(['0888x1']); + }); -describe.skipIf(!existsSync(samplePath))('parseOrder: the real 90-line sample', () => { - const result = () => parseOrder(readFileSync(samplePath, 'utf8')); + it('3: leading zeros are kept as strings', () => { + const r = parseOrder('0057 0000'); + expect(r.lines.map((l) => l.ending)).toEqual(['0057', '0000']); + }); - it('reads 338 tickets and leaves nothing unreadable', () => { - const r = result(); - expect(r.totalTickets).toBe(338); + it('4: bare endings separated by spaces', () => { + expect(read('7598 7896 7986 7877').lines).toEqual(['7598x1', '7896x1', '7986x1', '7877x1']); + }); + + it('5: runs of commas are noise', () => { + expect(read('3472,,,3484,,3422, 0887').lines).toEqual(['3472x1', '3484x1', '3422x1', '0887x1']); + }); + + it('6: ending, space, quantity', () => { + expect(read('7784 1 7764 1 3367 2').lines).toEqual(['7784x1', '7764x1', '3367x2']); + }); + + it('7: x quantity in every form', () => { + expect(read('0057x2 0057 x 2 0057X2 0057×2').lines).toEqual(['0057x2', '0057x2', '0057x2', '0057x2']); + }); + + it('8: * quantity', () => { + expect(read('9155*40\n8203*100').lines).toEqual(['9155x40', '8203x100']); + }); + + it('9: bracket quantities with no separators', () => { + expect(read('0888(1)0003(1)9077(1)').lines).toEqual(['0888x1', '0003x1', '9077x1']); + }); + + it('10: dash plus 1-3 digits is a quantity', () => { + expect(read('0003-2').lines).toEqual(['0003x2']); + }); + + it('10a: = and _ quantities', () => { + expect(read('8910=3 0159_1').lines).toEqual(['8910x3', '0159x1']); + }); + + it('11: dash plus 4 digits is a range, qty 1 per ending', () => { + const r = parseOrder('5104-5106'); + expect(r.lines).toEqual([ + { ending: '5104', qty: 1, line: 1, fromRange: true }, + { ending: '5105', qty: 1, line: 1, fromRange: true }, + { ending: '5106', qty: 1, line: 1, fromRange: true }, + ]); + }); + + it('12: a backwards range is unreadable', () => { + expect(read('5106-5104')).toMatchObject({ lines: [], unreadable: ['5106-5104:bad_range'] }); + }); + + it('13: broken endings are never padded, truncated or corrected', () => { + expect(read('888 12345 O888 08?8 0888')).toMatchObject({ + lines: ['0888x1'], + unreadable: ['888:stray_number', '12345:bad_ending_length', 'O888:mixed_characters', '08?8:unclear_digit'], + }); + }); + + it('14: zero and oversized quantities are unreadable', () => { + expect(read('0057x0 0057x5000')).toMatchObject({ + lines: [], + unreadable: ['0057x0:zero_quantity', '0057x5000:quantity_too_large'], + }); + }); + + it('15: repeated endings are added up and listed', () => { + const r = parseOrder('0888 0888x3'); + expect(r.lines).toHaveLength(2); + expect(r.totalTickets).toBe(4); + expect(r.repeated).toEqual([{ ending: '0888', occurrences: 2, totalQty: 4 }]); + }); + + it('16: totals are ignored, with no entry and no question', () => { + const r = read('Totel 54 number\nTotal..50\nTotal =20\nമൊത്തം 29\n250'); + expect(r.lines).toEqual([]); expect(r.unreadable).toEqual([]); + expect(r.ignored).toEqual(['Totel', '54', 'number', 'Total', '50', 'Total', '=20', 'മൊത്തം', '29', '250']); }); - it('matches the stated totals 54 and 250', () => { - const r = result(); - expect(r.totals.find((t) => t.stated === 54)?.matched).toBe(true); - expect(r.totals.find((t) => t.stated === 250)?.matched).toBe(true); + it('17: WhatsApp prefix is stripped, name and time digits are not read', () => { + expect(read('[12/10, 14:35] Shop 2: 0888')).toEqual({ lines: ['0888x1'], unreadable: [], ignored: [], tickets: 1 }); }); - it('finds the repeated endings', () => { - expect([...result().repeated].sort()).toEqual(['0003', '0057', '0888', '6678']); + it('18: greeting words are ignored, numbers still read', () => { + expect(read('Hi sir, order: 0888 0057x2')).toEqual({ + lines: ['0888x1', '0057x2'], + unreadable: [], + ignored: ['Hi', 'sir', 'order:'], + tickets: 3, + }); }); - it('reports 27 endings not covered by any total', () => { - expect(result().uncovered).toHaveLength(27); + it('20: a number after a quantity on an entry line is unreadable', () => { + expect(read('3367 2 250')).toMatchObject({ lines: ['3367x2'], unreadable: ['250:stray_number'] }); + }); +}); + +describe('parseOrder: output details', () => { + it('records the 1-based line of every entry', () => { + expect(parseOrder('0888\n\n0057x2').lines.map((l) => l.line)).toEqual([1, 3]); + }); + + it('mergeLines sums per ending in first-seen order', () => { + expect(mergeLines(parseOrder('0888 0057x2 0888x3').lines)).toEqual([ + { ending: '0888', qty: 4 }, + { ending: '0057', qty: 2 }, + ]); + }); + + it('treats a non-string as empty instead of throwing', () => { + expect(parseOrder(undefined as unknown as string).lines).toEqual([]); }); }); diff --git a/src/parser/parseOrder.ts b/src/parser/parseOrder.ts index 31d8c9b..4cb1e57 100644 --- a/src/parser/parseOrder.ts +++ b/src/parser/parseOrder.ts @@ -1,5 +1,87 @@ -import type { ParseResult } from './types'; +import { lex } from './lexer'; +import { MAX_INPUT_CHARS } from './limits'; +import { normalize } from './normalize'; +import { stripPrefix } from './prefix'; +import { scanLine, type ScanItem } from './scan'; +import type { Ignored, OrderLine, ParseResult, RepeatedEnding, Unreadable } from './types'; -export function parseOrder(_text: string): ParseResult { - throw new Error('parseOrder is not implemented yet: port the tested parser into src/parser/parseOrder.ts'); +export interface ScanResult { + bodies: string[]; + items: ScanItem[]; +} + +export function scanOrder(text: string): ScanResult { + const bodies: string[] = []; + const items: ScanItem[] = []; + normalize(text).forEach((raw, index) => { + const { body } = stripPrefix(raw); + bodies.push(body); + items.push(...scanLine(body, lex(body), index + 1)); + }); + return { bodies, items }; +} + +const pad4 = (n: number): string => String(n).padStart(4, '0'); + +export function parseOrder(input: string): ParseResult { + const text = typeof input === 'string' ? input : ''; + if (text.length > MAX_INPUT_CHARS) { + return { + lines: [], + totalTickets: 0, + ignored: [], + repeated: [], + unreadable: [{ text: `${text.slice(0, 40)}…`, line: 1, reason: 'input_too_long' }], + }; + } + + const lines: OrderLine[] = []; + const ignored: Ignored[] = []; + const unreadable: Unreadable[] = []; + for (const item of scanOrder(text).items) { + switch (item.kind) { + case 'entry': + lines.push({ ending: item.ending, qty: item.qty, line: item.line, fromRange: false }); + break; + case 'range': + for (let e = Number(item.from); e <= Number(item.to); e++) { + lines.push({ ending: pad4(e), qty: 1, line: item.line, fromRange: true }); + } + break; + case 'ignored': + ignored.push({ text: item.text, line: item.line, reason: item.reason }); + break; + case 'unreadable': + unreadable.push({ text: item.text, line: item.line, reason: item.reason }); + break; + } + } + + return { + lines, + totalTickets: lines.reduce((sum, l) => sum + l.qty, 0), + ignored, + repeated: findRepeated(lines), + unreadable, + }; +} + +function findRepeated(lines: OrderLine[]): RepeatedEnding[] { + const byEnding = new Map(); + for (const l of lines) { + const r = byEnding.get(l.ending); + if (r) { + r.occurrences++; + r.totalQty += l.qty; + } else { + byEnding.set(l.ending, { ending: l.ending, occurrences: 1, totalQty: l.qty }); + } + } + return [...byEnding.values()].filter((r) => r.occurrences > 1); +} + +export function mergeLines(lines: OrderLine[]): { ending: string; qty: number }[] { + const merged = new Map(); + for (const l of lines) merged.set(l.ending, (merged.get(l.ending) ?? 0) + l.qty); + return [...merged].map(([ending, qty]) => ({ ending, qty })); } diff --git a/src/parser/prefix.ts b/src/parser/prefix.ts new file mode 100644 index 0000000..92db6a8 --- /dev/null +++ b/src/parser/prefix.ts @@ -0,0 +1,17 @@ +const DATE = String.raw`\d{1,2}[/.-]\d{1,2}`; +const TIME = String.raw`\d{1,2}[:.]\d{2}(?:[:.]\d{2})?(?: ?[ap]\.?m\.?)?`; +const NAME = String.raw`[^:\n]{1,60}:`; + +const APP_COPY = new RegExp(String.raw`^\[${DATE}(?:[/.-]\d{2,4})?,? ${TIME}\] ?${NAME} ?`, 'i'); +const EXPORT = new RegExp(String.raw`^${DATE}[/.-]\d{2,4},? ${TIME} - ${NAME} ?`, 'i'); + +export interface Stripped { + body: string; + removed: number; +} + +export function stripPrefix(line: string): Stripped { + const m = APP_COPY.exec(line) ?? EXPORT.exec(line); + if (!m) return { body: line, removed: 0 }; + return { body: line.slice(m[0].length), removed: m[0].length }; +} diff --git a/src/parser/scan.ts b/src/parser/scan.ts new file mode 100644 index 0000000..c0a5ef3 --- /dev/null +++ b/src/parser/scan.ts @@ -0,0 +1,221 @@ +import { MAX_QTY, MAX_RANGE } from './limits'; +import type { Atom } from './lexer'; +import type { UnreadableReason } from './types'; + +export type ScanItem = + | { kind: 'entry'; ending: string; qty: number; text: string; line: number } + | { kind: 'range'; from: string; to: string; text: string; line: number } + | { kind: 'ignored'; reason: 'text' | 'lone_number'; text: string; line: number } + | { kind: 'unreadable'; reason: UnreadableReason; text: string; line: number }; + +interface Loose { + kind: 'loose'; + reason: 'stray_number' | 'orphan_quantity'; + text: string; + line: number; +} + +type Item = ScanItem | Loose; + +const QTY_OPS = new Set(['x', '*', '=', '_']); +const CONTINUATION_OPS = new Set(['x', '*', '=', '_', '(', '-']); + +type Step = { ok: true; item: ScanItem; next: number } | { ok: false; reason: UnreadableReason; next: number }; + +export function scanLine(body: string, atoms: Atom[], line: number): ScanItem[] { + const n = atoms.length; + const at = (k: number): Atom | undefined => atoms[k]; + const isOp = (a: Atom | undefined, ...ops: string[]): boolean => + a !== undefined && a.kind === 'op' && ops.includes(a.text); + const isDigits = (a: Atom | undefined): boolean => a !== undefined && (a.kind === 'num' || a.kind === 'baddigit'); + const slice = (from: number, toExclusive: number): string => + body.slice((atoms[from] as Atom).start, (atoms[toExclusive - 1] as Atom).end); + + const isNoiseDot = (k: number): boolean => { + const a = at(k); + if (!isOp(a, '.')) return false; + const next = at(k + 1); + const decimal = + isDigits(at(k - 1)) && (a as Atom).glued && isDigits(next) && (next as Atom).glued && (next as Atom).text.length <= 3; + return !decimal; + }; + + const clusterEnd = (k: number): number => { + let j = k + 1; + while (j < n && (atoms[j] as Atom).glued && (atoms[j] as Atom).kind !== 'sep' && !isNoiseDot(j)) j++; + return j; + }; + + const hasDigits = (from: number, to: number): boolean => { + for (let k = from; k < to; k++) if (isDigits(atoms[k])) return true; + return false; + }; + + const subclusterEnd = (k: number, end: number): number => { + let j = k + 1; + while (j < end) { + const a = atoms[j - 1] as Atom; + const b = atoms[j] as Atom; + if ((a.kind === 'word' && b.kind === 'op') || (a.kind === 'op' && b.kind === 'word')) break; + j++; + } + return j; + }; + + const quantity = (digits: string): { ok: true; qty: number } | { ok: false; reason: UnreadableReason } => { + if (digits.length > 3) return { ok: false, reason: 'quantity_too_large' }; + const qty = Number(digits); + if (qty === 0) return { ok: false, reason: 'zero_quantity' }; + if (qty > MAX_QTY) return { ok: false, reason: 'quantity_too_large' }; + return { ok: true, qty }; + }; + + const withQty = (i: number, digits: string, next: number): Step => { + const q = quantity(digits); + if (!q.ok) return { ok: false, reason: q.reason, next }; + const ending = (atoms[i] as Atom).text; + return { ok: true, item: { kind: 'entry', ending, qty: q.qty, text: slice(i, next), line }, next }; + }; + + const single = (i: number): Step => ({ + ok: true, + item: { kind: 'entry', ending: (atoms[i] as Atom).text, qty: 1, text: slice(i, i + 1), line }, + next: i + 1, + }); + + const parseEntry = (i: number): Step => { + const k = i + 1; + const a = at(k); + const b = at(k + 1); + + if (isOp(a, '-')) { + const gluedBoth = (a as Atom).glued && b !== undefined && b.glued; + if (b !== undefined && b.kind === 'num') { + if (!gluedBoth) return { ok: false, reason: 'bad_range', next: k + 2 }; + if (b.text.length === 4) { + const from = Number((atoms[i] as Atom).text); + const to = Number(b.text); + if (to < from) return { ok: false, reason: 'bad_range', next: k + 2 }; + if (to - from + 1 > MAX_RANGE) return { ok: false, reason: 'range_too_large', next: k + 2 }; + const item: ScanItem = { kind: 'range', from: (atoms[i] as Atom).text, to: b.text, text: slice(i, k + 2), line }; + return { ok: true, item, next: k + 2 }; + } + if (b.text.length <= 3) return withQty(i, b.text, k + 2); + return { ok: false, reason: 'bad_ending_length', next: k + 2 }; + } + if ((a as Atom).glued) return { ok: false, reason: 'mixed_characters', next: k + 1 }; + return single(i); + } + + if (a !== undefined && a.kind === 'op' && QTY_OPS.has(a.text)) { + if (b !== undefined && b.kind === 'num') return withQty(i, b.text, k + 2); + if (a.glued) return { ok: false, reason: 'mixed_characters', next: k + 1 }; + return single(i); + } + + if (isOp(a, '(')) { + if (b !== undefined && b.kind === 'num' && isOp(at(k + 2), ')')) return withQty(i, b.text, k + 3); + if ((a as Atom).glued) return { ok: false, reason: 'mixed_characters', next: k + 1 }; + return single(i); + } + + if (a !== undefined && a.kind === 'num' && !a.glued && a.text.length <= 3) { + const after = at(k + 1); + const free = after === undefined || !after.glued || after.kind === 'sep' || isNoiseDot(k + 1); + if (free) return withQty(i, a.text, k + 1); + } + + return single(i); + }; + + const continuationReason = (item: ScanItem, next: Atom): UnreadableReason => { + if (next.kind === 'op' && CONTINUATION_OPS.has(next.text)) { + return item.kind === 'range' ? 'bad_range' : 'double_quantity'; + } + if (next.kind === 'op' && (next.text === '/' || next.text === ':')) return 'date_or_time'; + if (next.kind === 'baddigit') return 'unsupported_digits'; + if (next.kind === 'word' && /^[?xX]+$/.test(next.text)) return 'unclear_digit'; + return 'mixed_characters'; + }; + + const classify = (from: number, to: number): Item => { + const text = slice(from, to); + const part = atoms.slice(from, to); + if (part.some((a) => a.kind === 'baddigit')) return { kind: 'unreadable', reason: 'unsupported_digits', text, line }; + const words = part.filter((a) => a.kind === 'word'); + if (words.length > 0) { + const unclear = words.every((w) => /^[?xX]+$/.test(w.text)); + return { kind: 'unreadable', reason: unclear ? 'unclear_digit' : 'mixed_characters', text, line }; + } + if (part.some((a) => a.kind === 'op' && (a.text === '/' || a.text === ':'))) { + return { kind: 'unreadable', reason: 'date_or_time', text, line }; + } + const nums = part.filter((a) => a.kind === 'num'); + if (nums.some((a) => a.text.length >= 5)) return { kind: 'unreadable', reason: 'bad_ending_length', text, line }; + if (nums.length === 1) { + const num = nums[0] as Atom; + if (part.length === 1) { + return num.text.length <= 3 ? { kind: 'loose', reason: 'stray_number', text, line } : { kind: 'unreadable', reason: 'mixed_characters', text, line }; + } + const [first, second, third] = part; + const opThenNum = + part.length === 2 && first !== undefined && first.kind === 'op' && (QTY_OPS.has(first.text) || first.text === '-') && second === num; + const bracketed = part.length === 3 && isOp(first, '(') && second === num && isOp(third, ')'); + if (num.text.length <= 3 && (opThenNum || bracketed)) { + return { kind: 'loose', reason: 'orphan_quantity', text, line }; + } + } + return { kind: 'unreadable', reason: 'mixed_characters', text, line }; + }; + + const items: Item[] = []; + let entryLine = false; + let i = 0; + while (i < n) { + const a = atoms[i] as Atom; + if (a.kind === 'sep' || isNoiseDot(i)) { + i++; + continue; + } + + if (a.kind === 'num' && a.text.length === 4) { + entryLine = true; + const step = parseEntry(i); + const next = at(step.next); + const prev = at(step.next - 1); + const glued = next !== undefined && next.glued && next.kind !== 'sep' && !isNoiseDot(step.next); + const chain = glued && isOp(prev, ')') && next.kind === 'num' && next.text.length === 4; + const wordAfterOp = glued && next.kind === 'word' && prev !== undefined && prev.kind === 'op'; + if (step.ok && (!glued || chain || wordAfterOp)) { + items.push(step.item); + i = step.next; + continue; + } + const reason = step.ok ? continuationReason(step.item, next as Atom) : step.reason; + const end = step.next < n && glued ? clusterEnd(step.next) : Math.max(step.next, i + 1); + items.push({ kind: 'unreadable', reason, text: slice(i, end), line }); + i = end; + continue; + } + + const end = clusterEnd(i); + if (!hasDigits(i, end)) { + items.push({ kind: 'ignored', reason: 'text', text: slice(i, end), line }); + i = end; + continue; + } + for (let s = i; s < end; ) { + const sub = subclusterEnd(s, end); + if (!hasDigits(s, sub)) items.push({ kind: 'ignored', reason: 'text', text: slice(s, sub), line }); + else items.push(classify(s, sub)); + s = sub; + } + i = end; + } + + return items.map((it): ScanItem => { + if (it.kind !== 'loose') return it; + if (entryLine) return { kind: 'unreadable', reason: it.reason, text: it.text, line: it.line }; + return { kind: 'ignored', reason: 'lone_number', text: it.text, line: it.line }; + }); +} diff --git a/src/parser/types.ts b/src/parser/types.ts index a4267d4..38f42e4 100644 --- a/src/parser/types.ts +++ b/src/parser/types.ts @@ -1,28 +1,60 @@ - export interface OrderLine { /** The last 4 digits, as a string so leading zeros survive ("0057"). */ ending: string; - /** How many tickets. Always at least 1. */ + /** How many tickets. Always 1 to MAX_QTY. */ qty: number; + /** 1-based input line it came from. */ + line: number; + /** True when expanded from a range such as 5104-5106. */ + fromRange: boolean; } -/** A total the sender wrote, such as "Totel 54 number" or a bare "250" on its own line. */ -export interface TotalCheck { - /** The number the sender wrote. */ - stated: number; - /** True if the tickets before that total add up to it (by ticket count or by line count). */ - matched: boolean; +export interface Ignored { + /** The exact characters skipped. */ + text: string; + line: number; + /** `text`: no digits at all. `lone_number`: a 1-3 digit number on a line with no entry (usually a total). */ + reason: 'text' | 'lone_number'; +} + +export type UnreadableReason = + | 'bad_ending_length' // 5 or more digits (could be a full ticket number) + | 'stray_number' // 1-3 digits on a line with entries, not a quantity + | 'mixed_characters' // digits glued to letters or symbols (O888, 0888a, 0888.5) + | 'unclear_digit' // digits glued to ? or x (08?8, 0xx8) + | 'date_or_time' // 12/10/2026, 14:35 + | 'zero_quantity' + | 'quantity_too_large' + | 'double_quantity' // 0057x2x3 + | 'orphan_quantity' // x2 or (3) with no ending before it, on a line with entries + | 'bad_range' // backwards range, spaced dash, range with a quantity + | 'range_too_large' + | 'unsupported_digits' // Malayalam or other non-ASCII digits + | 'input_too_long'; + +/** Something containing digits that the parser will not guess about. The bot asks the owner. */ +export interface Unreadable { + /** The exact characters, for the bot to show. */ + text: string; + line: number; + reason: UnreadableReason; +} + +export interface RepeatedEnding { + ending: string; + occurrences: number; + /** Sum of qty over all occurrences (repeats are added up). */ + totalQty: number; } export interface ParseResult { + /** Every occurrence, in input order. */ lines: OrderLine[]; /** Sum of qty over all lines. */ totalTickets: number; - totals: TotalCheck[]; - /** Endings that no stated total covers. The bot lists them and asks. */ - uncovered: string[]; - /** Endings that appear more than once. The bot asks whether to add them up. */ - repeated: string[]; - /** Text the parser could not read. It is never guessed: the bot asks. */ - unreadable: string[]; + /** Words and lone numbers skipped. Informational only. */ + ignored: Ignored[]; + /** Endings that occur more than once, in order of first occurrence. Shown, not asked about. */ + repeated: RepeatedEnding[]; + unreadable: Unreadable[]; } diff --git a/src/results/job.test.ts b/src/results/job.test.ts index d565e4d..107c453 100644 --- a/src/results/job.test.ts +++ b/src/results/job.test.ts @@ -1,13 +1,21 @@ import { readFileSync } from 'node:fs'; import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; -import { makeDb } from '../testing/helpers'; +import { hasPdftotext, makeDb } from '../testing/helpers'; import { extractPdfText } from './pdfText'; import { processPdf, runResultsJob } from './job'; import { parseResult, validateDraw } from './parseResult'; import { parsePortalListing, selectRecent } from './portal'; import { storeDraw } from './store'; +if (!hasPdftotext) { + console.warn( + '[job.test] pdftotext is not installed: tests that read the real PDF are SKIPPED. ' + + 'Install poppler (conda install -c conda-forge poppler, or sudo apt install poppler-utils) and re-run.', + ); +} +const withPdftotext = it.skipIf(!hasPdftotext); + const FX = join(__dirname, '../../test/fixtures/results'); const layout = (n: string) => readFileSync(join(FX, n), 'utf8'); const sk73Pdf = readFileSync(join(FX, 'sk73_2026-10-16.pdf')); @@ -76,19 +84,23 @@ describe('storeDraw', () => { }); describe('processPdf (real pdftotext on the real SK-73 PDF)', () => { - it('parses and validates clean', async () => { + withPdftotext('parses and validates clean', async () => { const { draw, issues } = await processPdf(sk73Pdf); expect(draw?.schemeCode).toBe('SK'); expect(issues).toEqual([]); }); - it('catches a PDF that is not the draw the listing claims', async () => { + withPdftotext('catches a PDF that is not the draw the listing claims', async () => { const entry = { name: 'x', code: 'KN', drawNo: 645, date: '2026-10-15', url: 'u#1' }; const { issues } = await processPdf(sk73Pdf, entry); expect(issues.some((i) => i.message.includes('listing says'))).toBe(true); }); - it('reports an unreadable PDF instead of throwing', async () => { + it('says how to fix a missing pdftotext instead of failing silently', async () => { + await expect(extractPdfText(sk73Pdf, 'pdftotext-not-installed-xyz')).rejects.toThrow(/install poppler-utils/); + }); + + withPdftotext('reports an unreadable PDF instead of throwing', async () => { const { draw, issues } = await processPdf(Buffer.from('not a pdf'), undefined, extractPdfText); expect(draw).toBeNull(); expect(issues[0]?.level).toBe('ERROR'); @@ -99,7 +111,7 @@ describe('runResultsJob', () => { const fetchImpl = (async () => new Response(LISTING)) as typeof fetch; const base = { today: '2026-10-16', days: 3, fetchImpl }; - it('stores a good draw, holds a mismatched one, reports a failed fetch, skips ready ones on rerun', async () => { + withPdftotext('stores a good draw, holds a mismatched one, reports a failed fetch, skips ready ones on rerun', async () => { const db = makeDb(); const getPdf = async (e: { code: string }) => { if (e.code === 'SK') return sk73Pdf; diff --git a/src/testing/helpers.ts b/src/testing/helpers.ts index cfcb8b2..d1a30f5 100644 --- a/src/testing/helpers.ts +++ b/src/testing/helpers.ts @@ -1,3 +1,4 @@ +import { spawnSync } from 'node:child_process'; import type { Db } from '../db'; import { openDb } from '../db'; import { runMigrations } from '../db/migrate'; @@ -40,3 +41,8 @@ export function seedFixture(db: Db): Fixture { return { shops, admin, admin2, shopUser, disabledUser }; } + +export const hasPdftotext: boolean = (() => { + const r = spawnSync('pdftotext', ['-v'], { encoding: 'utf8' }); + return !r.error; +})(); diff --git a/src/testing/parser.ts b/src/testing/parser.ts new file mode 100644 index 0000000..53f4a62 --- /dev/null +++ b/src/testing/parser.ts @@ -0,0 +1,33 @@ +import { parseOrder } from '../parser/parseOrder'; +import type { ParseResult } from '../parser/types'; + +export interface Read { + lines: string[]; + unreadable: string[]; + ignored: string[]; + tickets: number; +} + +export function compact(r: ParseResult): Read { + return { + lines: r.lines.map((l) => `${l.ending}x${l.qty}`), + unreadable: r.unreadable.map((u) => `${u.text}:${u.reason}`), + ignored: r.ignored.map((i) => i.text), + tickets: r.totalTickets, + }; +} + +export function read(text: string): Read { + return compact(parseOrder(text)); +} + +/** Small deterministic PRNG (LCG) so generated tests reproduce from a seed. */ +export function rng(seed: number): { int: (maxExclusive: number) => number; pick: (xs: readonly T[]) => T } { + let s = seed >>> 0 || 1; + const next = (): number => { + s = (Math.imul(s, 1664525) + 1013904223) >>> 0; + return s / 0x100000000; + }; + const int = (maxExclusive: number): number => Math.floor(next() * maxExclusive); + return { int, pick: (xs: readonly T[]): T => xs[int(xs.length)] as T }; +} diff --git a/test/fixtures/orders/owner-2026-11-03.expected.json b/test/fixtures/orders/owner-2026-11-03.expected.json new file mode 100644 index 0000000..bd74711 --- /dev/null +++ b/test/fixtures/orders/owner-2026-11-03.expected.json @@ -0,0 +1,134 @@ +{ + "totalTickets": 115, + "lines": [ + "8910 x3 @1", + "7810 x1 @2", + "0129 x1 @3", + "0189 x1 @4", + "7086 x1 @5", + "1155 x2 @6", + "7569 x1 @7", + "1825 x2 @8", + "9081 x1 @9", + "9760 x1 @10", + "8891 x1 @11", + "8069 x2 @12", + "7081 x2 @13", + "2130 x2 @14", + "3195 x1 @15", + "1739 x1 @16", + "8960 x2 @17", + "7890 x2 @18", + "9310 x2 @19", + "1874 x1 @20", + "3417 x1 @21", + "5584 x1 @22", + "0019 x1 @23", + "3591 x1 @24", + "1239 x1 @25", + "1098 x2 @26", + "7815 x1 @27", + "0719 x1 @28", + "7315 x1 @29", + "9724 x1 @30", + "7310 x1 @31", + "6250 x1 @32", + "0102 x1 @33", + "1598 x1 @34", + "9815 x1 @35", + "8951 x1 @36", + "2055 x1 @37", + "8491 x1 @38", + "5173 x1 @39", + "3175 x1 @42", + "0072 x2 @43", + "0207 x1 @44", + "0702 x1 @45", + "0754 x1 @46", + "7145 x1 @47", + "1657 x1 @48", + "0848 x1 @49", + "5393 x1 @50", + "0159 x1 @53", + "9015 x1 @54", + "7502 x1 @55", + "1348 x1 @56", + "8670 x1 @57", + "2755 x1 @58", + "4406 x1 @61", + "6044 x1 @61", + "4460 x1 @61", + "1113 x1 @61", + "4604 x1 @62", + "6440 x1 @62", + "0464 x1 @62", + "0446 x1 @62", + "0644 x1 @63", + "4046 x1 @63", + "4640 x1 @63", + "4064 x1 @63", + "1775 x1 @64", + "3127 x1 @64", + "2070 x1 @64", + "9415 x1 @64", + "9516 x1 @65", + "1131 x1 @65", + "1311 x1 @65", + "3111 x1 @65", + "7740 x2 @69", + "7741 x2 @70", + "7742 x1 @71", + "7743 x1 @72", + "7744 x1 @73", + "7745 x1 @74", + "7746 x1 @75", + "7747 x1 @76", + "7748 x1 @77", + "7749 x2 @78", + "0810 x1 @79", + "0811 x2 @80", + "0812 x2 @81", + "0813 x1 @82", + "0814 x1 @83", + "0815 x2 @84", + "0816 x1 @85", + "0817 x2 @86", + "0818 x1 @87", + "0819 x2 @88", + "9086 x1 @89" + ], + "repeated": [], + "unreadable": [], + "ignored": [ + { + "text": "Total", + "line": 40, + "reason": "text" + }, + { + "text": "50", + "line": 40, + "reason": "lone_number" + }, + { + "text": "Total", + "line": 66, + "reason": "text" + }, + { + "text": "=20", + "line": 66, + "reason": "lone_number" + }, + { + "text": "മൊത്തം", + "line": 90, + "reason": "text" + }, + { + "text": "29", + "line": 90, + "reason": "lone_number" + } + ] +} diff --git a/test/fixtures/orders/owner-2026-11-03.txt b/test/fixtures/orders/owner-2026-11-03.txt new file mode 100644 index 0000000..1291f8c --- /dev/null +++ b/test/fixtures/orders/owner-2026-11-03.txt @@ -0,0 +1,90 @@ +8910=3 +7810=1 +0129=1 +0189=1 +7086=1 +1155=2 +7569=1 +1825=2 +9081=1 +9760=1 +8891=1 +8069=2 +7081=2 +2130=2 +3195=1 +1739=1 +8960=2 +7890=2 +9310=2 +1874=1 +3417=1 +5584=1 +0019=1 +3591=1 +1239=1 +1098=2 +7815=1 +0719=1 +7315=1 +9724=1 +7310=1 +6250=1 +0102=1 +1598=1 +9815=1 +8951=1 +2055=1 +8491=1 +5173=1 +Total..50 + +3175-1 +0072-2 +0207-1 +0702=1 +0754-1 +7145-1 +1657-1 +0848-1 +5393-1 + + +0159_1 +9015_1 +7502_1 +1348_1 +8670_1 +2755_1 + + +4406,6044,4460,1113 +4604,6440,0464,0446 +0644,4046,4640,4064 +1775,3127,2070,9415 +9516,1131,1311,3111 +Total =20 + + +7740(2) +7741(2) +7742(1) +7743(1) +7744(1) +7745(1) +7746(1) +7747(1) +7748(1) +7749(2) +0810(1) +0811(2) +0812(2) +0813(1) +0814(1) +0815(2) +0816(1) +0817(2) +0818(1) +0819(2) +9086(1) +മൊത്തം 29