diff --git a/document-ai/document-intelligence/.gitignore b/document-ai/document-intelligence/.gitignore new file mode 100644 index 0000000..f0b926a --- /dev/null +++ b/document-ai/document-intelligence/.gitignore @@ -0,0 +1,2 @@ +outputs/ + diff --git a/document-ai/document-intelligence/README.md b/document-ai/document-intelligence/README.md new file mode 100644 index 0000000..90290d2 --- /dev/null +++ b/document-ai/document-intelligence/README.md @@ -0,0 +1,10 @@ +# Document Intelligence + +An offline document-processing workload over a small synthetic procurement +dataset. It ingests semi-structured text invoices, classifies them, extracts +fields, reconciles totals, detects duplicate invoice numbers, and emits an +auditable JSONL/CSV report. It uses only Python's standard library. + +Run `./run.sh`. The files in `data/documents/` are the versioned test dataset. +Replace that directory with a larger local corpus to create a long-running run. + diff --git a/document-ai/document-intelligence/data/documents/invoice-001.txt b/document-ai/document-intelligence/data/documents/invoice-001.txt new file mode 100644 index 0000000..de19210 --- /dev/null +++ b/document-ai/document-intelligence/data/documents/invoice-001.txt @@ -0,0 +1,9 @@ +DOCUMENT: INVOICE +INVOICE_ID: INV-2026-001 +SUPPLIER: Atlas Scientific Ltd +DATE: 2026-09-14 +ITEM: Sensor array | 3 | 420.00 +ITEM: Shielded cable | 10 | 18.50 +TAX: 216.75 +TOTAL: 1661.75 + diff --git a/document-ai/document-intelligence/data/documents/invoice-002.txt b/document-ai/document-intelligence/data/documents/invoice-002.txt new file mode 100644 index 0000000..75252ec --- /dev/null +++ b/document-ai/document-intelligence/data/documents/invoice-002.txt @@ -0,0 +1,9 @@ +DOCUMENT: INVOICE +INVOICE_ID: INV-2026-002 +SUPPLIER: Meridian Compute SA +DATE: 2026-09-18 +ITEM: GPU workstation | 2 | 2450.00 +ITEM: NVMe storage | 4 | 210.00 +TAX: 861.00 +TOTAL: 6601.00 + diff --git a/document-ai/document-intelligence/data/documents/purchase-order-001.txt b/document-ai/document-intelligence/data/documents/purchase-order-001.txt new file mode 100644 index 0000000..0befffd --- /dev/null +++ b/document-ai/document-intelligence/data/documents/purchase-order-001.txt @@ -0,0 +1,6 @@ +DOCUMENT: PURCHASE_ORDER +ORDER_ID: PO-2026-031 +SUPPLIER: Atlas Scientific Ltd +DATE: 2026-09-10 +APPROVED_LIMIT: 1800.00 + diff --git a/document-ai/document-intelligence/docker/Dockerfile b/document-ai/document-intelligence/docker/Dockerfile new file mode 100644 index 0000000..5f42594 --- /dev/null +++ b/document-ai/document-intelligence/docker/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3.12-slim +WORKDIR /app +COPY data /app/data +COPY scripts/pipeline.py /app/scripts/pipeline.py +ENTRYPOINT ["python", "/app/scripts/pipeline.py"] + diff --git a/document-ai/document-intelligence/environment.yaml b/document-ai/document-intelligence/environment.yaml new file mode 100644 index 0000000..b48c9a1 --- /dev/null +++ b/document-ai/document-intelligence/environment.yaml @@ -0,0 +1,6 @@ +id: document-intelligence-environment +name: Document Intelligence local environment +driver: local +runtimes: [{id: local, type: local, containerRuntime: docker}] +resources: [{id: workstation, runtimeId: local, cpuCores: 4, memoryBytes: 4294967296}] + diff --git a/document-ai/document-intelligence/run.sh b/document-ai/document-intelligence/run.sh new file mode 100755 index 0000000..98818a7 --- /dev/null +++ b/document-ai/document-intelligence/run.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +root=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd); rm -rf "$root/outputs" +python3 "$root/scripts/pipeline.py" --output "$root/outputs" --stage all +"$root/validate.sh" + diff --git a/document-ai/document-intelligence/scope.yaml b/document-ai/document-intelligence/scope.yaml new file mode 100644 index 0000000..e0ab019 --- /dev/null +++ b/document-ai/document-intelligence/scope.yaml @@ -0,0 +1,4 @@ +id: document-intelligence-scope +name: Document Intelligence local scope +environmentVersionIds: [document-intelligence-v1] + diff --git a/document-ai/document-intelligence/scripts/pipeline.py b/document-ai/document-intelligence/scripts/pipeline.py new file mode 100644 index 0000000..10ac1cf --- /dev/null +++ b/document-ai/document-intelligence/scripts/pipeline.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +import argparse, csv, hashlib, json +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +def dump(p,v): p.parent.mkdir(parents=True,exist_ok=True); p.write_text(json.dumps(v,indent=2)+"\n") +def ingest(out): + docs=[] + for p in sorted((ROOT/'data/documents').glob('*.txt')): + text=p.read_text(); docs.append({'file':p.name,'sha256':hashlib.sha256(text.encode()).hexdigest(),'text':text}) + dump(out/'ingested.json',docs) +def classify(out): + docs=json.loads((out/'ingested.json').read_text()) + dump(out/'classified.json',[{**d,'type':next(x.split(':',1)[1].strip().lower() for x in d['text'].splitlines() if x.startswith('DOCUMENT:'))} for d in docs]) +def extract(out): + result=[] + for d in json.loads((out/'classified.json').read_text()): + fields={}; items=[] + for line in d['text'].splitlines(): + if ': ' not in line: continue + key,val=line.split(': ',1) + if key=='ITEM': + name,qty,price=[x.strip() for x in val.split('|')]; items.append({'name':name,'quantity':int(qty),'unit_price':float(price)}) + else: fields[key.lower()]=val + result.append({'file':d['file'],'sha256':d['sha256'],'type':d['type'],'fields':fields,'items':items}) + dump(out/'extracted.json',result) +def reconcile(out): + docs=json.loads((out/'extracted.json').read_text()); rows=[]; seen=set() + for d in docs: + if d['type']!='invoice': continue + f=d['fields']; subtotal=sum(i['quantity']*i['unit_price'] for i in d['items']); total=float(f['total']); tax=float(f['tax']); iid=f['invoice_id'] + rows.append({'invoice_id':iid,'supplier':f['supplier'],'subtotal':subtotal,'tax':tax,'total':total,'balanced':abs(subtotal+tax-total)<.01,'duplicate':iid in seen}); seen.add(iid) + dump(out/'reconciliation.json',rows) + with (out/'invoice-summary.csv').open('w',newline='') as fp: + w=csv.DictWriter(fp,fieldnames=list(rows[0])); w.writeheader(); w.writerows(rows) +def report(out): + rows=json.loads((out/'reconciliation.json').read_text()); ok=all(r['balanced'] and not r['duplicate'] for r in rows) + dump(out/'report.json',{'status':'passed' if ok else 'failed','documents':len(json.loads((out/'ingested.json').read_text())),'invoices':len(rows),'gross_total':sum(r['total'] for r in rows)}) + artifacts=['ingested.json','classified.json','extracted.json','reconciliation.json','invoice-summary.csv','report.json'] + dump(out/'manifest.json',{'showcase':'document-intelligence','status':'success' if ok else 'failed','artifacts':artifacts}) +STAGES={'ingest':ingest,'classify':classify,'extract':extract,'reconcile':reconcile,'report':report} +def main(): + p=argparse.ArgumentParser();p.add_argument('--stage',choices=[*STAGES,'all'],default='all');p.add_argument('--output',required=True);a=p.parse_args();o=Path(a.output);o.mkdir(parents=True,exist_ok=True) + for s in list(STAGES) if a.stage=='all' else [a.stage]: STAGES[s](o) +if __name__=='__main__':main() + diff --git a/document-ai/document-intelligence/topology.yaml b/document-ai/document-intelligence/topology.yaml new file mode 100644 index 0000000..34d62cf --- /dev/null +++ b/document-ai/document-intelligence/topology.yaml @@ -0,0 +1,3 @@ +id: document-intelligence-topology +name: Document Intelligence local topology +links: [] diff --git a/document-ai/document-intelligence/validate.sh b/document-ai/document-intelligence/validate.sh new file mode 100755 index 0000000..c65e125 --- /dev/null +++ b/document-ai/document-intelligence/validate.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu +root=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +python3 - "$root" <<'PY' +import json,sys +from pathlib import Path +r=Path(sys.argv[1]);m=json.loads((r/'outputs/manifest.json').read_text());q=json.loads((r/'outputs/report.json').read_text()) +assert m['status']=='success' and q['documents']==3 and q['invoices']==2 and q['gross_total']==8262.75 +assert all((r/'outputs'/x).is_file() for x in m['artifacts']);print('validated document-intelligence') +PY + diff --git a/document-ai/document-intelligence/workflow.yaml b/document-ai/document-intelligence/workflow.yaml new file mode 100644 index 0000000..8f5b3ab --- /dev/null +++ b/document-ai/document-intelligence/workflow.yaml @@ -0,0 +1,11 @@ +name: document-intelligence +spec: + namespace: showcase + image: ghcr.io/uffescience/akoflow-examples/document-intelligence:v1 + activities: + - {name: ingest, runtime: local, command: {entrypoint: python, arguments: ["/app/scripts/pipeline.py", "--stage", ingest, "--output", "."], workingDirectory: /workspace}, cpuLimit: "1", memoryLimit: 256Mi} + - {name: classify, runtime: local, command: {entrypoint: python, arguments: ["/app/scripts/pipeline.py", "--stage", classify, "--output", "."], workingDirectory: /workspace}, cpuLimit: "1", memoryLimit: 256Mi, dependsOn: [ingest]} + - {name: extract, runtime: local, command: {entrypoint: python, arguments: ["/app/scripts/pipeline.py", "--stage", extract, "--output", "."], workingDirectory: /workspace}, cpuLimit: "2", memoryLimit: 512Mi, dependsOn: [classify]} + - {name: reconcile, runtime: local, command: {entrypoint: python, arguments: ["/app/scripts/pipeline.py", "--stage", reconcile, "--output", "."], workingDirectory: /workspace}, cpuLimit: "1", memoryLimit: 256Mi, dependsOn: [extract]} + - {name: report, runtime: local, command: {entrypoint: python, arguments: ["/app/scripts/pipeline.py", "--stage", report, "--output", "."], workingDirectory: /workspace}, cpuLimit: "1", memoryLimit: 256Mi, dependsOn: [reconcile]} +