Datalab has released OmniParseBench, an open OCR benchmark for document parsers. It scores parsers with 16,288 yes/no tests on 2,937 pages from 2,343 documents. The pages span 92 languages. The dataset and code are public. It helps engineers answer a sharper question than a leaderboard does: which parser handles documents like mine? Datalab sells a parser, and its own top setting, Datalab accurate, ranks second on the launch leaderboard, behind Gemini 3.8 Flash.
Is it deployable? Yes. OmniParseBench is an evaluation harness, not a model. It installs with uv pip install omni-parse-bench, needs Python 3.11+, and scores outputs offline with no API keys.
TL;DR
- Size: 16,288 tests, 2,937 pages, 2,343 documents, 22 suites, 92 languages.
- Runs on: any machine with Python 3.11+. Scoring needs no network. Vendor runs need each vendor’s API key.
- Performance: the top 3 reported systems sit within 1.3 headline points.
- Best: Gemini 3.8 Flash leads at 92.9; Datalab accurate follows at 92.7.*
- Bottom line: every score is a share of pass/fail tests, sliceable by tag.
* used in building some tests
What is OmniParseBench?
OmniParseBench is a document-parsing benchmark from Datalab where every test is a yes/no question about 1 page. A parser’s score is the share of tests it passes. Datalab built it to be clear, auditable and explorable.
Why build another OCR benchmark?
Datalab names two problems with existing benchmarks. olmOCR-bench showed that pass/fail unit tests work. Datalab now calls it saturated. In its view, top systems are separated by output conventions and overfitting, not reading. Each olmOCR-bench test also sits in 1 bucket, set by its page’s source. A handwritten table gets averaged into whichever bucket its page landed in.
ParseBench from LlamaIndex splits parsing into 5 capabilities, each with its own metric. Datalab argues those scores share no unit, so they cannot be compared or combined cleanly.
How does OmniParseBench scoring work?
The final score is the mean of 3 family scores: text, tables and layout. Five test types feed those families:
- present: a text, number or equation appears (text)
- order: text A comes before text B (text)
- repeat: a line appears exactly N times (text)
- table_cell: a cell sits under given headings and beside given neighbors (tables)
- layout_kind: the block holding a line is a heading, text or table (layout)
Numbers must match digit for digit, though 1,240 matches 1240. Equations are compared by what KaTeX renders, so frac{a}{b} equals dfrac{a}{b}. Each decision is pass, fail, unsupported or error. The last 2 are left out of every score. No model judges the outputs; the scorer is open code.
A test checks one or more args, each of which is the content at one place on the page. Each arg carries tags on 5 axes: structure, rendering, role, script and language. Scripts use ISO 15924 codes; languages use ISO 639-3 codes. Tags never decide a verdict or the headline. They only select tests, so any slice gets its own score.
The table tag covers 6,767 tests on 1,000 pages from 875 documents. English covers 10,089 tests. Non-English languages cover 1,931 tests on 529 pages from 321 documents.
Every test also records its derivation, meaning how its answer was established. There are 10 derivations. They include publisher JATS XML, the PDF text layer, authored pages, olmOCR-bench tests, model panels and human review.
‘}).join(”);
function ty(i){$$(‘.ty’).forEach(function(x){x.classList.toggle(‘on’,+x.dataset.i===i)});var t=TY[i];
$(‘#tydet’).innerHTML=’‘+t[0]+’ checks that ‘+t[1]+’. Family: ‘+t[2]+’.’}
$$(‘.ty’).forEach(function(x){x.onclick=function(){ty(+x.dataset.i)}});ty(0);
/* scorer */
var base=”SummarynQuarterly revenue rose 12% on the year.nnQ2Q3n3.9CELLnnOutlook”;
var SC=[
{k:’A · README output’,html:base.replace(‘CELL’,’4.1‘),v:[‘pass’,’pass’,’fail’],why:[”,”,”no cell matching ‘4.2’ in any table”]},
{k:’B · Cell fixed’,html:base.replace(‘CELL’,’4.2‘),v:[‘pass’,’pass’,’pass’],why:[”,”,”]},
{k:’C · Headings swapped’,html:base.replace(‘CELL’,’4.1′).replace(‘Summary’,’Outlook‘).replace(/Outlook</h2>$/,’Summary‘),v:[‘pass’,’fail’,’fail’],why:[”,’Summary is not found before Outlook’,”no cell matching ‘4.2’ in any table”]}];
var TQ=[‘t1 · present: “Quarterly revenue rose 12%” appears (max_diffs 1)’,’t2 · order: “Summary” comes before “Outlook”‘,’t3 · table_cell: cell “4.2” sits under top heading “Q3″‘];
var cur=0;
$(‘#scen’).innerHTML=SC.map(function(c,i){return ‘‘}).join(”);
$$(‘#scen .btn’).forEach(function(b){b.onclick=function(){runScen(+b.dataset.i)}});
function runScen(i){cur=i;var c=SC[i];$$(‘#scen .btn’).forEach(function(b){b.setAttribute(‘aria-pressed’,+b.dataset.i===i?’true’:’false’)});
$(‘#html’).innerHTML=c.html;
$(‘#tests’).innerHTML=TQ.map(function(q,j){return ‘
why: ‘+c.why[j]+’
‘:”)+’
‘+c.v[j]+’
‘}).join(”);
var p=c.v.filter(function(x){return x===’pass’}).length;$(‘#sc’).textContent=p+’ / 3 = ‘+(p/3).toFixed(3);$(‘#scm’).style.width=”0″;
$$(‘#tests .badge’).forEach(function(b,j){setTimeout(function(){b.classList.add(‘show’)},150+j*280)});
setTimeout(function(){$(‘#scm’).style.width=(p/3*100)+’%’},950);setTimeout(post,60);}
runScen(0);
/* leaderboard */
var LB=[[‘Gemini 3.8 Flash*’,92.9,91.3,95.2,92.2],[‘Datalab accurate’,92.7,90.8,91.8,95.6,1],[‘Datalab balanced’,91.6,90.0,89.2,95.5,1],[‘Reducto’,90.8,85.4,91.2,95.8],[‘LlamaParse’,90.4,88.2,92.6,90.4],[‘Mistral OCR’,89.5,82.8,91.9,93.8],[‘Claude Sonnet 5.5*’,88.7,93.1,95.4,77.5],[‘GPT-5.6 Sol’,87.8,85.4,91.6,86.5],[‘Extend’,83.3,64.1,92.5,93.4],[‘Azure’,83.0,74.6,81.7,92.8],[‘Tesseract’,null,17.6,0.0,null]];
var M=[‘Headline’,’Text’,’Tables’,’Layout’],mi=0;
$(‘#metric’).innerHTML=M.map(function(m,i){return ‘‘}).join(”);
$$(‘#metric .btn’).forEach(function(b){b.onclick=function(){mi=+b.dataset.i;$$(‘#metric .btn’).forEach(function(x){x.setAttribute(‘aria-pressed’,x===b?’true’:’false’)});drawLB()}});
function drawLB(){var rows=LB.map(function(r){return {n:r[0],v:r[1+mi],dl:r[5]}});
rows.sort(function(a,b){return (b.v===null?-1:b.v)-(a.v===null?-1:a.v)});
$(‘#lb’).innerHTML=rows.map(function(r){var na=r.v===null;return ‘
‘+r.n+’‘+(na?(mi===0?’none’:’unsup.’):r.v.toFixed(1))+’
‘}).join(”);
requestAnimationFrame(function(){requestAnimationFrame(function(){$$(‘#lb i’).forEach(function(i){i.style.width=i.dataset.w+’%’})})});setTimeout(post,60);}
drawLB();
/* tags */
var TG={structure:[[‘table’,6767,1000,875],[‘multi_column’,3517],[‘math’,3316],[‘form’,1028]],rendering:[[‘handwriting’,923],[‘tiny’,832],[‘rotated’,824],[‘degraded’,655]],role:[[‘list’,861],[‘heading’,749],[‘header_footer’,227],[‘footnote’,104],[‘caption’,81],[‘figure’,69],[‘code’,16,6,6]],language:[[‘english’,10089,1930,1612],[‘multilingual (non-English)’,1931,529,321]]};
var gk=’structure’;
$(‘#grp’).innerHTML=Object.keys(TG).map(function(k){return ‘‘}).join(”);
$$(‘#grp .btn’).forEach(function(b){b.onclick=function(){gk=b.dataset.k;$$(‘#grp .btn’).forEach(function(x){x.setAttribute(‘aria-pressed’,x===b?’true’:’false’)});drawTags()}});
function fmt(n){return n.toLocaleString(‘en-US’)}
function drawTags(){var a=TG[gk],mx=Math.max.apply(null,a.map(function(r){return r[1]}));
$(‘#tg’).innerHTML=a.map(function(r,i){return ‘
‘+r[0]+’‘+fmt(r[1])+’
‘}).join(”);
$$(‘#tg .bar’).forEach(function(b){b.onclick=function(){tgSel(+b.dataset.i)}});
requestAnimationFrame(function(){requestAnimationFrame(function(){$$(‘#tg i’).forEach(function(i){i.style.width=i.dataset.w+’%’})})});tgSel(0);setTimeout(post,60);}
function tgSel(i){var r=TG[gk][i];$$(‘#tg .bar’).forEach(function(b){b.classList.toggle(‘sel’,+b.dataset.i===i)});
var t=”“+r[0]+’: ‘+fmt(r[1])+’ tests’;
if(r[2])t+=’ on ‘+fmt(r[2])+’ pages from ‘+fmt(r[3])+’ documents.’;else t+=’. Page and document counts for this tag are shown in Datalab’s fig. 2.’;
if(r[0]===’code’)t+=’ A thin slice: the scorer reports counts so you can see it.’;
$(‘#tgdet’).innerHTML=t;}
drawTags();
/* copy */
$$(‘.cp’).forEach(function(b){b.onclick=function(){var tx=b.previousElementSibling.textContent;
try{navigator.clipboard.writeText(tx).then(function(){b.textContent=”Copied”},function(){b.textContent=”Select + copy”})}catch(e){b.textContent=”Select + copy”}
setTimeout(function(){b.textContent=”Copy”},1500)}});
window.addEventListener(‘load’,post);window.addEventListener(‘resize’,post);setTimeout(post,300);
})();
