from pathlib import Path
import re,json
items={};issues=[]
paths=list(Path('work').glob('baqarah-corpus-*.txt'))+list(Path('work').glob('bq-corpus-round-*.txt'))
for p in paths:
 text=re.sub(r'L\d+:\s?', '\n',p.read_text())
 text=re.sub(r'cite[^†]*†([^]*)',r'\1',text)
 chunks=re.split(r'\(2:(\d+):(\d+)\)',text)
 for i in range(1,len(chunks),3):
  v,w=int(chunks[i]),int(chunks[i+1]);lines=[x.strip() for x in chunks[i+2].splitlines() if x.strip()]
  if len(lines)<2:continue
  gloss=re.split(r'\s+(?:[A-Z][A-Z0-9]*|[A-Z]+\s*[–-])\s*–\s*',lines[1])[0]
  gloss=re.split(r'\s+Image\b',gloss)[0].strip()
  if re.search('[ء-ي]',gloss) or ' – ' in gloss:issues.append((v,w,gloss))
  if (v,w) in items and items[v,w]!=gloss:issues.append(('conflict',v,w,items[v,w],gloss))
  items[v,w]=gloss
verses=json.loads(Path('work/baqarah-verses.json').read_text());glosses=[];missing=[]
for v in verses:
 count=len([w for w in v['arabic'].split() if re.search('[ء-يٱ]',w)])
 row=[items.get((v['n'],w)) for w in range(1,count+1)]
 if None in row:missing.append(dict(verse=v['n'],count=count,missing=[i+1 for i,x in enumerate(row) if x is None]))
 extras=[w for n,w in items if n==v['n'] and w>count]
 if extras:issues.append(('extra',v['n'],extras))
 glosses.append(row)
Path('work/baqarah-gloss-audit.json').write_text(json.dumps(dict(words=len(items),missing=missing,issues=issues),ensure_ascii=False,indent=2))
if not missing and not issues:Path('work/baqarah-glosses.json').write_text(json.dumps(glosses,ensure_ascii=False,indent=2))
print('Files',len(paths),'Words',len(items),'Missing verses',len(missing),'Issues',len(issues));print('First issues',issues[:8]);print('Missing',[(x['verse'],len(x['missing'])) for x in missing][:20])