"""Refresh only ClinVar artifacts; never edit a view config or gene/cCRE matrix."""
import bisect
import collections
import csv
import gzip
import hashlib
import json
from pathlib import Path
from urllib.parse import unquote

SOURCE = Path('/net/public_html/users/voy/clinvar_hg38_all.vcf.gz')
SOURCE_URL = 'https://users.abdenlab.org/voy/clinvar_hg38_all.vcf.gz'
FIELDS = ['variant_key','vcf_id','chrom','start','end','ref','alt','allele_id','annotated_gene','significance','consequence','phenotypes','review_stars','review_status']

def sha(p):
    with p.open('rb') as f:return hashlib.file_digest(f,'sha256').hexdigest()

def write_csv(p,fields,rows):
    with p.open('w',newline='') as f:
        w=csv.DictWriter(f,fieldnames=fields);w.writeheader();w.writerows(rows)

def update_many(targets, source=SOURCE):
    states={}
    for root,chrom in targets:
        root=Path(root)
        with (root/f'{chrom}.matrix.csv').open() as f:
            r=csv.reader(f);next(r);ids=[row[0] for row in r]
        assert len(ids)==len(set(ids)) and all(e.startswith(chrom+':') for e in ids)
        intervals=sorted((int(e.split(':')[1].split('-')[0]),int(e.split('-')[1]),e) for e in ids)
        starts=[a for a,b,e in intervals];maxends=[];end=0
        for a,b,e in intervals:end=max(end,b);maxends.append(end)
        f=(root/f'{chrom}.clinvar.variants.csv').open('w',newline='')
        w=csv.DictWriter(f,fieldnames=FIELDS);w.writeheader()
        states[chrom]=dict(root=root,ids=ids,intervals=intervals,starts=starts,maxends=maxends,
            file=f,writer=w,hits={e:[] for e in ids},keys=set(),significance=collections.Counter(),count=0,
            protected={n:sha(root/n) for n in [f'{chrom}.matrix.csv',f'{chrom}.ccre.matrix.csv',f'{chrom}.ccre_metadata.csv',f'{chrom}.json']})
    total=0;assembly=False
    try:
        with gzip.open(source,'rt') as f:
            for line in f:
                if line.startswith('#'):
                    assembly |= line.strip()=='##reference=GRCh38'
                    continue
                total+=1
                # Avoid parsing INFO on unrelated chromosomes.
                chrom=line.split('\t',1)[0]
                if chrom not in states:continue
                s=states[chrom];r=line.rstrip('\n').split('\t')
                assert len(r)==8
                assert not any(c in r[4] for c in '<>[]*,'), 'Unsupported variant representation'
                info={k:unquote(v) for k,v in (part.split('=',1) for part in r[7].split(';') if '=' in part)}
                start=int(r[1])-1;end=start+len(r[3]);key=f'{chrom}:{r[1]}:{r[3]}:{r[4]}'
                assert key not in s['keys'],f'Duplicate variant {key}'
                s['keys'].add(key);s['count']+=1;s['significance'][info.get('CLNSIG','')]+=1
                v=dict(variant_key=key,vcf_id=r[2],chrom=chrom,start=start,end=end,ref=r[3],alt=r[4],
                    allele_id=info.get('ALLELEID',''),annotated_gene=info.get('GENE',''),
                    significance=info.get('CLNSIG',''),consequence=info.get('CONSQNAME',''),
                    phenotypes=info.get('PHENOTYPES',''),review_stars=info.get('STARS',''),review_status=info.get('REVIEWSTATUS',''))
                s['writer'].writerow(v)
                lo=bisect.bisect_right(s['maxends'],start);hi=bisect.bisect_left(s['starts'],end)
                for a,b,e in s['intervals'][lo:hi]:
                    if a<end and b>start:s['hits'][e].append((v,min(b,end)-max(a,start)))
        assert assembly, 'Source must declare GRCh38'
    finally:
        for s in states.values():s['file'].close()
    source_hash=sha(source);reports={}
    for chrom,s in states.items():
        root=s['root'];ids=s['ids'];hits=s['hits']
        write_csv(root/f'{chrom}.clinvar.matrix.csv',['element_id','ClinVar overlap count'],
            (dict(element_id=e,**{'ClinVar overlap count':len(hits[e])}) for e in ids))
        write_csv(root/f'{chrom}.clinvar.links.csv',['element_id','overlap_bp',*FIELDS],
            (dict(element_id=e,overlap_bp=bp,**v) for e in ids for v,bp in hits[e]))
        def metadata():
            for e in ids:
                details=' || '.join(f"{v['variant_key']} [VCF ID={v['vcf_id']}; ALLELEID={v['allele_id']}; {v['significance']}; gene={v['annotated_gene']}; consequence={v['consequence']}; stars={v['review_stars']}; review={v['review_status']}; phenotype={v['phenotypes']}]" for v,bp in hits[e]) or 'No overlapping variant in the supplied ClinVar file'
                yield dict(element_id=e,clinvar_count=len(hits[e]),clinvar_details=details)
        write_csv(root/f'{chrom}.clinvar.metadata.csv',['element_id','clinvar_count','clinvar_details'],metadata())
        children=[dict(name=label,color=color,set=[[e,None] for e in ids if bool(hits[e])==present])
            for label,color,present in [('With ClinVar overlap',[255,140,0],True),('No ClinVar overlap',[110,110,110],False)]]
        (root/f'{chrom}.clinvar.obsSets.json').write_text(json.dumps(dict(version='0.1.3',datatype='obs',tree=[dict(name='ClinVar overlap',children=children)]),indent=2)+'\n')
        assert all(sha(root/n)==h for n,h in s['protected'].items())
        report=dict(source_file=source.name,source_path=str(source),source_url=SOURCE_URL,source_sha256=source_hash,
            source_records=total,**{chrom+'_records':s['count']},elements=len(ids),
            overlapping_variants=len({v['variant_key'] for hh in hits.values() for v,bp in hh}),
            elements_with_overlap=sum(bool(hh) for hh in hits.values()),element_variant_pairs=sum(map(len,hits.values())),
            max_variants_per_element=max(map(len,hits.values())),clinical_significance_counts=dict(s['significance']),
            filtering='Chromosome only; no clinical-significance or phenotype filter',
            overlap_convention='0-based half-open REF span [POS-1, POS-1+len(REF)); includes indel anchors',
            protected_files_unchanged=True,protected_sha256=s['protected'],browser_rendering_verified=False)
        (root/'clinvar.validation.json').write_text(json.dumps(report,indent=2)+'\n');reports[chrom]=report
    return reports

def update_clinvar(root,chrom,source=SOURCE):
    return update_many([(root,chrom)],source)[chrom]

if __name__=='__main__':
    import argparse
    p=argparse.ArgumentParser(description=__doc__);p.add_argument('--chrom',required=True,choices=['chr19']);p.add_argument('--source',type=Path,default=SOURCE)
    args=p.parse_args();print(json.dumps(update_clinvar(Path(__file__).resolve().parent,args.chrom,args.source),indent=2))