"""usage: python3 merge_comments.py <research_dir>

Join raw comments with every coding pass (analysis/coded-*.jsonl) into analysis/comments-coded.json (one row per
comment) and analysis/comment-coding.json (denominators). Refuses (exit 1) — nothing is "mostly coded":
  - a comment with no code (every fetched comment is read and coded; no sampling)
  - a code for a cid that is not in raw/comments-all.jsonl (wrong batch or a typo), or the same cid coded twice
  - a category outside CATS
  - the channel owner's reply not coded 'owner', or an audience comment coded 'owner'
Owner replies are kept and read but are never in the audience denominator or the negatives."""
import collections, glob, json, os, sys

CATS = {'praise', 'discussion', 'question', 'request', 'criticism', 'misunderstanding', 'personal_story',
        'giveaway_entry', 'off_topic', 'spam_promo', 'owner', 'other'}


def merge(B):
    raw = [json.loads(l) for l in open(f'{B}/raw/comments-all.jsonl')]
    raw_ids = {r['cid'] for r in raw}
    codes, errors = {}, []
    for p in sorted(glob.glob(f'{B}/analysis/coded-*.jsonl')):
        name = os.path.basename(p)[len('coded-'):-len('.jsonl')]
        for n, l in enumerate(open(p), 1):
            if not l.strip(): continue
            c = json.loads(l); c['pass'] = name
            if c['cid'] in codes:
                errors.append(f"duplicate code {c['cid']} in {name} and {codes[c['cid']]['pass']}")
            elif c['cid'] not in raw_ids:
                errors.append(f"code for unknown cid {c['cid']} ({name}:{n})")
            elif c.get('cat') not in CATS:
                errors.append(f"invalid category {c.get('cat')!r} for {c['cid']} ({name}:{n})")
            else:
                codes[c['cid']] = c
    vids = {v['id']: v for v in json.load(open(f'{B}/analysis/videos.json'))}
    rows, missing = [], []
    for r in raw:
        c = codes.get(r['cid'])
        if not c: missing.append(r['cid']); continue
        if r['is_owner'] and c['cat'] != 'owner':
            errors.append(f"owner reply {r['cid']} coded {c['cat']!r} (must be 'owner', excluded from audience)")
        if not r['is_owner'] and c['cat'] == 'owner':
            errors.append(f"audience comment {r['cid']} coded 'owner'")
        v = vids.get(r['video'], {})
        rows.append({**r, 'cat': c['cat'], 'tags': c.get('tags', []), 'neg': bool(c.get('neg', False)) and not r['is_owner'],
                     'note': c.get('note', ''), 'pass': c['pass'], 'title': v.get('title'), 'fmt': v.get('fmt'),
                     'upload_date': v.get('upload_date')})
    aud = [r for r in rows if not r['is_owner']]
    summary = {'fetched': len(raw), 'coded': len(codes), 'joined': len(rows), 'uncoded': len(missing),
               'owner_replies': sum(r['is_owner'] for r in raw), 'audience': sum(not r['is_owner'] for r in raw),
               'audience_coded': len(aud), 'audience_negative': sum(r['neg'] for r in aud),
               'by_category': dict(collections.Counter(r['cat'] for r in aud).most_common()),
               'videos_with_comments': len({r['video'] for r in rows}), 'passes': sorted({c['pass'] for c in codes.values()}),
               'errors': errors + ([f'{len(missing)} comments have no code yet'] if missing else [])}
    return rows, summary


if __name__ == '__main__':
    if len(sys.argv) != 2: sys.exit(__doc__)
    B = sys.argv[1]
    rows, s = merge(B)
    json.dump(rows, open(f'{B}/analysis/comments-coded.json', 'w'), ensure_ascii=False)
    json.dump(s, open(f'{B}/analysis/comment-coding.json', 'w'), ensure_ascii=False, indent=1)
    print(f"fetched {s['fetched']} = audience {s['audience']} + owner replies {s['owner_replies']}; "
          f"coded {s['coded']}, uncoded {s['uncoded']}; audience negatives {s['audience_negative']}")
    print(s['by_category'])
    if s['errors']:
        for e in s['errors'][:20]: print('✗', e, file=sys.stderr)
        sys.exit(f"{len(s['errors'])} coding error(s) — every fetched comment is read and coded once (no sampling)")