"""Build analysis/videos.json and raw/comments-all.jsonl from raw/videos/*.info.json.

usage: python3 inventory.py <research_dir>

Format: an ID listed in raw/list-shorts.tsv is a Short; otherwise duration <= 180 s falls back to Short.
Every comment and reply is kept verbatim with its author flag (author_is_uploader = the channel owner's reply).
IDs in raw/ids.txt without an info.json are printed as missing (never silently dropped).

The public comment count comes ONLY from raw/public-counts.tsv (fetch_videos.sh writes it before fetching comments):
once comments are fetched, yt-dlp rewrites info.json comment_count to the fetched number, so it can never show a gap.
No public count → public_comment_count None (unknown), never the fetched count. Counts of 1,000+ ending in 00 are
parsed from a rounded label ("1.3K") and flagged public_count_approx."""
import glob, json, os, sys

def load_short_ids(r):
    p = os.path.join(r, 'raw', 'list-shorts.tsv')
    # first field; also accept a literal backslash-t (an unquoted --print template wrote that on 2026-10-12)
    return {l.replace('\\t', '\t').split('\t')[0].strip() for l in open(p) if l.strip()} if os.path.exists(p) else set()

def fmt_of(d, short_ids):
    if short_ids:
        return 'short' if d['id'] in short_ids else 'long'
    return 'short' if (d.get('duration') or 10**9) <= 180 else 'long'

def load_public_counts(r):
    """id -> (comment_count, epoch) from raw/public-counts.tsv; NA/None stay None."""
    p = os.path.join(r, 'raw', 'public-counts.tsv'); out = {}
    if not os.path.exists(p): return out
    for l in open(p):
        f = l.replace('\\t', '\t').rstrip('\n').split('\t')
        if len(f) < 2 or not f[0].strip(): continue
        n = int(f[1]) if f[1].strip().isdigit() else None
        out[f[0].strip()] = (n, int(f[3]) if len(f) > 3 and f[3].strip().isdigit() else None)
    return out


def approx(n):
    return n is not None and n >= 1000 and n % 100 == 0


def build(r):
    short_ids = load_short_ids(r)
    pub = load_public_counts(r)
    rows, allc = [], []
    for f in glob.glob(os.path.join(r, 'raw', 'videos', '*.info.json')):
        d = json.load(open(f))
        cs = d.get('comments') or []
        rows.append(dict(id=d['id'], title=d['title'], upload_date=d.get('upload_date'), ts=d.get('timestamp'),
                         views=d.get('view_count'), likes=d.get('like_count'),
                         public_comment_count=pub.get(d['id'], (None, None))[0],
                         public_count_approx=approx(pub.get(d['id'], (None, None))[0]),
                         public_count_epoch=pub.get(d['id'], (None, None))[1],
                         comments_fetched=len(cs), duration=d.get('duration'), fmt=fmt_of(d, short_ids),
                         description=d.get('description'), chapters=[c['title'] for c in d.get('chapters') or []],
                         tags=d.get('tags'), fetched_epoch=d.get('epoch')))
        for c in cs:
            allc.append(dict(video=d['id'], cid=c['id'], parent=c.get('parent'), author=c.get('author'),
                             is_owner=bool(c.get('author_is_uploader')), text=c.get('text'), likes=c.get('like_count'),
                             ts=c.get('timestamp'), time_text=c.get('_time_text')))
    rows.sort(key=lambda x: x['ts'] or 0, reverse=True)
    os.makedirs(os.path.join(r, 'analysis'), exist_ok=True)
    json.dump(rows, open(os.path.join(r, 'analysis', 'videos.json'), 'w'), ensure_ascii=False, indent=1)
    with open(os.path.join(r, 'raw', 'comments-all.jsonl'), 'w') as f:
        for c in allc: f.write(json.dumps(c, ensure_ascii=False) + '\n')
    ids_p = os.path.join(r, 'raw', 'ids.txt')
    missing = sorted(set(open(ids_p).read().split()) - {x['id'] for x in rows}) if os.path.exists(ids_p) else []
    gap = [(x['id'], x['public_comment_count'], x['comments_fetched']) for x in rows
           if x['public_comment_count'] is not None and x['comments_fetched'] < x['public_comment_count']]
    unknown = [x['id'] for x in rows if x['public_comment_count'] is None]
    return rows, allc, missing, gap, unknown

if __name__ == '__main__':
    if len(sys.argv) != 2: sys.exit(__doc__)
    rows, allc, missing, gap, unknown = build(sys.argv[1])
    print(f"videos {len(rows)} (long {sum(x['fmt']=='long' for x in rows)}, short {sum(x['fmt']=='short' for x in rows)})"
          f" · comments {len(allc)} (owner {sum(c['is_owner'] for c in allc)})")
    print('missing info.json:', missing or 'none')
    print('public count > fetched (capped/sorted fetch, held or deleted comments):', gap or 'none')
    print('public comment count unknown (no raw/public-counts.tsv row):', unknown or 'none')