#!/bin/bash
# usage: fetch_videos.sh <research_dir> [sleep_s=2] [max_comments=all] [sort=new]
# Public data only (yt-dlp, no cookies, no login, serial) for each ID in raw/ids.txt:
# (--ignore-config: a personal yt-dlp config with cookies is never inherited, so "cookies": false below stays true)
#   1. a metadata-only read first -> raw/public-counts.tsv (id, comment_count, view_count, epoch). This is the public
#      comment count: once comments are fetched, yt-dlp rewrites info.json comment_count to the number it fetched,
#      so it can never show a gap.
#   2. info.json + comments/replies (cap and sort recorded in raw/fetch-params.json) + ko/en auto-subs -> raw/videos/.
# Serial on purpose: hundreds of parallel calls drew HTTP 429 and a bot check. Skips IDs already fetched.
# If a video hits a bot check, leave it missing; do not add cookies or log in to get past it.
# Intended for analysing your own channel. Long run (about 30 s per video with many comments): run it in the background.
set -uo pipefail
[ $# -ge 1 ] || { sed -n 2,10p "$0"; exit 2; }
R=$1; SL=${2:-2}; MAXC=${3:-all}; SORT=${4:-new}
mkdir -p "$R/raw/videos"
PC="$R/raw/public-counts.tsv"; touch "$PC"
python3 "$(dirname "$0")/fetch_params.py" "$R/raw/fetch-params.json" \
  "{\"tool\": \"yt-dlp\", \"max_comments\": \"$MAXC\", \"comment_sort\": \"$SORT\", \"cookies\": false}" || exit 2
while read -r id; do
  [ -z "$id" ] && continue
  [ -f "$R/raw/videos/$id.info.json" ] && continue
  grep -q "^$id	" "$PC" || yt-dlp --ignore-config --skip-download --print "%(id)s	%(comment_count)s	%(view_count)s	%(epoch)s" \
    "https://www.youtube.com/watch?v=$id" >> "$PC" 2>> "$R/raw/fetch.log" || echo "FAIL-COUNT $id" >> "$R/raw/fetch.log"
  yt-dlp --ignore-config --skip-download --write-info-json --write-comments --write-auto-subs --write-subs \
    --sub-langs "ko,ko-orig,en-orig" --sub-format vtt \
    --extractor-args "youtube:lang=ko;comment_sort=$SORT;max_comments=$MAXC,all,all,all" \
    -o "$R/raw/videos/%(id)s.%(ext)s" "https://www.youtube.com/watch?v=$id" >> "$R/raw/fetch.log" 2>&1 \
    || echo "FAIL $id" >> "$R/raw/fetch.log"
  sleep "$SL"
done < "$R/raw/ids.txt"
grep -c '^FAIL' "$R/raw/fetch.log" || true
echo done