"""YouTube search and transcript extraction via yt-dlp for the v3.0.0 pipeline.

Uses yt-dlp (https://github.com/yt-dlp/yt-dlp) for both YouTube search and
transcript extraction. No API keys needed — just have yt-dlp installed.

Inspired by Peter Steinberger's toolchain approach (yt-dlp + summarize CLI).
"""

import json
import math
import os
import re
import shlex
import shutil
import sys
import tempfile
import urllib.error
import urllib.request
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import Any, Dict, List, Optional, Set, Tuple

# Depth configurations: how many videos to search / transcribe
DEPTH_CONFIG = {
    "quick": 6,
    "default": 8,
    "deep": 40,
}

TRANSCRIPT_LIMITS = {
    "quick": 0,
    "default": 2,
    "deep": 8,
}

# Max words to keep from each transcript
TRANSCRIPT_MAX_WORDS = 5000

from . import http, log, subproc
from .relevance import token_overlap_relevance as _compute_relevance


def extract_transcript_highlights(transcript: str, topic: str, limit: int = 5) -> list[str]:
    """Extract quotable highlights from a YouTube transcript.

    Filters filler (subscribe, welcome back, etc.), scores sentences by
    specificity (numbers, proper nouns, topic relevance), and returns
    the top highlights.
    """
    if not transcript:
        return []

    sentences = re.split(r'(?<=[.!?])\s+', transcript)

    # Fallback for punctuation-free transcripts (common with auto-captions):
    # chunk into ~20-word segments so they pass the 8-50 word filter.
    if len(sentences) <= 1 and len(transcript.split()) > 50:
        words = transcript.split()
        sentences = [' '.join(words[i:i+20]) for i in range(0, len(words), 20)]

    filler = [
        r"^(hey |hi |what's up|welcome back|in today's video|don't forget to)",
        r"(subscribe|like and comment|hit the bell|check out the link|down below)",
        r"^(so |and |but |okay |alright |um |uh )",
        r"(thanks for watching|see you (next|in the)|bye)",
    ]

    topic_words = [w.lower() for w in topic.lower().split() if len(w) > 2]

    candidates = []
    for sent in sentences:
        sent = sent.strip()
        words = sent.split()
        if len(words) < 8 or len(words) > 50:
            continue
        if any(re.search(p, sent, re.IGNORECASE) for p in filler):
            continue

        score = 0
        if re.search(r'\d', sent):
            score += 2
        if re.search(r'[A-Z][a-z]+', sent):
            score += 1
        if '?' in sent:
            score += 1
        sent_lower = sent.lower()
        if any(w in sent_lower for w in topic_words):
            score += 2

        candidates.append((score, sent))

    candidates.sort(key=lambda x: -x[0])
    return [sent for _, sent in candidates[:limit]]


def _log(msg: str):
    log.source_log("YouTube", msg, tty_only=False)


def is_ytdlp_installed() -> bool:
    """Check if yt-dlp is available locally, or if SSH routing is configured.

    When LAST30DAYS_YOUTUBE_SSH_HOST is set, returns True without a local check —
    yt-dlp lives on the remote host. Failures surface naturally on first use.
    """
    if _ytdlp_ssh_host():
        return True
    return shutil.which("yt-dlp") is not None


# Host aliases must be plain hostnames / SSH config aliases — no flags, no
# shell metacharacters. Rejects any value that could be reinterpreted by ssh
# (or the surrounding shell) as something other than a destination.
_SSH_HOST_ALIAS_RE = re.compile(r"^[a-zA-Z0-9._-]+$")


def _ytdlp_ssh_host() -> Optional[str]:
    """Return SSH host alias if yt-dlp should be routed via SSH, else None.

    Set LAST30DAYS_YOUTUBE_SSH_HOST=<ssh-alias> (e.g. 'macmini') in the environment
    to route yt-dlp through SSH for residential IP egress. This bypasses
    YouTube's bot-wall on datacenter IPs (Hetzner, DigitalOcean, AWS, etc.)
    where ytsearch returns 0 results regardless of cookies.

    The remote host must have yt-dlp installed and reachable via the named
    SSH alias (configured in ~/.ssh/config). On macOS hosts with Homebrew,
    add brew shellenv to ~/.zshenv (not just ~/.zprofile) so non-login SSH
    shells find yt-dlp on PATH.

    Validation: host value must match ``[A-Za-z0-9._-]+``. Anything starting
    with ``-`` or containing shell/SSH metacharacters is rejected with a
    stderr warning and treated as unset, so a misconfigured or attacker-
    controlled value can't slip through as an SSH option flag or proxy command.
    The ``--`` option terminator in ``_wrap_ytdlp_cmd`` is a second line of
    defense; this regex closes the door on the env var ever reaching ssh
    in the first place.

    To use a value from ~/.config/last30days/.env, export it into the
    environment before invoking the engine, e.g. in a wrapper:
        set -a; source ~/.config/last30days/.env; set +a
        python3 last30days.py "..."
    """
    host = os.environ.get("LAST30DAYS_YOUTUBE_SSH_HOST", "").strip()
    if not host:
        return None
    if not _SSH_HOST_ALIAS_RE.match(host):
        sys.stderr.write(
            f"[youtube_yt] WARNING: LAST30DAYS_YOUTUBE_SSH_HOST={host!r} "
            "does not look like a plain hostname/alias; ignoring. "
            "Expected pattern: letters, digits, dot, underscore, hyphen.\n"
        )
        return None
    return host


def _wrap_ytdlp_cmd(cmd: List[str]) -> List[str]:
    """Wrap a yt-dlp command list with `ssh <host>` when SSH routing is set.

    Args are shell-quoted to survive the remote shell. Uses BatchMode=yes so
    a misconfigured key fails fast instead of hanging on a password prompt.
    The `--` option terminator prevents an SSH option-injection if
    LAST30DAYS_YOUTUBE_SSH_HOST were ever set to a value starting with `-`.
    """
    host = _ytdlp_ssh_host()
    if not host:
        return cmd
    remote_cmd = " ".join(shlex.quote(a) for a in cmd)
    return ["ssh", "-o", "BatchMode=yes", "--", host, remote_cmd]


def _extract_core_subject(topic: str) -> str:
    """Extract core subject from verbose query for YouTube search.

    NOTE: 'tips', 'tricks', 'tutorial', 'guide', 'review', 'reviews'
    are intentionally KEPT — they're YouTube content types that improve search.
    """
    from .query import extract_core_subject
    # YouTube-specific noise set: smaller than default, keeps content-type words
    _YT_NOISE = frozenset({
        'best', 'top', 'good', 'great', 'awesome', 'killer',
        'latest', 'new', 'news', 'update', 'updates',
        'trending', 'hottest', 'popular', 'viral',
        'practices', 'features',
        'recommendations', 'advice',
        'prompt', 'prompts', 'prompting',
        'methods', 'strategies', 'approaches',
        # Temporal/meta words — planner generates these but they don't
        # appear in YouTube titles, so strip them for better search.
        'last', 'days', 'recent', 'recently', 'month', 'week',
        'january', 'february', 'march', 'april', 'may', 'june',
        'july', 'august', 'september', 'october', 'november', 'december',
        '2025', '2026', '2027',
        'music', 'public', 'appearances', 'developments', 'discussions', 'coverage',
    })
    return extract_core_subject(topic, noise=_YT_NOISE)


def _infer_query_intent(topic: str) -> str:
    """Tiny local intent classifier for YouTube query expansion."""
    text = topic.lower().strip()
    if re.search(r"\b(vs|versus|compare|difference between)\b", text):
        return "comparison"
    if re.search(r"\b(how to|tutorial|guide|setup|step by step|deploy|install|configure|troubleshoot|error|fix|debug)\b", text):
        return "how_to"
    if re.search(r"\b(thoughts on|worth it|should i|opinion|review)\b", text):
        return "opinion"
    if re.search(r"\b(pricing|feature|features|best .* for)\b", text):
        return "product"
    return "breaking_news"


def expand_youtube_queries(topic: str, depth: str) -> List[str]:
    """Generate multiple YouTube search queries from a topic.

    Mirrors reddit.py's expand_reddit_queries() pattern:
    1. Extract core subject (strip noise words)
    2. Include original topic if different from core
    3. Add intent-specific OR-joined content-type variants
    4. Cap by depth: 1 for quick, 2 for default, 3 for deep

    Returns 1-3 query strings depending on depth.
    """
    core = _extract_core_subject(topic)
    queries = [core]

    # Include cleaned original topic as variant if different from core
    original_clean = topic.strip().rstrip('?!.')
    if core.lower() != original_clean.lower() and len(original_clean.split()) <= 8:
        queries.append(original_clean)

    qtype = _infer_query_intent(topic)

    # Intent-specific YouTube content-type variants
    if qtype == "opinion":
        queries.append(f"{core} review OR reaction OR breakdown")
    elif qtype == "product":
        queries.append(f"{core} review OR comparison OR unboxing")
    elif qtype == "comparison":
        queries.append(f"{core} vs OR compared OR head to head")
    elif qtype == "how_to":
        queries.append(f"{core} tutorial OR guide OR explained")
    else:
        # breaking_news / general — YouTube content types
        queries.append(f"{core} review OR reaction OR breakdown")

    # Deep depth: add full-length content variant
    if depth == "deep":
        queries.append(f"{core} full OR complete OR official")

    # Cap by depth budget
    caps = {"quick": 1, "default": 2, "deep": 3}
    cap = caps.get(depth, 2)
    return queries[:cap]


def search_youtube(
    topic: str,
    from_date: str,
    to_date: str,
    depth: str = "default",
) -> Dict[str, Any]:
    """Search YouTube via yt-dlp. No API key needed.

    Args:
        topic: Search topic
        from_date: Start date (YYYY-MM-DD)
        to_date: End date (YYYY-MM-DD)
        depth: 'quick', 'default', or 'deep'

    Returns:
        Dict with 'items' list of video metadata dicts.
    """
    if not is_ytdlp_installed():
        return {"items": [], "error": "yt-dlp not installed"}

    count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
    core_topic = _extract_core_subject(topic)

    _log(f"Searching YouTube for '{core_topic}' (since {from_date}, count={count})")

    # yt-dlp search with full metadata (no --flat-playlist so dates are real).
    # NOTE: --dateafter intentionally omitted — YouTube search returns
    # relevance-sorted results and strict date filtering returns 0 for
    # evergreen topics. Python soft filter (below) handles date filtering.
    cmd = [
        "yt-dlp",
        "--ignore-config",
        "--no-cookies-from-browser",
        f"ytsearch{count}:{core_topic}",
        "--dump-json",
        "--no-warnings",
        "--no-download",
    ]
    cmd = _wrap_ytdlp_cmd(cmd)

    try:
        result = subproc.run_with_timeout(cmd, timeout=120)
    except subproc.SubprocTimeout:
        _log("YouTube search timed out (120s)")
        return {"items": [], "error": "Search timed out"}
    except FileNotFoundError:
        return {"items": [], "error": "yt-dlp not found"}

    stdout = result.stdout
    if not stdout.strip():
        _log("YouTube search returned 0 results")
        return {"items": []}

    # Parse JSON-per-line output
    items = []
    for line in stdout.strip().split("\n"):
        line = line.strip()
        if not line:
            continue
        try:
            video = json.loads(line)
        except json.JSONDecodeError:
            continue

        video_id = video.get("id", "")
        view_count = video.get("view_count") if video.get("view_count") is not None else 0
        like_count = video.get("like_count") if video.get("like_count") is not None else 0
        comment_count = video.get("comment_count") if video.get("comment_count") is not None else 0
        upload_date = video.get("upload_date", "")  # YYYYMMDD

        # Convert YYYYMMDD to YYYY-MM-DD
        date_str = None
        if upload_date and len(upload_date) == 8:
            date_str = f"{upload_date[:4]}-{upload_date[4:6]}-{upload_date[6:8]}"

        description = str(video.get("description", ""))[:500]
        items.append({
            "video_id": video_id,
            "title": video.get("title", ""),
            "url": f"https://www.youtube.com/watch?v={video_id}",
            "channel_name": video.get("channel", video.get("uploader", "")),
            "date": date_str,
            "engagement": {
                "views": view_count,
                "likes": like_count,
                "comments": comment_count,
            },
            "duration": video.get("duration"),
            "relevance": _compute_relevance(core_topic, f"{video.get('title', '')} {description}"),
            "why_relevant": f"YouTube: {video.get('title', core_topic)[:60]}",
            "description": description,
        })

    # Soft date filter: prefer recent items but fall back to all if too few
    recent = [i for i in items if i["date"] and i["date"] >= from_date]
    if len(recent) >= 3:
        items = recent
        _log(f"Found {len(items)} videos within date range")
    else:
        _log(f"Found {len(items)} videos ({len(recent)} within date range, keeping all)")

    # Sort by views descending
    items.sort(key=lambda x: x["engagement"]["views"], reverse=True)

    return {"items": items}


def _clean_vtt(vtt_text: str) -> str:
    """Convert VTT subtitle format to clean plaintext."""
    # Strip VTT header
    text = re.sub(r'^WEBVTT.*?\n\n', '', vtt_text, flags=re.DOTALL)
    # Strip timestamps
    text = re.sub(r'\d{2}:\d{2}:\d{2}\.\d{3}\s*-->\s*\d{2}:\d{2}:\d{2}\.\d{3}.*\n', '', text)
    # Strip position/alignment tags
    text = re.sub(r'<[^>]+>', '', text)
    # Strip cue numbers
    text = re.sub(r'^\d+\s*$', '', text, flags=re.MULTILINE)
    # Deduplicate overlapping lines
    lines = text.strip().split('\n')
    seen = set()
    unique = []
    for line in lines:
        stripped = line.strip()
        if stripped and stripped not in seen:
            seen.add(stripped)
            unique.append(stripped)
    return re.sub(r'\s+', ' ', ' '.join(unique)).strip()


_YT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"


def _fetch_transcript_direct(
    video_id: str,
    timeout: int = 30,
    status: Optional[Dict[str, Any]] = None,
) -> Optional[str]:
    """Fetch YouTube transcript via direct HTTP without yt-dlp.

    Scrapes the watch page HTML for the captions track URL in
    ytInitialPlayerResponse, then fetches the VTT subtitle file.

    Args:
        video_id: YouTube video ID
        timeout: HTTP request timeout in seconds
        status: Optional dict mutated to record per-video signals. Sets
            ``status["no_caption_tracks"] = True`` when the player response
            confirms the uploader has no caption tracks (vs. fetch failure).

    Returns:
        Raw VTT text, or None if captions are unavailable.
    """
    watch_url = f"https://www.youtube.com/watch?v={video_id}"
    headers = {
        "User-Agent": _YT_USER_AGENT,
        "Accept-Language": "en-US,en;q=0.9",
    }

    # Step 1: Fetch the watch page HTML
    req = urllib.request.Request(watch_url, headers=headers)
    try:
        with urllib.request.urlopen(req, timeout=timeout) as resp:
            html = resp.read().decode("utf-8", errors="replace")
    except (urllib.error.URLError, urllib.error.HTTPError, OSError, TimeoutError) as exc:
        _log(f"Direct transcript: failed to fetch watch page for {video_id}: {exc}")
        return None

    # Step 2: Extract captions URL from ytInitialPlayerResponse
    # YouTube embeds this as a JS variable in the page HTML
    match = re.search(
        r'ytInitialPlayerResponse\s*=\s*(\{.+?\})\s*;(?:\s*var\s|\s*<\/script>)',
        html,
    )
    if not match:
        # Fallback: try the JSON embedded in the script tag
        match = re.search(
            r'var\s+ytInitialPlayerResponse\s*=\s*(\{.+?\})\s*;',
            html,
        )
    if not match:
        _log(f"Direct transcript: no ytInitialPlayerResponse found for {video_id}")
        return None

    try:
        player_response = json.loads(match.group(1))
    except json.JSONDecodeError:
        _log(f"Direct transcript: failed to parse ytInitialPlayerResponse for {video_id}")
        return None

    # Navigate to caption tracks
    captions = player_response.get("captions", {})
    renderer = captions.get("playerCaptionsTracklistRenderer", {})
    caption_tracks = renderer.get("captionTracks", [])

    if not caption_tracks:
        _log(f"Direct transcript: no caption tracks for {video_id}")
        if status is not None:
            status["no_caption_tracks"] = True
        return None

    # Find English track (prefer exact 'en', then any en variant, then first track)
    base_url = None
    for track in caption_tracks:
        lang = track.get("languageCode", "")
        if lang == "en":
            base_url = track.get("baseUrl")
            break
    if not base_url:
        for track in caption_tracks:
            lang = track.get("languageCode", "")
            if lang.startswith("en"):
                base_url = track.get("baseUrl")
                break
    if not base_url:
        # Fall back to first available track
        base_url = caption_tracks[0].get("baseUrl")
    if not base_url:
        _log(f"Direct transcript: no baseUrl in caption tracks for {video_id}")
        return None

    # Step 3: Fetch the VTT subtitle file
    sep = "&" if "?" in base_url else "?"
    vtt_url = f"{base_url}{sep}fmt=vtt"
    vtt_req = urllib.request.Request(vtt_url, headers=headers)
    try:
        with urllib.request.urlopen(vtt_req, timeout=timeout) as resp:
            vtt_text = resp.read().decode("utf-8", errors="replace")
    except (urllib.error.URLError, urllib.error.HTTPError, OSError, TimeoutError) as exc:
        _log(f"Direct transcript: failed to fetch VTT for {video_id}: {exc}")
        return None

    if not vtt_text or not vtt_text.strip():
        return None

    return vtt_text


def _fetch_transcript_ytdlp(video_id: str, temp_dir: str) -> Optional[str]:
    """Fetch transcript using yt-dlp (original implementation).

    Args:
        video_id: YouTube video ID
        temp_dir: Temporary directory for subtitle files

    Returns:
        Raw VTT text, or None if no captions available.
    """
    cmd = [
        "yt-dlp",
        "--ignore-config",
        "--no-cookies-from-browser",
        "--write-auto-subs",
        "--sub-lang", "en",
        "--sub-format", "vtt",
        "--skip-download",
        "--no-warnings",
        "-o", f"{temp_dir}/%(id)s",
        f"https://www.youtube.com/watch?v={video_id}",
    ]

    try:
        subproc.run_with_timeout(cmd, timeout=30)
    except subproc.SubprocTimeout:
        return None
    except FileNotFoundError:
        return None

    # yt-dlp may save as .en.vtt or .en-orig.vtt
    vtt_path = Path(temp_dir) / f"{video_id}.en.vtt"
    if not vtt_path.exists():
        # Try alternate naming
        for p in Path(temp_dir).glob(f"{video_id}*.vtt"):
            vtt_path = p
            break
        else:
            return None

    try:
        return vtt_path.read_text(encoding="utf-8", errors="replace")
    except OSError:
        return None


def fetch_transcript(
    video_id: str,
    temp_dir: str,
    status: Optional[Dict[str, Any]] = None,
) -> Optional[str]:
    """Fetch auto-generated transcript for a YouTube video.

    Uses yt-dlp when available (preferred, more robust). Falls back to
    direct HTTP transcript fetching when yt-dlp is not installed.

    Args:
        video_id: YouTube video ID
        temp_dir: Temporary directory for subtitle files
        status: Optional dict mutated by the direct-HTTP path to record
            per-video signals like ``no_caption_tracks``. Used to surface a
            captions-disabled count so the quality nudge avoids false-positive
            "stale yt-dlp" flags.

    Returns:
        Plaintext transcript string, or None if no captions available.
    """
    raw_vtt = None
    # When SSH-routing is on, the yt-dlp transcript path would write a VTT
    # file on the remote host that we can't easily read back. Skip it and
    # use the HTTP transcript fallback (different YouTube endpoint, less
    # bot-walled, works fine from datacenter IPs).
    ssh_host = _ytdlp_ssh_host()
    use_ytdlp = is_ytdlp_installed() and not ssh_host
    if use_ytdlp:
        raw_vtt = _fetch_transcript_ytdlp(video_id, temp_dir)
        if not raw_vtt:
            _log(f"yt-dlp transcript failed for {video_id}, trying direct HTTP fallback")
            raw_vtt = _fetch_transcript_direct(video_id, status=status)
    else:
        if ssh_host:
            _log("SSH-routing active, using direct HTTP transcript fetch")
        else:
            _log("yt-dlp not installed, using direct HTTP transcript fetch")
        raw_vtt = _fetch_transcript_direct(video_id, status=status)

    if not raw_vtt:
        _log(f"No transcript available for {video_id} (no captions found)")
        return None

    transcript = _clean_vtt(raw_vtt)

    # Truncate to max words
    words = transcript.split()
    if len(words) > TRANSCRIPT_MAX_WORDS:
        transcript = ' '.join(words[:TRANSCRIPT_MAX_WORDS]) + '...'

    return transcript if transcript else None


def fetch_transcripts_parallel(
    video_ids: List[str],
    max_workers: int = 5,
    out_captions_disabled: Optional[Set[str]] = None,
) -> Dict[str, Optional[str]]:
    """Fetch transcripts for multiple videos in parallel.

    Args:
        video_ids: List of YouTube video IDs
        max_workers: Max parallel fetches
        out_captions_disabled: Optional set mutated to record video_ids whose
            uploader confirmed no caption tracks (vs. transient fetch failures).
            Backward-compatible: callers that don't care can omit.

    Returns:
        Dict mapping video_id to transcript text (or None).
    """
    if not video_ids:
        return {}

    _log(f"Fetching transcripts for {len(video_ids)} videos")

    results = {}
    statuses: Dict[str, Dict[str, Any]] = {vid: {} for vid in video_ids}
    with tempfile.TemporaryDirectory() as temp_dir:
        with ThreadPoolExecutor(max_workers=max_workers) as executor:
            futures = {
                executor.submit(fetch_transcript, vid, temp_dir, statuses[vid]): vid
                for vid in video_ids
            }
            for future in as_completed(futures):
                vid = futures[future]
                try:
                    results[vid] = future.result()
                except OSError as exc:
                    _log(f"Transcript fetch error for {vid}: {exc}")
                    results[vid] = None
                except Exception as exc:
                    _log(f"Unexpected transcript error for {vid}: {type(exc).__name__}: {exc}")
                    results[vid] = None

    if out_captions_disabled is not None:
        for vid, st in statuses.items():
            if st.get("no_caption_tracks"):
                out_captions_disabled.add(vid)

    got = sum(1 for v in results.values() if v)
    errors = sum(1 for v in results.values() if v is None)
    _log(f"Got transcripts for {got}/{len(video_ids)} videos ({errors} failed)")
    return results


def search_and_transcribe(
    topic: str,
    from_date: str,
    to_date: str,
    depth: str = "default",
) -> Dict[str, Any]:
    """Full YouTube search: find videos, then fetch transcripts for top results.

    Uses expand_youtube_queries() to generate multiple search queries,
    runs yt-dlp for each, and merges/deduplicates results by video ID.

    Args:
        topic: Search topic
        from_date: Start date (YYYY-MM-DD)
        to_date: End date (YYYY-MM-DD)
        depth: 'quick', 'default', or 'deep'

    Returns:
        Dict with 'items' list. Each item has a 'transcript_snippet' field.
    """
    # Step 1: Multi-query search — run yt-dlp for each expanded query
    queries = expand_youtube_queries(topic, depth)
    seen_ids: Set[str] = set()
    items: List[Dict[str, Any]] = []
    for q in queries:
        search_result = search_youtube(q, from_date, to_date, depth)
        for item in search_result.get("items", []):
            vid = item.get("video_id", "")
            if vid and vid not in seen_ids:
                seen_ids.add(vid)
                items.append(item)

    # Sort merged results by views descending
    items.sort(key=lambda x: x.get("engagement", {}).get("views", 0), reverse=True)

    if not items:
        return search_result

    # Step 2: Fetch transcripts for top videos by views.
    # Try more candidates than the limit because some videos (music videos,
    # short clips) lack captions. Attempt up to 3x the limit so we have a
    # good chance of reaching the target number of successful transcripts.
    transcript_limit = TRANSCRIPT_LIMITS.get(depth, TRANSCRIPT_LIMITS["default"])
    transcripts: Dict[str, Optional[str]] = {}
    captions_disabled_ids: Set[str] = set()
    if transcript_limit > 0:
        attempt_count = min(len(items), transcript_limit * 3)
        candidate_ids = [item["video_id"] for item in items[:attempt_count]]
        _log(f"Fetching transcripts for up to {attempt_count} videos (target: {transcript_limit}): {candidate_ids}")
        transcripts = fetch_transcripts_parallel(
            candidate_ids, out_captions_disabled=captions_disabled_ids,
        )
    else:
        _log(f"Transcript limit is 0 for depth={depth}, skipping transcript fetch")

    # Step 3: Attach transcripts and extract highlights. Mark captions_disabled
    # so quality_nudge can subtract those videos from the degraded-ratio
    # denominator (uploader-disabled captions can never produce a transcript;
    # counting them was producing false-positive stale-yt-dlp nudges).
    core_topic = _extract_core_subject(topic)
    for item in items:
        vid = item["video_id"]
        transcript = transcripts.get(vid)
        item["transcript_snippet"] = transcript or ""
        item["transcript_highlights"] = extract_transcript_highlights(
            transcript or "", core_topic,
        )
        item["captions_disabled"] = vid in captions_disabled_ids

    return {"items": items}


def parse_youtube_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
    """Parse YouTube search response to normalized format.

    Returns:
        List of item dicts ready for normalization.
    """
    return response.get("items", [])


# ---------------------------------------------------------------------------
# ScrapeCreators YouTube API support
# ---------------------------------------------------------------------------

SCRAPECREATORS_YT_BASE = "https://api.scrapecreators.com/v1/youtube"


def _total_engagement(item: Dict[str, Any]) -> int:
    """Combined engagement score for ranking which videos to enrich."""
    eng = item.get("engagement", {})
    views = eng.get("views", 0) or 0
    likes = eng.get("likes", 0) or 0
    comments = eng.get("comments", 0) or 0
    return views + likes + comments


def enrich_with_comments(
    items: List[Dict[str, Any]],
    token: str,
    max_videos: int = 3,
    max_comments: int = 5,
) -> List[Dict[str, Any]]:
    """Enrich top YouTube videos with comment data from ScrapeCreators.

    For the top N videos by engagement, fetches comments via the SC API
    and attaches them as a ``top_comments`` field on each item.

    Args:
        items: YouTube items from search_and_transcribe() or search_youtube_sc()
        token: ScrapeCreators API key
        max_videos: How many videos to enrich with comments
        max_comments: Max comments to keep per video

    Returns:
        Items list (mutated in place) with top_comments added to enriched items.
    """
    if not items or not token or max_videos <= 0:
        return items

    ranked = sorted(items, key=_total_engagement, reverse=True)
    top_items = ranked[:max_videos]
    _log(f"Enriching comments for {len(top_items)} YouTube videos")

    from concurrent.futures import ThreadPoolExecutor, as_completed

    def _enrich_one(item: dict) -> bool:
        video_id = item.get("video_id", "")
        if not video_id:
            return False
        try:
            comments = _fetch_video_comments(video_id, token, max_comments)
            if comments:
                item["top_comments"] = comments
                return True
        except Exception as exc:
            _log(f"Comment enrichment failed for {video_id}: {exc}")
        return False

    enriched_count = 0
    with ThreadPoolExecutor(max_workers=min(4, len(top_items))) as executor:
        futures = {executor.submit(_enrich_one, item): item for item in top_items}
        for future in as_completed(futures):
            if future.result():
                enriched_count += 1

    _log(f"Enriched {enriched_count}/{len(top_items)} videos with comments")
    return items


def _fetch_video_comments(
    video_id: str,
    token: str,
    max_comments: int = 5,
) -> List[Dict[str, Any]]:
    """Fetch comments for a single YouTube video via ScrapeCreators.

    Args:
        video_id: YouTube video ID
        token: ScrapeCreators API key
        max_comments: Maximum comments to return

    Returns:
        List of comment dicts with author, text, likes, date.
    """
    video_url = f"https://www.youtube.com/watch?v={video_id}"
    try:
        data = http.get(
            f"{SCRAPECREATORS_YT_BASE}/video/comments",
            params={"url": video_url},
            headers=http.scrapecreators_headers(token),
            timeout=30,
            retries=2,
        )
    except Exception as exc:
        _log(f"Comment fetch error for {video_id}: {exc}")
        return []

    raw_comments = data.get("comments", data.get("data", []))
    comments = []
    for c in raw_comments[:max_comments]:
        text = c.get("text") or c.get("body") or c.get("content", "")
        if not text:
            continue

        # SC returns author as {"name": "@handle", ...}; legacy mocks may pass a string.
        author = c.get("author") or c.get("author_name", "")
        if isinstance(author, dict):
            author = author.get("name") or author.get("handle") or ""

        # SC nests likes under engagement.likes; legacy shapes used top-level keys.
        engagement = c.get("engagement") or {}
        likes = c.get("likes")
        if likes is None:
            likes = engagement.get("likes", 0) if isinstance(engagement, dict) else 0
        if not likes:
            likes = c.get("vote_count", 0)

        date = (
            c.get("date")
            or c.get("published_at")
            or c.get("publishedTime")
            or c.get("publishedTimeText", "")
        )

        comments.append({
            "author": author,
            "text": text[:400],
            "likes": likes,
            "date": date,
        })

    return comments


def search_youtube_sc(
    topic: str,
    from_date: str,
    to_date: str,
    depth: str = "default",
    token: str = None,
) -> Dict[str, Any]:
    """Search YouTube via ScrapeCreators API (fallback when yt-dlp is unavailable).

    Uses SC keyword search to find videos and SC transcript endpoint to
    fetch transcripts. Called by pipeline.py when yt-dlp fails.

    Args:
        topic: Search topic
        from_date: Start date (YYYY-MM-DD)
        to_date: End date (YYYY-MM-DD)
        depth: 'quick', 'default', or 'deep'
        token: ScrapeCreators API key

    Returns:
        Dict with 'items' list of video metadata dicts.
    """
    if not token:
        return {"items": [], "error": "No SCRAPECREATORS_API_KEY configured"}

    count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
    core_topic = _extract_core_subject(topic)
    _log(f"Searching YouTube via ScrapeCreators for '{core_topic}' (depth={depth})")

    # Step 1: Search
    raw_items = _sc_youtube_search(core_topic, token)
    if not raw_items:
        _log("SC YouTube search returned 0 results")
        return {"items": []}

    # Parse into normalized items
    items = []
    for i, raw in enumerate(raw_items[:count]):
        video_id = (
            raw.get("id") or raw.get("video_id") or raw.get("videoId") or ""
        )
        title = raw.get("title", "")
        channel = raw.get("channel") or raw.get("channel_name") or raw.get("uploader", "")
        description = str(raw.get("description", ""))[:500]
        view_count = raw.get("view_count") or raw.get("views", 0)
        like_count = raw.get("like_count") or raw.get("likes", 0)
        comment_count = raw.get("comment_count") or raw.get("comments", 0)

        # Date: try multiple field names
        date_str = raw.get("upload_date") or raw.get("date") or raw.get("published_at", "")
        if date_str and len(date_str) == 8 and date_str.isdigit():
            date_str = f"{date_str[:4]}-{date_str[4:6]}-{date_str[6:8]}"
        elif date_str and "T" in date_str:
            date_str = date_str[:10]

        url = raw.get("url", "")
        if not url and video_id:
            url = f"https://www.youtube.com/watch?v={video_id}"

        items.append({
            "video_id": video_id,
            "title": title,
            "url": url,
            "channel_name": channel,
            "date": date_str if date_str else None,
            "engagement": {
                "views": view_count or 0,
                "likes": like_count or 0,
                "comments": comment_count or 0,
            },
            "duration": raw.get("duration"),
            "relevance": _compute_relevance(core_topic, f"{title} {description}"),
            "why_relevant": f"YouTube: {title[:60]}" if title else f"YouTube: {core_topic}",
            "description": description,
        })

    # Soft date filter
    recent = [i for i in items if i["date"] and i["date"] >= from_date]
    if len(recent) >= 3:
        items = recent
        _log(f"Found {len(items)} videos within date range")
    else:
        _log(f"Found {len(items)} videos ({len(recent)} within date range, keeping all)")

    # Sort by views
    items.sort(key=lambda x: x["engagement"]["views"], reverse=True)

    # Step 2: Fetch transcripts for top videos
    transcript_limit = TRANSCRIPT_LIMITS.get(depth, TRANSCRIPT_LIMITS["default"])
    if transcript_limit > 0 and items:
        attempt_count = min(len(items), transcript_limit * 3)
        _log(f"Fetching SC transcripts for up to {attempt_count} videos (target: {transcript_limit})")
        for item in items[:attempt_count]:
            vid = item["video_id"]
            if not vid:
                continue
            transcript = _sc_fetch_transcript(vid, token)
            item["transcript_snippet"] = transcript or ""
            item["transcript_highlights"] = extract_transcript_highlights(
                transcript or "", core_topic,
            )
    else:
        for item in items:
            item["transcript_snippet"] = ""
            item["transcript_highlights"] = []

    _log(f"SC YouTube: {len(items)} videos returned")
    return {"items": items}


def _sc_youtube_search(keyword: str, token: str) -> List[Dict[str, Any]]:
    """Call ScrapeCreators YouTube search endpoint.

    Args:
        keyword: Search keyword
        token: ScrapeCreators API key

    Returns:
        List of raw video dicts from the API.
    """
    try:
        # SC's /v1/youtube/search rejects ?keyword= with HTTP 400; the canonical
        # parameter for that endpoint is `query`. Other SC endpoints use their
        # own per-endpoint param names so this was the lone outlier.
        data = http.get(
            f"{SCRAPECREATORS_YT_BASE}/search",
            params={"query": keyword},
            headers=http.scrapecreators_headers(token),
            timeout=30,
            retries=2,
        )
        return data.get("videos", data.get("data", data.get("items", [])))
    except Exception as exc:
        _log(f"SC YouTube search error: {exc}")
        return []


def _sc_fetch_transcript(video_id: str, token: str) -> Optional[str]:
    """Fetch transcript for a YouTube video via ScrapeCreators.

    Args:
        video_id: YouTube video ID
        token: ScrapeCreators API key

    Returns:
        Plaintext transcript string, or None if unavailable.
    """
    video_url = f"https://www.youtube.com/watch?v={video_id}"
    try:
        data = http.get(
            f"{SCRAPECREATORS_YT_BASE}/video/transcript",
            params={"url": video_url},
            headers=http.scrapecreators_headers(token),
            timeout=30,
            retries=1,
        )
    except Exception as exc:
        _log(f"SC transcript error for {video_id}: {exc}")
        return None

    transcript = data.get("transcript")
    if not transcript:
        return None

    if isinstance(transcript, list):
        transcript = " ".join(str(s) for s in transcript)

    # Clean VTT formatting if present
    transcript = _clean_vtt(transcript)

    # Truncate to max words
    words = transcript.split()
    if len(words) > TRANSCRIPT_MAX_WORDS:
        transcript = " ".join(words[:TRANSCRIPT_MAX_WORDS]) + "..."

    return transcript if transcript else None
