"""Historical corporate publications scraper — publicacoes.mj.pt.

⚠️  DORMANT — Whisper-based audio captcha bypass verified unreachable
    from our deployment.

The canonical portal for atos societários. Implemented as an ASP.NET WebForms
page with reCAPTCHA v2. In theory: drive with Playwright, switch to audio
mode, transcribe with local Whisper (~85-95% success per IEEE EuroS&PW 2025).

Reality (tested 2026-04-20): **Google's anti-bot gate blocks before the
audio is served**. Even headless Chromium patched with `patchright` (the
strongest Python stealth fork) gets `"Tente mais tarde — o seu computador
ou rede pode estar a enviar consultas automatizadas"` before the audio
source element is ever rendered. The Whisper layer is unreachable.

Root cause: Google's reputation system combines browser fingerprint with
**IP reputation** and **prior navigation history**. From a fresh datacenter
IP (LXC VPS-like block) with no browsing history, our session scores too
suspicious regardless of stealth patches.

To revive, one of these is needed:
  1. Residential proxy pool (BrightData / SmartProxy) — ~€3-10/mês
     → IP reputation fixed, audio mode served, Whisper then works
  2. Run this scraper from a real user browser via a desktop companion tool
     → cookie handoff to the server for result parsing only
  3. Contract a captcha-solving service (2Captcha/Anti-Captcha) — ~€3/1000
     → still probably hits the reputation gate first

For now the code below is kept as a structural reference; it's not imported
by the scheduler or admin routes. Re-enable by:
  * adding `playwright>=1.48`, `faster-whisper>=1.0`, and `patchright` to
    pyproject.toml
  * restoring the Dockerfile's `playwright install --with-deps chromium` step
  * re-adding the imports + `run_mj_backfill` function in scheduler.py
  * wiring the residential proxy into `fetch_mj_publications` (httpx proxies=…
    or PW context proxy= kwarg)

NorthData-scale scraping of this portal works because they have 20 years
of IP trust, rotating pools, and distributed instances. We're starting
fresh on a single IP — the economics don't favour us here.
"""
import asyncio
import logging
import re
from datetime import date, datetime
from typing import Any

logger = logging.getLogger(__name__)

SEARCH_URL = "https://publicacoes.mj.pt/pesquisa.aspx"
PAGE_TIMEOUT_MS = 45_000
MAX_PAGES = 25  # safety cap; a typical company has 0-50 publications
REQUEST_DELAY_MS = 1500  # between pagination clicks — polite

# Field names confirmed from prior recon (earlier agent pulled pubmj_pesquisa.html)
F_NIF = "ctl00$ContentPlaceHolderMain$txtDadosPubNif"
F_BTN = "ctl00$ContentPlaceHolderMain$btSearch"

# Dates in the GridView come as dd/mm/yyyy
_DATE_RE = re.compile(r"(\d{1,2})/(\d{1,2})/(\d{4})")


def _parse_pt_date(s: str | None) -> date | None:
    if not s:
        return None
    m = _DATE_RE.search(s)
    if not m:
        return None
    d, mm, y = m.groups()
    try:
        return date(int(y), int(mm), int(d))
    except ValueError:
        return None


async def fetch_mj_publications(nif: str) -> list[dict[str, Any]]:
    """Run a full NIPC search and return every publication row found. Returns
    [] when the company has no publications OR the scrape fails (logged,
    non-fatal). Designed to be called from an async scheduler worker."""
    from playwright.async_api import TimeoutError as PwTimeoutError
    from playwright.async_api import async_playwright

    from app.services.recaptcha_audio import solve_audio_recaptcha

    results: list[dict[str, Any]] = []
    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=True, args=["--no-sandbox"])
        try:
            ctx = await browser.new_context(
                locale="pt-PT",
                user_agent=(
                    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
                    "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
                ),
            )
            # Hide the webdriver flag — helps the initial silent reCAPTCHA check
            await ctx.add_init_script(
                "Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"
            )
            page = await ctx.new_page()
            page.set_default_timeout(PAGE_TIMEOUT_MS)

            await page.goto(SEARCH_URL, wait_until="domcontentloaded")
            # Wait for the ASP.NET form to wire up
            await page.wait_for_timeout(2500)

            # Fill NIPC
            await page.fill(f"input[name='{F_NIF}']", nif)

            # Solve the captcha
            solved = await solve_audio_recaptcha(page)
            if not solved:
                logger.warning("mj nif=%s: captcha not solved, skipping", nif)
                return []

            # Submit form. The name attribute on the button is the __EVENTTARGET.
            await page.click(f"input[name='{F_BTN}'], button[name='{F_BTN}']", timeout=5_000)
            await page.wait_for_load_state("domcontentloaded")
            await page.wait_for_timeout(1500)

            # Iterate through pages of results
            for page_idx in range(MAX_PAGES):
                page_items = await _extract_results_page(page, nif)
                if page_items:
                    results.extend(page_items)
                # Pagination: look for "Seguinte" / next-page postback link
                next_clicked = await _click_next_if_any(page)
                if not next_clicked:
                    break
                await page.wait_for_timeout(REQUEST_DELAY_MS)
        except PwTimeoutError as e:
            logger.warning("mj nif=%s: playwright timeout: %r", nif, e)
        except Exception as e:
            logger.warning("mj publications scrape failed nif=%s: %r", nif, e)
        finally:
            await browser.close()

    logger.info("mj nif=%s: %d publications scraped", nif, len(results))
    return results


async def _extract_results_page(page, nif: str) -> list[dict[str, Any]]:
    """Pull every publication row from the current GridView. The result page
    uses an ASP.NET GridView; structure varies slightly by deployment. We
    scan for tr elements that contain a date pattern AND reference the NIF we
    searched (loose match: the NIF may appear as 'NIPC 501290567' or similar
    inside the row text)."""
    rows = await page.evaluate(
        """(nif) => {
          // Candidates: rows of a GridView-like structure with a date cell
          const out = [];
          const candidates = Array.from(document.querySelectorAll("table tr, li.publication, .resultado"));
          for (const el of candidates) {
            const text = (el.innerText || "").trim();
            if (!text || text.length < 20) continue;
            if (!text.includes(nif)) continue;  // loose NIF check
            const link = el.querySelector("a[href]");
            const dateMatch = text.match(/(\\d{2}\\/\\d{2}\\/\\d{4})/);
            out.push({
              text: text.slice(0, 2000),
              href: link ? link.href : null,
              linkText: link ? (link.innerText || "").trim() : null,
              date: dateMatch ? dateMatch[1] : null,
            });
          }
          // Dedupe (nested container double-counting)
          const seen = new Set();
          return out.filter(r => {
            const k = `${r.date}|${r.text}`;
            if (seen.has(k)) return false;
            seen.add(k);
            return true;
          });
        }""",
        nif,
    )
    items: list[dict[str, Any]] = []
    for it in rows:
        text = it.get("text") or ""
        link_text = it.get("linkText") or ""
        # Title = link text when available, else the first line
        title = link_text or text.split("\n", 1)[0]
        items.append({
            "title": title.strip()[:500],
            "summary": text.strip()[:2000],
            "date": _parse_pt_date(it.get("date")),
            "type": None,  # the classifier infers from text
            "source_url": it.get("href"),
            "source": "mj",
            "raw": {"text": text, "href": it.get("href"), "date_raw": it.get("date")},
        })
    return items


async def _click_next_if_any(page) -> bool:
    """Click the 'Seguinte' pagination link if present. Returns True when a
    click happened AND the page state changed."""
    # ASP.NET GridView paginator typically uses anchors with JavaScript
    # __doPostBack(..., 'Page$N'). Text-based match is more portable.
    candidates = [
        "a:has-text('Seguinte')",
        "a:has-text('Próxima')",
        "a:has-text('Next')",
        "a:has-text('>')",
        "input[type=submit][value='Seguinte']",
    ]
    for sel in candidates:
        try:
            loc = page.locator(sel).first
            if await loc.count() == 0:
                continue
            if await loc.is_disabled():
                return False
            await loc.click(timeout=3_000)
            await page.wait_for_load_state("domcontentloaded")
            return True
        except Exception:
            continue
    return False


async def save_new_mj_publications(
    session, company_id: str, items: list[dict[str, Any]],
) -> int:
    """Persist scraped MJ rows using the existing dre_publications table.
    Relevance is computed by the shared classifier at insert time."""
    import hashlib
    import json as _json
    from sqlalchemy import text as sa_text
    from app.services.dre_classifier import classify

    new_count = 0
    for it in items:
        d = it.get("date")
        key = f"{company_id}|mj|{(it.get('title') or '').strip().lower()}|{d or ''}"
        h = hashlib.sha256(key.encode("utf-8")).hexdigest()
        cls = classify(it.get("title"), it.get("summary"))
        res = await session.execute(
            sa_text(
                """
                INSERT INTO dre_publications
                    (company_id, title, summary, date, type, source_url,
                     raw_json, dedup_hash, relevance, change_kind, source)
                VALUES
                    (:cid, :t, :s, :d, :ty, :u, CAST(:r AS JSONB), :h,
                     :rel, :ck, 'mj')
                ON CONFLICT (dedup_hash) DO NOTHING
                RETURNING id
                """
            ),
            {
                "cid": company_id,
                "t": it.get("title") or "",
                "s": it.get("summary"),
                "d": d,
                "ty": it.get("type"),
                "u": it.get("source_url"),
                "r": _json.dumps(it.get("raw") or {}, default=str),
                "h": h,
                "rel": cls.relevance,
                "ck": cls.change_kind,
            },
        )
        if res.first():
            new_count += 1
    await session.commit()
    return new_count