#!/usr/bin/env python3
"""
BidBrain daily run.

Reads the live stock off the logged in Motorway and Carwow screens, records
every car for the repeat appearance tracker, applies the gate and bans, reads
each Motorway survivor's own page for previous owners and the exact engine,
looks up Glass's and Cazana retail for the survivors, prices them, and builds
the cockpit page.

It reads screens only. No platform API. It never places a bid.

Run after 5pm or early morning, never between 3:30pm and 5:00pm (see CLAUDE.md).

  python3 daily_run.py            full run
  python3 daily_run.py --test 3   only value the 3 cheapest survivors, for a quick check
"""

import sys
import os
import re
import json
import datetime
from dataclasses import asdict
from playwright.sync_api import sync_playwright

from bidbrain import db, render, browser
from bidbrain.readers import motorway, carwow, glass, cazana
from bidbrain.pricing import (
    Car, Pricing, Assessment, assess, assess_all, pounds,
    banned_make, banned_engine, gate_failures,
)

CACHE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data", "last_run.json")
REVIEW_CACHE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data", "review.json")
PROGRESS = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data", "run_progress.json")


def _progress(phase, message="", done=0, total=0):
    """Write a small progress file the cockpit polls so Steven can see where a
    run is and whether it is still moving. phase is one of reading, valuing,
    writing, done, error. Best effort, never let a progress write break a run."""
    import time
    try:
        with open(PROGRESS, "w", encoding="utf-8") as f:
            json.dump({"phase": phase, "message": message, "done": done,
                       "total": total, "ts": time.time()}, f)
    except Exception:
        pass


def _save_review(keepers, sale_date):
    def pack(a):
        return {"status": a.status, "flags": a.flags,
                "car": {k: v for k, v in asdict(a.car).items() if not k.startswith("_")}}
    with open(REVIEW_CACHE, "w", encoding="utf-8") as f:
        json.dump({"sale_date": sale_date, "keepers": [pack(a) for a in keepers]}, f)


def _load_review():
    with open(REVIEW_CACHE, encoding="utf-8") as f:
        data = json.load(f)
    keepers = [Assessment(car=Car(**d["car"]), status=d["status"], flags=d.get("flags", []))
               for d in data["keepers"]]
    return keepers, data["sale_date"]


def add_photos():
    """Fetch the primary photo for each Motorway keeper from its page (the CSV
    export has no image), update the review cache and rebuild the review page.
    Carwow keepers already carry a photo from the listing card."""
    keepers, sale_date = _load_review()
    today = datetime.date.fromisoformat(sale_date)
    need = [a for a in keepers
            if a.car.source == "Motorway" and not a.car.photo_url and a.car.listing_url]
    print(f"Fetching photos for {len(need)} Motorway cars (Carwow already has them)")
    img_re = re.compile(r'https://[A-Za-z0-9._\-]*imgix[A-Za-z0-9._/\-]*?\.jpg')
    with sync_playwright() as p:
        ctx = browser.open_reader_context(p, "motorway", headless=True)
        page = ctx.pages[0] if ctx.pages else ctx.new_page()
        done = 0
        for i, a in enumerate(need, 1):
            try:
                page.goto(a.car.listing_url, wait_until="domcontentloaded", timeout=40000)
                page.wait_for_selector("#__NEXT_DATA__", state="attached", timeout=10000)
                blob = page.eval_on_selector("#__NEXT_DATA__", "el=>el.textContent")
                m = re.search(r'https://[A-Za-z0-9._\-]*imgix[A-Za-z0-9._/\-]*exterior[A-Za-z0-9._/\-]*?\.jpg', blob)
                if not m:
                    m = img_re.search(blob)
                if m:
                    a.car.photo_url = m.group(0) + "?w=640&auto=format,compress"
                    done += 1
            except Exception:
                pass
            if i % 25 == 0:
                print(f"  {i}/{len(need)} ({done} photos so far)")
        ctx.close()
    _save_review(keepers, sale_date)
    with open("cockpit.html", "w", encoding="utf-8") as f:
        f.write(render.render_review(keepers, today))
    print(f"Done. {done} photos added. Review page rebuilt.")


def review(sale_date=None):
    """Read and gate both platforms for tomorrow's sales, but do not value yet.
    Build the review page so Steven can tick the cars he would not buy. The
    gate passers are cached so the survivors can be valued afterwards."""
    db.init_db()
    sale_date = sale_date or datetime.date.today().isoformat()
    today = datetime.date.fromisoformat(sale_date)
    print(f"BidBrain review run for sale date {sale_date}\n")
    with sync_playwright() as p:
        mw_cars = _read_motorway(p, sale_date)
        cw_cars = _read_carwow(p, sale_date)
    all_cars = mw_cars + cw_cars
    assessments = [_assess(c, today) for c in all_cars]
    keepers = [a for a in assessments if a.status in ("held", "shortlist")]
    keepers.sort(key=lambda a: a.car.reserve if a.car.reserve is not None else 10 ** 9)
    _save_review(keepers, sale_date)
    with open("cockpit.html", "w", encoding="utf-8") as f:
        f.write(render.render_review(keepers, today))
    print(f"\n{len(keepers)} cars passed the gate and bans across both platforms.")
    print("Review page written to cockpit.html. Tick the ones to drop, then paste the box back.")


def _save_cache(shortlist, held, rejected, sale_date):
    """Save the run so the page can be re rendered instantly without re valuing,
    handy while tuning the cockpit design."""
    def pack(a):
        d = {"status": a.status, "reasons": a.reasons, "flags": a.flags,
             "car": {k: v for k, v in asdict(a.car).items() if not k.startswith("_")}}
        d["pricing"] = asdict(a.pricing) if a.pricing else None
        return d
    data = {"sale_date": sale_date,
            "shortlist": [pack(a) for a in shortlist],
            "held": [pack(a) for a in held],
            "rejected": [pack(a) for a in rejected]}
    with open(CACHE, "w", encoding="utf-8") as f:
        json.dump(data, f)


def _load_cache():
    with open(CACHE, encoding="utf-8") as f:
        data = json.load(f)
    def unpack(d):
        a = Assessment(car=Car(**d["car"]), status=d["status"],
                       reasons=d.get("reasons", []), flags=d.get("flags", []))
        if d.get("pricing"):
            a.pricing = Pricing(**d["pricing"])
        return a
    return ([unpack(x) for x in data["shortlist"]],
            [unpack(x) for x in data["held"]],
            [unpack(x) for x in data["rejected"]],
            data["sale_date"])


def _write_pages(shortlist, held, rejected, today, sale_date):
    """Write cockpit.html (with hidden cars suppressed and bid stars pre ticked)
    and the hidden cars page. Used by both run() and render_only() so the two
    paths never diverge."""
    hidden = db.hidden_regs()
    keep = lambda a: db._norm_reg(a.car.reg) not in hidden
    shortlist = [a for a in shortlist if keep(a)]
    held = [a for a in held if keep(a)]
    bidset = db.bids_for(sale_date)
    with open("cockpit.html", "w", encoding="utf-8") as f:
        f.write(render.render_page(shortlist, held, rejected, today=today,
                                   show_excluded=True, show_rejected=False,
                                   bidset=bidset, sale_date=sale_date))
    with open("hidden.html", "w", encoding="utf-8") as f:
        f.write(render.render_hidden(today))
    return len(shortlist)


def render_only():
    """Rebuild cockpit.html from the last saved run, no network."""
    shortlist, held, rejected, sale_date = _load_cache()
    today = datetime.date.fromisoformat(sale_date)
    n = _write_pages(shortlist, held, rejected, today, sale_date)
    print(f"Re rendered cockpit.html from the last run ({n} shortlisted after hidden cars removed).")


def _list_gate_ok(car, today):
    """The gate checks we can make from the list alone, used to decide which
    cars are worth opening the detail page and valuing. Owners and exact engine
    are confirmed later, on the car's page."""
    if banned_make(car):
        return False
    fails = gate_failures(car, today)
    # Ignore the owners line here, it is not on the list card yet.
    fails = [f for f in fails if "previous owners" not in f.lower()]
    return not fails


def _assess(car, today):
    """Assess a car. Motorway cars come from the brief filtered CSV which
    enforces the 210 mile limit but gives a location not exact miles, so trust
    the filter for distance on those. Carwow cars carry real distance."""
    return assess(car, today, assume_distance_ok=(car.source == "Motorway"))


def _read_motorway(p, sale_date):
    # The whole brief filtered list comes from Motorway's own CSV export, which
    # carries owners, engine, CAP and the link, so no card scraping or detail
    # page reads are needed.
    cars = motorway.read_export(p)
    db.record_sightings(cars, "Motorway", sale_date)
    print(f"Motorway: read {len(cars)} cars from the filtered CSV export")
    return cars


def _read_carwow(p, sale_date):
    cars = []
    ctx = browser.open_reader_context(p, "carwow", headless=True)
    try:
        page = ctx.pages[0] if ctx.pages else ctx.new_page()
        page.goto(browser.SITES["carwow"]["stock_url"], wait_until="domcontentloaded", timeout=45000)
        page.wait_for_selector('[data-listing-id]', timeout=30000)
        cars = carwow.parse_listing(page.content())
        db.record_sightings(cars, "Carwow", sale_date)
        print(f"Carwow: read {len(cars)} auction cars")

        today = datetime.date.fromisoformat(sale_date)
        candidates = [c for c in cars if _list_gate_ok(c, today)]
        print(f"Carwow: {len(candidates)} pass the list level checks, reading their pages for owners")
        for c in candidates:
            try:
                carwow.enrich_from_detail(page, c)
            except Exception as e:
                print(f"  could not read detail for {c.reg or c._listing_id}: {e}")
    except Exception as e:
        print(f"Carwow: could not read ({e}). Carrying on with Motorway.")
    finally:
        ctx.close()
    return cars


def run(test_limit=None, sale_date=None):
    db.init_db()
    sale_date = sale_date or datetime.date.today().isoformat()
    today = datetime.date.fromisoformat(sale_date)
    print(f"BidBrain daily run for sale date {sale_date}\n")
    _progress("reading", "Reading Motorway stock")

    with sync_playwright() as p:
        mw_cars = _read_motorway(p, sale_date)
        _progress("reading", "Reading Carwow stock")
        cw_cars = _read_carwow(p, sale_date)
        all_cars = mw_cars + cw_cars

        # Drop cars Steven has hidden so they are never valued or shown again.
        hidden = db.hidden_regs()
        before = len(all_cars)
        all_cars = [c for c in all_cars if db._norm_reg(c.reg) not in hidden]
        if before - len(all_cars):
            print(f"Suppressed {before - len(all_cars)} cars Steven has hidden")

        # Assess both platforms with owners and engine now known.
        assessments = [_assess(c, today) for c in all_cars]
        gate_passers = [a for a in assessments if a.status in ("shortlist", "held")]
        print(f"\n{len(gate_passers)} cars across both platforms pass the full gate and bans, valuing them now")

        # Value the gate passers, cheapest reserve first, optionally limited.
        gate_passers.sort(key=lambda a: a.car.reserve or 0)
        to_value = gate_passers[:test_limit] if test_limit else gate_passers
        if test_limit:
            print(f"(test mode, valuing only the {len(to_value)} cheapest)")

        _progress("valuing", f"Valuing {len(to_value)} cars", 0, len(to_value))
        for i, a in enumerate(to_value):
            c = a.car
            try:
                c.glass_retail = glass.lookup(p, c.reg, c.mileage)
                c.cazana_retail = cazana.lookup(p, c.reg, c.mileage)
                print(f"  {c.reg} {c.make} {c.model}: Glass's {pounds(c.glass_retail)}, Cazana {pounds(c.cazana_retail)}")
            except Exception as e:
                print(f"  valuation failed for {c.reg}: {e}")
            _progress("valuing", f"Valued {i + 1} of {len(to_value)}: {c.reg} {c.make} {c.model}",
                      i + 1, len(to_value))

    # Re-assess with valuations in, then split.
    valued = [_assess(a.car, today) for a in to_value]
    shortlist = [a for a in valued if a.status == "shortlist"]
    held = [a for a in valued if a.status == "held"]
    rejected = [a for a in assessments if a.status == "rejected"]

    # Repeat appearance flags on the shortlist.
    for a in shortlist + held:
        summ = db.repeat_summary(a.car.reg, sale_date)
        for line in db.repeat_flag_lines(summ):
            a.flags.append(line)

    shortlist.sort(key=lambda a: a.car.reserve)
    db.save_assessments(shortlist + held + rejected)
    _save_cache(shortlist, held, rejected, sale_date)

    # Write the cockpit (rejected hidden from Steven, held back kept) and the
    # hidden cars page. _write_pages also pre ticks his bid stars.
    _progress("writing", "Building the cockpit page", len(to_value), len(to_value))
    _write_pages(shortlist, held, rejected, today, sale_date)
    _progress("done", f"Finished. {len(shortlist)} shortlisted, {len(held)} held.",
              len(to_value), len(to_value))

    print(f"\nShortlist: {len(shortlist)} priced. Held: {len(held)}. Pages written: cockpit.html, hidden.html.")
    print("Serve them with python3 serve.py, then open http://localhost:8765/cockpit.html")
    for a in shortlist:
        print(f"  reserve {pounds(a.car.reserve):>8}  max bid {pounds(a.pricing.max_bid):>8}  {a.car.year} {a.car.make} {a.car.model} {a.car.derivative}")


def hidden_notes():
    """Print the cars Steven has hidden with a reason that have not yet been
    turned into rules, so the assistant can mine them into pricing.py."""
    rows = db.hidden_notes(only_new=True)
    if not rows:
        print("No new hide notes to review.")
        return
    print(f"{len(rows)} cars hidden with a reason, not yet turned into rules:\n")
    for r in rows:
        name = r.get("name") or f"{r.get('make','')} {r.get('model','')}".strip()
        print(f"  {r['reg']:10} {name}")
        print(f"      reason: {r.get('reason','')}")
        print(f"      hidden: {r.get('hidden_at','')}\n")
    print("Turn the recurring reasons into rules in bidbrain/pricing.py, then "
          "mark them reviewed with db.mark_notes_reviewed([...]).")


if __name__ == "__main__":
    if "--render-only" in sys.argv:
        render_only()
    elif "--review" in sys.argv:
        review()
    elif "--photos" in sys.argv:
        add_photos()
    elif "--hidden-notes" in sys.argv:
        hidden_notes()
    else:
        tl = None
        if "--test" in sys.argv:
            i = sys.argv.index("--test")
            tl = int(sys.argv[i + 1]) if i + 1 < len(sys.argv) else 3
        try:
            run(test_limit=tl)
        except Exception as e:
            # Record the failure so the cockpit shows an error instead of an
            # endless spinner, then re raise so it still logs and exits non zero.
            _progress("error", f"Run failed: {e}")
            raise