Classifier rules, copied verbatim from data/classify.py in the JournalistLabs repo.
The stoplist, the include list (which sets role), the editor_kind patterns, the seniority
patterns and the beat lexicon are the lists that produced the role, editor_kind and
beat columns behind every figure in this dataset.

"""JournalistLabs title/headline classifier.

Rerunnable: `from data.classify import classify_row; classify_row(title, headline)`
Every list below is data, not magic. Change a list, rerun tools/pull_people.py --classify-only.
"""
import re

# ---------------------------------------------------------------- exclusions
# Applied to the TITLE first. A hit here makes the row non-editorial no matter
# what else the title says ("marketing editor", "product editor", "ad sales").
STOPLIST = [
    r"\bsales\b", r"\bsalesperson\b", r"\bsales\s+(director|manager|executive|lead|rep|representative)\b",
    r"\bmarketing\b", r"\bbrand\s+(manager|strategist|director)\b", r"\bgrowth\b",
    r"\badvertis", r"\bad\s+(sales|ops|operations|traffic)\b", r"\bmedia\s+buyer\b",
    r"\baccount\s+(manager|executive|director|coordinator|supervisor)\b",
    r"\bhuman\s+resources\b", r"\bhr\b", r"\brecruit", r"\btalent\s+(acquisition|partner)\b",
    r"\bpayroll\b", r"\bpeople\s+(operations|partner)\b",
    r"\bengineer\b", r"\bengineering\b", r"\bdeveloper\b", r"\bdevops\b", r"\bqa\b",
    r"\bsoftware\b", r"\bdata\s+(engineer|scientist|analyst)\b", r"\bit\s+support\b",
    r"\bproduct\s+(manager|owner|director|designer|lead|marketing)\b", r"\bproduct\b",
    r"\bcontent\s+strategist\b", r"\bcontent\s+marketing\b", r"\bseo\b",
    r"\bcopywriter\b", r"\bcopy\s?writer\b", r"\bcontent\s+writer\b", r"\btechnical\s+writer\b",
    r"\bgrant\s+writer\b", r"\bproposal\s+writer\b", r"\bghost\s?writer\b", r"\bbid\s+writer\b",
    r"\bpublic\s+relations\b", r"\bpr\s+(manager|executive|officer)\b", r"\bcommunications\s+(manager|officer|director|specialist)\b",
    r"\bfinance\b", r"\baccountant\b", r"\baccounting\b", r"\blegal\s+counsel\b", r"\bparalegal\b",
    r"\bfacilities\b", r"\bjanitor\b", r"\bsecurity\s+guard\b", r"\bdriver\b",
    r"\bcustomer\s+(success|support|service)\b", r"\bsubscription", r"\bcirculation\b",
    r"\bteacher\b", r"\bprofessor\b", r"\blecturer\b", r"\bstudent\b",
]
STOP_RE = [re.compile(p) for p in STOPLIST]

# ------------------------------------------------------------------ includes
# (pattern, role). First matching entry in this order wins the role.
INCLUDE = [
    (r"\bbureau\s+chief\b", "editor"),
    (r"\bnews\s+director\b", "editor"),
    (r"\bmanaging\s+editor\b", "editor"),
    (r"\beditor[\s\-]in[\s\-]chief\b", "editor"),
    (r"\bsub[\s\-]?editor\b", "editor"),
    (r"\bcorrespondent\b", "correspondent"),
    (r"\bcolumnist\b", "columnist"),
    (r"\banchor\b", "anchor/presenter"),
    (r"\bpresenter\b", "anchor/presenter"),
    (r"\bnews\s+host\b", "anchor/presenter"),
    (r"\bnewsreader\b", "anchor/presenter"),
    (r"\bphoto\s?journalist\b", "reporter"),
    (r"\bjournalist\b", "reporter"),
    (r"\breporter\b", "reporter"),
    (r"\bcritic\b", "critic"),
    (r"\breviewer\b", "critic"),
    (r"\b(news|newsroom|editorial)\s+producer\b", "producer"),
    (r"\bproducer[,/\s\-]+(reporter|journalist)\b", "producer"),
    (r"\b(staff|feature|features)\s+writer\b", "other-editorial"),
    (r"\beditor\b", "editor"),
    (r"\beditorial\b", "other-editorial"),
]
INCLUDE_RE = [(re.compile(p), r) for p, r in INCLUDE]

# Writers we deliberately do NOT count as editorial (spec: staff/feature only).
# Tracked so the exclusion is visible in the numbers rather than silent.
WRITER_OTHER = re.compile(r"\b(senior|contributing|lead|freelance|news|sports|science|business|political)\s+writer\b|^writer$")

EDITOR_KIND = [
    (re.compile(r"\bvideo\s+editor\b|\bfilm\s+editor\b"), "video"),
    (re.compile(r"\bphoto\s*(graphy)?\s+editor\b|\bpicture\s+editor\b"), "photo"),
    (re.compile(r"\bcopy\s+editor\b|\bsub[\s\-]?editor\b"), "copy"),
    (re.compile(r"\baudio\s+editor\b|\bsound\s+editor\b"), "audio"),
]

# ----------------------------------------------------------------- seniority
SENIORITY = [
    (re.compile(r"\bchief\b|\beditor[\s\-]in[\s\-]chief\b|\bhead\s+of\b"), "chief"),
    (re.compile(r"\b(executive|exec)\b|\bdirector\b|\bvp\b|\bvice\s+president\b"), "executive"),
    (re.compile(r"\bdeputy\b|\bassociate\b|\bassistant\b"), "deputy"),
    (re.compile(r"\bsenior\b|\bsnr\b|\bsr\.?\b|\bprincipal\b|\blead\b"), "senior"),
    (re.compile(r"\bstaff\b|\bjunior\b|\btrainee\b|\bintern\b|\bgraduate\b"), "staff"),
]

# ---------------------------------------------------------------- beat lexicon
# Ordered. beat_primary = the first beat in this order that matches.
BEATS = [
    ("ai", r"\b(a\.?i\.?|artificial intelligence|machine learning|generative ai|llm|chatgpt|openai)\b"),
    ("crypto", r"\b(crypto|cryptocurrency|bitcoin|blockchain|web3|defi|nft)\b"),
    ("cybersecurity", r"\b(cyber ?security|cyber|infosec|information security|hacking|ransomware|data breach)\b"),
    ("technology", r"\b(tech|technology|technolog\w+|software|startups?|gadgets?|consumer tech|silicon valley|semiconductor)\b"),
    ("energy", r"\b(energy|oil|gas|petroleum|renewables?|nuclear|utilities|power sector|solar)\b"),
    ("climate/environment", r"\b(climate|environment\w*|sustainab\w+|conservation|wildlife|biodiversity|emissions|net zero)\b"),
    ("health", r"\b(health\w*|medical|medicine|pharma\w*|hospital|nhs|public health|mental health|biotech)\b"),
    ("science", r"\b(science|scientific|space|astronomy|physics|research corresponden\w+)\b"),
    ("finance/markets", r"\b(financ\w+|markets?|banking|bank|investing|investment|equit\w+|bonds?|hedge fund|private equity|venture capital|fintech|wall street|trading)\b"),
    ("economics", r"\b(econom\w+|inflation|gdp|central bank|federal reserve|macro)\b"),
    ("business", r"\b(business|corporate|companies|retail|commerce|m and a|dealbook|supply chain)\b"),
    ("politics", r"\b(politic\w+|congress|parliament|white house|westminster|capitol hill|election\w*|campaign trail|government|policy|senate|lobbying)\b"),
    ("defence/security", r"\b(defen[sc]e|military|pentagon|army|navy|air force|nato|intelligence services|war corresponden\w+|conflict)\b"),
    ("immigration", r"\b(immigration|migrant\w*|migration|refugee\w*|asylum|border)\b"),
    ("crime/legal", r"\b(crime|criminal|courts?|court\w*|legal|law|justice|police|homicide|trial)\b"),
    ("education", r"\b(education|higher ed|schools? (reporter|editor|corresponden\w+)|education (reporter|editor|corresponden\w+|desk))\b"),
    ("real estate", r"\b(real estate|property|housing|mortgage|rentals?|construction|architecture)\b"),
    ("sports", r"\b(sports?|football|soccer|basketball|nba|nfl|mlb|nhl|cricket|rugby|tennis|golf|olympics?|motorsport|f1|formula one|athletics)\b"),
    ("entertainment", r"\b(entertainment|celebrit\w+|hollywood|films?|movies?|streaming|showbiz|music industry|gaming|video games?)\b"),
    ("culture/arts", r"\b(culture|cultural|arts?|theatre|theater|books?|literary|museum|classical music|dance|opera)\b"),
    ("fashion", r"\b(fashion|style|beauty|luxury|couture|runway)\b"),
    ("food", r"\b(food|restaurants?|dining|culinary|drinks?|wine|chef|recipes?)\b"),
    ("travel", r"\b(travel|tourism|aviation|airlines?|destinations?|hospitality)\b"),
    ("world/foreign", r"\b(world|foreign|global affairs|middle east|latin america|geopolitic\w+|diplomacy|foreign corresponden\w+)\b"),
    ("local", r"\b(local|metro|city hall|county|neighbou?rhood)\b"),
    # Appended, not inserted: a row that already carries another beat keeps
    # that beat as beat_primary, so every earlier count is unchanged.
    ("investigative", r"\b(investigative|investigations)\b"),
]
BEATS_RE = [(b, re.compile(p)) for b, p in BEATS]

_WS = re.compile(r"\s+")


def _norm(s):
    if not s:
        return ""
    return _WS.sub(" ", str(s).lower().replace("&", " and ")).strip()


def classify_row(title, headline=None):
    """Return dict: editorial, role, editor_kind, seniority, beat_primary, beats, writer_other."""
    t = _norm(title)
    h = _norm(headline)
    out = {"editorial": False, "role": "", "editor_kind": "", "seniority": "unknown",
           "beat_primary": "", "beats": "", "writer_other": False}
    if not t:
        return out
    if any(r.search(t) for r in STOP_RE):
        return out
    role = ""
    for rx, rl in INCLUDE_RE:
        if rx.search(t):
            role = rl
            break
    if not role:
        if WRITER_OTHER.search(t):
            out["writer_other"] = True
        return out
    out["editorial"] = True
    out["role"] = role
    if role == "editor":
        for rx, kind in EDITOR_KIND:
            if rx.search(t):
                out["editor_kind"] = kind
                break
        if not out["editor_kind"]:
            out["editor_kind"] = "text"
    for rx, lvl in SENIORITY:
        if rx.search(t):
            out["seniority"] = lvl
            break
    blob = t + " || " + h
    hits = [b for b, rx in BEATS_RE if rx.search(blob)]
    if hits:
        out["beat_primary"] = hits[0]
        out["beats"] = ";".join(hits)
    return out


# --------------------------------------------------------- platform domains
# pdl_company_dataset stores a link-in-bio, social or freelance-marketplace URL
# as the company website for thousands of small media companies. Those rows
# collapse onto one domain and would otherwise look like a giant newsroom.
PLATFORM_DOMAINS = {
    "youtube.com", "linktr.ee", "fiverr.com", "upwork.com", "linkedin.com",
    "instagram.com", "bit.ly", "tiktok.com", "facebook.com", "twitter.com",
    "x.com", "google.com", "sites.google.com", "apple.com", "amazon.com",
    "substack.com", "medium.com", "wordpress.com", "wix.com", "wixsite.com",
    "blogspot.com", "weebly.com", "squarespace.com", "static.com", "dan.com",
    "remove.bg", "ted.com", "patreon.com", "behance.net", "about.me",
    "carrd.co", "beacons.ai", "t.me", "whatsapp.com", "gmail.com",
    "contently.com", "muckrack.com", "journoportfolio.com", "github.io",
    "issuu.com", "spotify.com", "soundcloud.com", "vimeo.com", "podbean.com",
    "anchor.fm", "buzzsprout.com", "wordpress.org", "godaddy.com",
    "omicsonline.org", "esquiresolutions.com",
}