"""Facts for the report "Britain has 19 royal journalists on file and 2 cybersecurity reporters". python3 tools/report_royal_vs_cyber.py python3 tools/report_royal_vs_cyber.py --sanity Reads the published base (data/journalists.csv.gz, platform_domain == 0 and media_typed == 1, 127,602 rows) and rebuilds the outlet table the way tools/export.py does, by importing that module's helpers, so outlet names, countries and types match the site's /outlets/ pages. Two match rules are printed side by side everywhere: title-only the lexicon is run over the job title title+head the lexicon is run over title + " || " + headline The headline is a free-text LinkedIn bio. It carries Royal Air Force, Royal Opera House, Royal Caribbean and the Royal Society, so the title-only rule is the one the report quotes and the headline-inclusive rule is printed beside it. Writes the dataset CSVs under static/data/britain-19-royal-journalists-2-cybersecurity-reporters/. No person name and no headline text is written to any CSV. """ import collections import csv import gzip import os import re import sys ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, ROOT) sys.path.insert(0, os.path.join(ROOT, "tools")) import export as E # noqa: E402 (tools/export.py) import data.classify as C # noqa: E402 SLUG = "britain-19-royal-journalists-2-cybersecurity-reporters" OUT = os.path.join(ROOT, "static", "data", SLUG) EXPECT_CLASSIFIED = 173133 EXPECT_BASE = 127602 EXPECT_GB = 12094 UK = "GB" # ── lexicons ───────────────────────────────────────────────────────── # Every regex here is also shipped as lexicon.txt in the dataset folder. # Word-start, not whole-word: \broyal matches royal, royals and royalty. # The scout (tools/scout_titles.py) used the whole-word \broyal\b; both counts # are printed, and the three rows the widening adds are named. ROYAL_START = re.compile(r"\broyal", re.I) ROYAL_WORD = re.compile(r"\broyal\b", re.I) # Titles removed by hand from the royal set, with the reason. Matched # case-insensitively as a substring of the lowercased title. ROYAL_REMOVE = [ ("royal air force", "Royal Air Force, an armed service, not the monarchy"), ("royal navy", "Royal Navy, an armed service, not the monarchy"), ("royal marine", "Royal Marines, an armed service, not the monarchy"), ("royal opera", "Royal Opera House, an arts venue"), ("royal shakespeare", "Royal Shakespeare Company, a theatre company"), ("royal ballet", "a ballet company"), ("royal college", "a Royal College, a professional body"), ("royal society", "a Royal Society, a learned society"), ("royal academy", "a Royal Academy, a learned or arts body"), ("royal conservatoire", "a conservatoire, a music school"), ("royal television society", "the Royal Television Society, an awards body"), ("royal caribbean", "Royal Caribbean, a cruise line"), ("royal botanic", "a botanic garden"), ("royal courts of justice", "a court building, a law round not a royal one"), ("royal literary fund", "a writers' charity"), ("royal meteorological", "the Royal Meteorological Society, a learned society"), ("royal geograph", "the Royal Geographical Society, a learned society"), ("mount royal", "Mount Royal, a place name (Montreal)"), ("kansas city royals", "a baseball team"), ("royals live host", "an in-game host at a regional sports broadcaster " "(FanDuel Sports Network), not a monarchy round"), ] # Narrow: the word cybersecurity, however it is spaced or hyphenated. CYBER_NARROW = re.compile(r"\bcyber ?security\b|\bcyber-security\b", re.I) # Widened, one regex per word so each match can name the word that matched. # national security is deliberately NOT here: it is a defence and intelligence # round (Pentagon, MoD, spy agencies, wars), not a computer-security round, and # 66 of the 127,602 rows carry it in the title, which would swamp the count with # a different beat. The number is printed so the choice can be checked. WIDE = [ ("cybersecurity", r"\bcyber ?security\b|\bcyber-security\b"), ("cyber (bare)", r"\bcyber\b(?! ?security)"), ("hacker", r"\bhackers?\b"), ("hacking", r"\bhacking\b"), ("infosec", r"\binfosec\b"), ("information security", r"\binformation security\b"), ("ransomware", r"\bransomware\b"), ("malware", r"\bmalware\b"), ("data breach", r"\bdata breach(?:es)?\b"), ("surveillance", r"\bsurveillance\b"), ("privacy", r"\bprivacy\b"), ("data protection", r"\bdata protection\b"), ("GDPR", r"\bgdpr\b"), ] WIDE_RE = [(w, re.compile(p, re.I)) for w, p in WIDE] # The privacy subset, quoted on its own in the report. PRIVACY_WORDS = {"privacy", "data protection", "GDPR"} NATIONAL_SECURITY = re.compile(r"\bnational security\b", re.I) # The technology beat lexicon, taken from data/classify.py rather than retyped. TECH_RE = dict(C.BEATS_RE)["technology"] BEAT_COMPARATORS = ["technology", "ai", "crypto", "sports", "health", "politics"] # The site classifier's own cybersecurity beat regex, imported rather than # retyped. It is wider than this report's narrow rule: it carries bare cyber, # infosec, information security, hacking, ransomware and data breach. Run over # the job title it reaches 28 rows where the narrow rule reaches 23, and the # five-row difference is printed below so the number on /journalists/ # cybersecurity/ and the number in this report reconcile. CYBER_CLASSIFIER = dict(C.BEATS_RE)["cybersecurity"] # Bare cyber, both forms. The lexicon uses the lookahead so a row already # counted by the narrow cybersecurity rule is not counted twice. BARE_CYBER_LOOKAHEAD = re.compile(r"\bcyber\b(?! ?security)", re.I) BARE_CYBER_PLAIN = re.compile(r"\bcyber\b", re.I) # Which lexicon each regex comes from. OWN_REGEXES = ["royal (word-start)", "royal (whole word)", "cybersecurity (narrow)", "security (widened)", "privacy subset", "national security (excluded)", "title noun", "the royal hand-removal list"] CLASSIFIER_REGEXES = ["sports", "politics", "health", "technology", "ai", "crypto", "cybersecurity"] # The noun the royal title carries. First match wins, in this order. NOUN_ORDER = [ ("correspondent", re.compile(r"\bcorrespondents?\b", re.I)), ("editor", re.compile(r"\beditors?\b", re.I)), ("reporter", re.compile(r"\breporters?\b", re.I)), ] # The named UK outlets the report walks through, whether or not they are in the # top 15 by UK rows. Keys are the canonical outlet keys in the base. NAMED_UK = [ "BBC", "The Guardian", "ITV", "The Times", "The Telegraph", "Financial Times", "sky.com", "Bloomberg", "itn.co.uk", "thesun.co.uk", "pa.media", "mirror.co.uk", "express.co.uk", "independent.co.uk", "dailymail.co.uk", "The Economist", "channel4.com", "gbnews.com", ] # Cells of 1 to 4 are printed as the number, not as "<5". Every table here is # either a small named list printed in full, which is the site's exception to # the suppression rule, or a per-outlet count whose rows also appear in one of # those lists. Suppressing them would remove the report's subject. No table in # this file has a person as its unit. def ordinal(n): if 10 <= n % 100 <= 20: return "%dth" % n return "%d%s" % (n, {1: "st", 2: "nd", 3: "rd"}.get(n % 10, "th")) def norm(text): return re.sub(r"\s+", " ", (text or "")).strip() def blob(row): return norm(row["title"]) + " || " + norm(row["headline"]) def royal_reason(title): low = (title or "").lower() for needle, reason in ROYAL_REMOVE: if needle in low: return reason return "" def noun_of(title): for label, rx in NOUN_ORDER: if rx.search(title or ""): return label return "other" def wide_words(text): return [w for w, rx in WIDE_RE if rx.search(text or "")] # ── load ───────────────────────────────────────────────────────────── def load(): records = E.load_outlet_records() rows = [] classified = dropped_platform = dropped_not_media = 0 with gzip.open(os.path.join(ROOT, "data", "journalists.csv.gz"), "rt", newline="") as fh: for row in csv.DictReader(fh): classified += 1 if row["platform_domain"] == "1": dropped_platform += 1 continue if row["media_typed"] != "1": dropped_not_media += 1 continue rows.append(row) rows_by_outlet = collections.Counter() gb_by_outlet = collections.Counter() domains_by_outlet = collections.defaultdict(set) company_names = collections.defaultdict(collections.Counter) country_of_outlet = collections.defaultdict(collections.Counter) for row in rows: key = row["outlet"] rows_by_outlet[key] += 1 domains_by_outlet[key].add(row["company_domain"]) if row["company_name"]: company_names[key][row["company_name"]] += 1 if row["country_code"]: country_of_outlet[key][row["country_code"]] += 1 if row["country_code"] == UK: gb_by_outlet[key] += 1 outlets = {} for key, count in rows_by_outlet.items(): domains = sorted(domains_by_outlet[key]) recs = [r for d in domains for r in records.get(d, [])] modal = company_names[key].most_common(1) name = E.outlet_name(key, domains, recs, modal[0][0] if modal else "") entry = { "key": key, "name": name, "rows": count, "domains": domains, "country_code": E.outlet_country_code(recs) or E.fallback_country_code(country_of_outlet[key]), "type": E.outlet_type(recs), "parent": bool(set(domains) & E.PARENT_DOMAINS), } entry["country"] = E.COUNTRIES.get(entry["country_code"], "") outlets[key] = entry def publishable(k): e = outlets[k] return not e["parent"] and e["name"] and e["country"] and e["type"] ranked = [k for k, _ in rows_by_outlet.most_common() if publishable(k)] gb_ranked = [k for k, _ in gb_by_outlet.most_common() if publishable(k)] return dict( rows=rows, classified=classified, dropped_platform=dropped_platform, dropped_not_media=dropped_not_media, outlets=outlets, publishable={k for k in outlets if publishable(k)}, rows_by_outlet=rows_by_outlet, gb_by_outlet=gb_by_outlet, top50=ranked[:50], gb_ranked=gb_ranked, ) def oname(D, key): e = D["outlets"].get(key) return (e["name"] if e and e["name"] else key) def odomain(D, key): """Every company domain the canonical outlet key covers, semicolon-joined.""" e = D["outlets"].get(key) return ";".join(e["domains"]) if e else key # ── the report ─────────────────────────────────────────────────────── def main(): os.makedirs(OUT, exist_ok=True) D = load() rows = D["rows"] N = len(rows) gb = [r for r in rows if r["country_code"] == UK] NGB = len(gb) print("=" * 72) print("7. FUNNEL") print("=" * 72) print("classified editorial rows %d (expected %d)" % (D["classified"], EXPECT_CLASSIFIED)) print(" dropped, platform domain %d" % D["dropped_platform"]) print(" dropped, outlet not media-typed %d" % D["dropped_not_media"]) print("published base %d (expected %d)" % (N, EXPECT_BASE)) print(" of which country_code == GB %d (expected %d)" % (NGB, EXPECT_GB)) print(" GB share of the base %.1f%%" % (100.0 * NGB / N)) assert D["classified"] == EXPECT_CLASSIFIED and N == EXPECT_BASE assert NGB == EXPECT_GB # ── 1. UK royal titles ─────────────────────────────────────────── print() print("=" * 72) print("1. UK ROWS WHOSE JOB TITLE NAMES THE ROYALS") print("=" * 72) raw = [r for r in gb if ROYAL_START.search(r["title"])] word = [r for r in gb if ROYAL_WORD.search(r["title"])] print("rule: the job title matches %s (case-insensitive, word-start)" % ROYAL_START.pattern) print("raw matches, title only, of %d UK rows: %d" % (NGB, len(raw))) print(" the scout's whole-word rule \\broyal\\b gives: %d" % len(word)) added = [r for r in raw if r not in word] print(" the word-start rule adds %d rows, all of them plural Royals:" % len(added)) for r in added: print(" %-60s %s" % (norm(r["title"])[:60], oname(D, r["outlet"]))) removed = [r for r in raw if royal_reason(r["title"])] kept = [r for r in raw if not royal_reason(r["title"])] print() print("hand-removed from the raw %d: %d" % (len(raw), len(removed))) if not removed: print(" none. No UK row's title carries Royal Air Force, Royal Opera,") print(" Royal Shakespeare, Mount Royal or any other non-monarchy use") print(" of the word. Those appear in the headline, not the title, and") print(" the headline-inclusive figure below shows what they add.") for r in removed: print(" - %-55s %-22s %s" % (norm(r["title"])[:55], oname(D, r["outlet"]), royal_reason(r["title"]))) print("kept: %d of %d UK rows" % (len(kept), NGB)) nouns = collections.Counter(noun_of(r["title"]) for r in kept) roles = collections.Counter(r["role"] for r in kept) print("noun in the kept titles: " + ", ".join( "%s %d" % (k, nouns[k]) for k in ["correspondent", "editor", "reporter", "other"] if nouns[k])) print("classifier role column: " + ", ".join( "%s %d" % (k, v) for k, v in roles.most_common())) print() print("the kept rows, grouped by outlet (no names; a small named list is") print("printed in full, so no cell is suppressed here):") by_outlet = collections.defaultdict(list) for r in kept: by_outlet[r["outlet"]].append(r) royal_uk_outlets = sorted(by_outlet, key=lambda k: (-len(by_outlet[k]), oname(D, k).lower())) for key in royal_uk_outlets: print(" %s (%d)" % (oname(D, key), len(by_outlet[key]))) for r in sorted(by_outlet[key], key=lambda r: norm(r["title"])): print(" %-55s %s" % (norm(r["title"])[:55], r["role"])) exact_corr = [r for r in kept if re.search(r"\broyal correspondent\b", r["title"], re.I)] print() print("the phrase royal correspondent appears in %d of the %d kept titles." % (len(exact_corr), len(kept))) print("The other %d read royal editor, deputy royal editor, royal reporter" % (len(kept) - len(exact_corr))) print("or name the royals inside a wider title.") uk_outlet_kept = [r for r in kept if D["outlets"][r["outlet"]]["country_code"] == UK] print("%d of the %d kept rows sit at an outlet table A puts in the United" % (len(uk_outlet_kept), len(kept))) print("Kingdom; the other %d are UK-located people at outlets table A puts" % (len(kept) - len(uk_outlet_kept))) print("elsewhere: " + ", ".join(sorted( "%s (%s)" % (oname(D, r["outlet"]), D["outlets"][r["outlet"]]["country"] or "no country") for r in kept if D["outlets"][r["outlet"]]["country_code"] != UK))) print("country_code is the person's own country, never the outlet's.") raw_blob = [r for r in gb if ROYAL_START.search(blob(r))] removed_blob = [r for r in raw_blob if royal_reason(blob(r)) and not ROYAL_START.search(r["title"])] print() print("headline included (title + \" || \" + headline), same regex:") print(" raw matches of %d UK rows: %d" % (NGB, len(raw_blob))) print(" of those, %d match on the headline only" % (len(raw_blob) - len(raw))) print(" %d of the headline-only rows carry a non-monarchy Royal" % len(removed_blob)) print(" (Royal Air Force, Royal Opera House, Royal Caribbean, the Royal") print(" Society, the Royal Courts of Justice), which is why the report") print(" quotes the title-only figure.") hl_reasons = collections.Counter(royal_reason(blob(r)) for r in removed_blob) for reason, n in hl_reasons.most_common(): print(" %-3d %s" % (n, reason)) # ── 2. UK cybersecurity titles ─────────────────────────────────── print() print("=" * 72) print("2. UK ROWS WHOSE JOB TITLE NAMES CYBERSECURITY") print("=" * 72) uk_cyber = [r for r in gb if CYBER_NARROW.search(r["title"])] print("rule: the job title matches %s" % CYBER_NARROW.pattern) print("matches, title only, of %d UK rows: %d" % (NGB, len(uk_cyber))) for r in uk_cyber: print(" %-45s %s" % (norm(r["title"])[:45], oname(D, r["outlet"]))) uk_cyber_blob = [r for r in gb if CYBER_NARROW.search(blob(r))] print("headline included: %d of %d UK rows" % (len(uk_cyber_blob), NGB)) for r in uk_cyber_blob: if r not in uk_cyber: print(" + %-40s %-18s (matched in the headline)" % (norm(r["title"])[:40], oname(D, r["outlet"]))) print() print("widened lexicon, title only. Each match names the word that matched.") uk_wide = [] for r in gb: words = wide_words(r["title"]) if words: uk_wide.append((r, words)) print("matches: %d of %d UK rows" % (len(uk_wide), NGB)) for r, words in uk_wide: print(" %-45s %-18s %s" % (norm(r["title"])[:45], oname(D, r["outlet"]), "; ".join(words))) uk_word_counts = collections.Counter() for _, words in uk_wide: for w in words: uk_word_counts[w] += 1 print(" per word: " + ", ".join( "%s %d" % (w, uk_word_counts.get(w, 0)) for w, _ in WIDE)) uk_wide_blob = [r for r in gb if wide_words(blob(r))] print("widened lexicon, headline included: %d of %d UK rows" % (len(uk_wide_blob), NGB)) print("national security in the title, UK: %d of %d rows. It is kept OUT of" % (sum(1 for r in gb if NATIONAL_SECURITY.search(r["title"])), NGB)) print(" the lexicon: it names a defence and intelligence round, not a") print(" computer-security round.") # ── 3. the largest UK outlets ──────────────────────────────────── print() print("=" * 72) print("3. ROYAL, CYBER AND TECHNOLOGY TITLES AT THE LARGEST OUTLETS BY UK ROWS") print("=" * 72) gb_rows_by_outlet = collections.defaultdict(list) for r in gb: gb_rows_by_outlet[r["outlet"]].append(r) def counts_for(key): rs = gb_rows_by_outlet.get(key, []) return { "uk_rows": len(rs), "royal": sum(1 for r in rs if ROYAL_START.search(r["title"]) and not royal_reason(r["title"])), "cyber": sum(1 for r in rs if CYBER_NARROW.search(r["title"])), "cyber_wide": sum(1 for r in rs if wide_words(r["title"])), "tech": sum(1 for r in rs if TECH_RE.search(C._norm(r["title"]))), } print("UK rows = rows whose country_code is GB at that canonical outlet.") print("Every column is a title-only count over those same UK rows.") print("Both tables rank the same publishable canonical outlets the site's") print("/outlets/ pages use: parent-company domains and outlets missing a") print("name, a country or a type are excluded.") print() print("TABLE A. The 25 largest outlets by UK-located rows, whatever country") print("the outlet itself is in. outlet_country is the outlet's country, not") print("the person's.") print("%-4s %-30s %-16s %8s %6s %6s %6s %6s" % ("rank", "outlet", "outlet country", "UK rows", "royal", "cyber", "wide", "tech")) all_ranked = D["gb_ranked"] table_a = [] for rank, key in enumerate(all_ranked[:25], start=1): c = counts_for(key) table_a.append((rank, key, c)) print("%-4d %-30s %-16s %8d %6d %6d %6d %6d" % (rank, oname(D, key)[:30], (D["outlets"][key]["country"] or "")[:16], c["uk_rows"], c["royal"], c["cyber"], c["cyber_wide"], c["tech"])) non_uk_a = [k for _, k, _ in table_a if D["outlets"][k]["country_code"] != UK] print("%d of the 25 are outlets table A puts outside the United Kingdom: %s" % (len(non_uk_a), ", ".join(oname(D, k) for k in non_uk_a))) print() print("TABLE B. The 20 largest UK-country outlets by UK-located rows.") print("Same columns, restricted to outlets whose own country is GB.") uk_country_ranked = [k for k in all_ranked if D["outlets"][k]["country_code"] == UK] print("%-4s %-30s %8s %6s %6s %6s %6s" % ("rank", "outlet", "UK rows", "royal", "cyber", "wide", "tech")) table_b = [] for rank, key in enumerate(uk_country_ranked[:20], start=1): c = counts_for(key) table_b.append((rank, key, c)) print("%-4d %-30s %8d %6d %6d %6d %6d" % (rank, oname(D, key)[:30], c["uk_rows"], c["royal"], c["cyber"], c["cyber_wide"], c["tech"])) gb_news = "gbnews.com" rank_all = ([k for k in all_ranked].index(gb_news) + 1 if gb_news in all_ranked else 0) rank_uk = ([k for k in uk_country_ranked].index(gb_news) + 1 if gb_news in uk_country_ranked else 0) print() print("GB News, %d UK rows: %s of all outlets by UK rows, %s of the" % (counts_for(gb_news)["uk_rows"], ordinal(rank_all), ordinal(rank_uk))) print("UK-country outlets. Both ranks are printed because the report walks") print("through it and it is in neither top 20 on the all-outlet ranking.") zero_cyber_a = [k for _, k, c in table_a if c["cyber"] == 0] any_royal_a = [k for _, k, c in table_a if c["royal"] > 0] print("table A: %d of the 25 have no cybersecurity title, %d have at least" % (len(zero_cyber_a), len(any_royal_a))) print("one royal title.") zero_cyber_b = [k for _, k, c in table_b if c["cyber"] == 0] any_royal_b = [k for _, k, c in table_b if c["royal"] > 0] print("table B: %d of the 20 have no cybersecurity title, %d have at least" % (len(zero_cyber_b), len(any_royal_b))) print("one royal title.") # ── 4. the world ───────────────────────────────────────────────── print() print("=" * 72) print("4. CYBERSECURITY TITLES WORLDWIDE") print("=" * 72) world_cyber = [r for r in rows if CYBER_NARROW.search(r["title"])] w_outlets = collections.Counter(r["outlet"] for r in world_cyber) print("title-only matches: %d of %d rows, across %d canonical outlets" % (len(world_cyber), N, len(w_outlets))) for r in sorted(world_cyber, key=lambda r: (oname(D, r["outlet"]).lower(), norm(r["title"]))): print(" %-55s %-28s %s" % (norm(r["title"])[:55], oname(D, r["outlet"])[:28], r["country_code"])) print("by outlet: " + ", ".join("%s %d" % (oname(D, k), n) for k, n in w_outlets.most_common())) print("by country of the person: " + ", ".join( "%s %d" % (c, n) for c, n in collections.Counter(r["country_code"] for r in world_cyber).most_common())) world_cyber_blob = [r for r in rows if CYBER_NARROW.search(blob(r))] print("headline included: %d of %d rows" % (len(world_cyber_blob), N)) print() print("the 50 largest newsrooms on file, as the site's /outlets/ table") print("ranks them (parent-company domains and untyped or unnamed outlets") print("excluded), with their cybersecurity-title count:") print("%-34s %8s %6s %6s" % ("outlet", "rows", "cyber", "wide")) for key in D["top50"]: rs = [r for r in rows if r["outlet"] == key] n_c = sum(1 for r in rs if CYBER_NARROW.search(r["title"])) n_w = sum(1 for r in rs if wide_words(r["title"])) print("%-34s %8d %6d %6d" % (oname(D, key)[:34], len(rs), n_c, n_w)) top50_cyber = {k: sum(1 for r in rows if r["outlet"] == k and CYBER_NARROW.search(r["title"])) for k in D["top50"]} zero50 = [k for k in D["top50"] if top50_cyber[k] == 0] print("%d of the 50 have none; %d have at least one." % (len(zero50), 50 - len(zero50))) world_wide = [(r, wide_words(r["title"])) for r in rows if wide_words(r["title"])] print() print("widened lexicon, title only: %d of %d rows" % (len(world_wide), N)) ww = collections.Counter() for _, words in world_wide: for w in words: ww[w] += 1 print(" per word: " + ", ".join("%s %d" % (w, ww.get(w, 0)) for w, _ in WIDE)) print(" the widening adds %d rows to the %d cybersecurity titles:" % (len(world_wide) - len(world_cyber), len(world_cyber))) for r, words in world_wide: if not CYBER_NARROW.search(r["title"]): print(" %-50s %-22s %s" % (norm(r["title"])[:50], oname(D, r["outlet"])[:22], "; ".join(words))) world_wide_blob = [r for r in rows if wide_words(blob(r))] print(" headline included: %d of %d rows" % (len(world_wide_blob), N)) priv = [(r, [w for w in wide_words(r["title"]) if w in PRIVACY_WORDS]) for r in rows] priv = [(r, w) for r, w in priv if w] print() print("privacy / data protection / GDPR in the title: %d of %d rows" % (len(priv), N)) for r, words in priv: print(" %-50s %-22s %s %s" % (norm(r["title"])[:50], oname(D, r["outlet"])[:22], r["country_code"], "; ".join(words))) priv_blob = [r for r in rows if any(rx.search(blob(r)) for w, rx in WIDE_RE if w in PRIVACY_WORDS)] print(" headline included: %d of %d rows" % (len(priv_blob), N)) print(" at the 50 largest newsrooms: %d" % sum(1 for r, _ in priv if r["outlet"] in set(D["top50"]))) print(" UK rows: %d of %d" % (sum(1 for r, _ in priv if r["country_code"] == UK), NGB)) # ── 5. comparators ─────────────────────────────────────────────── print() print("=" * 72) print("5. THE SAME UNIT FOR OTHER BEATS (title-only counts)") print("=" * 72) beats = dict(C.BEATS_RE) print("%-16s %10s %10s %10s %10s" % ("beat", "world", "per 1,000", "UK", "per 1,000")) comp_rows = [] for name in BEAT_COMPARATORS: rx = beats[name] w = sum(1 for r in rows if rx.search(C._norm(r["title"]))) u = sum(1 for r in gb if rx.search(C._norm(r["title"]))) comp_rows.append((name, w, u, rx.pattern, "data/classify.py")) print("%-16s %10d %10.2f %10d %10.2f" % (name, w, 1000.0 * w / N, u, 1000.0 * u / NGB)) extra = [ ("royals (kept)", len([r for r in rows if ROYAL_START.search(r["title"]) and not royal_reason(r["title"])]), len(kept), ROYAL_START.pattern + " minus the hand-removed titles"), ("cybersecurity (this report)", len(world_cyber), len(uk_cyber), CYBER_NARROW.pattern), ("cyber, widened", len(world_wide), len(uk_wide), " | ".join(p for _, p in WIDE)), ("privacy", len(priv), sum(1 for r, _ in priv if r["country_code"] == UK), " | ".join(p for w, p in WIDE if w in PRIVACY_WORDS)), ] for name, w, u, pattern in extra: comp_rows.append((name, w, u, pattern, "this report")) print("%-16s %10d %10.2f %10d %10.2f" % (name, w, 1000.0 * w / N, u, 1000.0 * u / NGB)) # ── 6. provenance, reconciliation, dataset checks ──────────────── print() print("=" * 72) print("6. LEXICON PROVENANCE AND RECONCILIATION") print("=" * 72) print("this report's own regexes: " + ", ".join(OWN_REGEXES)) print("the site classifier's own regexes, imported from data/classify.py:") print(" " + ", ".join(CLASSIFIER_REGEXES)) print() cls_world = [r for r in rows if CYBER_CLASSIFIER.search(C._norm(r["title"]))] cls_uk = [r for r in gb if CYBER_CLASSIFIER.search(C._norm(r["title"]))] print("the classifier's cybersecurity beat regex over the job title:") print(" %s" % CYBER_CLASSIFIER.pattern) print(" world %d of %d, UK %d of %d" % (len(cls_world), N, len(cls_uk), NGB)) print("this report's narrow cybersecurity regex over the job title:") print(" %s" % CYBER_NARROW.pattern) print(" world %d of %d, UK %d of %d" % (len(world_cyber), N, len(uk_cyber), NGB)) cls_extra = [r for r in cls_world if not CYBER_NARROW.search(r["title"])] print(" the %d-row difference, every title:" % len(cls_extra)) for r in sorted(cls_extra, key=lambda r: norm(r["title"])): print(" %-58s %-24s %s" % (norm(r["title"])[:58], oname(D, r["outlet"])[:24], r["country_code"])) print(" all %d rows of the narrow rule are inside the classifier's %d." % (sum(1 for r in world_cyber if CYBER_CLASSIFIER.search(C._norm(r["title"]))), len(cls_world))) print(" The classifier's beat regex is what /journalists/cybersecurity/") print(" counts, over title + headline; over the title alone it is the %d" % len(cls_world)) print(" above.") cls_blob = [r for r in rows if CYBER_CLASSIFIER.search(C._norm(blob(r)))] cls_blob_uk = [r for r in gb if CYBER_CLASSIFIER.search(C._norm(blob(r)))] print(" Over title + headline, the field pair the classifier actually") print(" reads, the same regex reaches %d rows worldwide and %d UK rows." % (len(cls_blob), len(cls_blob_uk))) comp_rows.append(("cybersecurity (site classifier)", len(cls_world), len(cls_uk), CYBER_CLASSIFIER.pattern, "data/classify.py")) print(" added to beat_title_counts.csv as cybersecurity (site classifier)") print(" beside cybersecurity (this report) %d." % len(world_cyber)) print() bare_look = [r for r in rows if BARE_CYBER_LOOKAHEAD.search(r["title"])] bare_plain = [r for r in rows if BARE_CYBER_PLAIN.search(r["title"])] print("bare cyber, both forms, world titles:") print(" %-26s %d of %d" % (BARE_CYBER_LOOKAHEAD.pattern, len(bare_look), N)) print(" %-26s %d of %d" % (BARE_CYBER_PLAIN.pattern, len(bare_plain), N)) print(" the lexicon uses the lookahead. The one row plain cyber adds is") print(" already counted by the narrow cybersecurity rule, because its") print(" title spells the word with a space:") for r in bare_plain: if r not in bare_look: print(" %-58s %s" % (norm(r["title"])[:58], oname(D, r["outlet"]))) print() world_royal_raw = [r for r in rows if ROYAL_START.search(r["title"])] world_royal_removed = [r for r in world_royal_raw if royal_reason(r["title"])] print("the hand-removal list over world titles: %d of %d raw royal matches" % (len(world_royal_removed), len(world_royal_raw))) for r in world_royal_removed: print(" - %-52s %-26s %s" % (norm(r["title"])[:52], oname(D, r["outlet"])[:26], royal_reason(r["title"]))) print() matched = [] seen_ids = set() for r in list(raw) + [x for x, _ in uk_wide] + list(world_cyber): if id(r) not in seen_ids: seen_ids.add(id(r)) matched.append(r) stamps = sorted(r["fetched_at"] for r in matched if r["fetched_at"]) print("fetched_at over the %d matched rows the CSVs publish:" % len(matched)) print(" min %s" % stamps[0]) print(" median %s" % stamps[len(stamps) // 2]) print(" max %s" % stamps[-1]) print() dupe_rows = [r for r in gb if r["outlet"] == "gbnews.com" and re.fullmatch(r"\s*royal correspondent\s*", (r["title"] or ""), re.I)] print("dedupe check. Rows at GB News whose title is exactly Royal") print("Correspondent: %d. Distinct full_name among them: %d. Distinct" % (len(dupe_rows), len({r["full_name"] for r in dupe_rows}))) print("company_domain among them: %d. The round-2 dedupe key is" % len({r["company_domain"] for r in dupe_rows})) print("(full_name, title, company_domain), so two rows with one name could") print("not both survive it; the count above says whether they are two") print("people. No name is printed.") # ── CSVs ───────────────────────────────────────────────────────── write_csvs(D, rows, gb, kept, removed, raw, uk_wide, table_a, table_b, world_cyber, top50_cyber, comp_rows, cls_world, cls_uk, cls_extra, matched, dupe_rows) print() print("=" * 72) print("HEADLINES") print("=" * 72) print("UK rows whose title names the royals, kept: %d of %d UK rows" % (len(kept), NGB)) print("UK rows whose title names cybersecurity: %d of %d UK rows" % (len(uk_cyber), NGB)) print("world rows whose title names cybersecurity: %d of %d rows, %d outlets" % (len(world_cyber), N, len(w_outlets))) print("of the 50 largest newsrooms, with none: %d of 50" % len(zero50)) print("wrote 8 CSVs, lexicon.txt and a copy of this script to " "static/data/%s/" % SLUG) # ── CSVs ───────────────────────────────────────────────────────────── def write_csvs(D, rows, gb, kept, removed, raw, uk_wide, table_a, table_b, world_cyber, top50_cyber, comp_rows, cls_world, cls_uk, cls_extra, matched, dupe_rows): N = len(rows) NGB = len(gb) def w(name, header, body): with open(os.path.join(OUT, name), "w", newline="", encoding="utf-8") as fh: out = csv.writer(fh) out.writerow(header) out.writerows(body) # uk_royal_titles.csv: every UK row the royal regex reaches, on either # rule, with the hand-read verdict. matched_in says which field matched; # only the title rows are in the report's count. body = [] seen = {id(r) for r in raw} rows_out = [(r, "title") for r in raw] rows_out += [(r, "headline") for r in gb if ROYAL_START.search(blob(r)) and id(r) not in seen] for r, where in sorted(rows_out, key=lambda t: ( t[1], bool(royal_reason(blob(t[0]))), oname(D, t[0]["outlet"]).lower(), norm(t[0]["title"]))): text = r["title"] if where == "title" else blob(r) reason = royal_reason(text) body.append([norm(r["title"]), oname(D, r["outlet"]), odomain(D, r["outlet"]), r["role"], noun_of(r["title"]), where, "removed" if reason else "kept", reason, "yes" if ROYAL_WORD.search(text) else "no"]) w("uk_royal_titles.csv", ["title", "outlet", "outlet_domain", "role", "title_noun", "matched_in", "kept_or_removed", "reason", "matches_whole_word_royal"], body) # uk_security_titles.csv body = [] seen = {id(r) for r, _ in uk_wide} for r, words in uk_wide: body.append([norm(r["title"]), oname(D, r["outlet"]), odomain(D, r["outlet"]), r["role"], "title", "; ".join(words), "yes" if CYBER_NARROW.search(r["title"]) else "no"]) for r in gb: if id(r) in seen: continue words = wide_words(blob(r)) if words: body.append([norm(r["title"]), oname(D, r["outlet"]), odomain(D, r["outlet"]), r["role"], "headline", "; ".join(words), "yes" if CYBER_NARROW.search(blob(r)) else "no"]) w("uk_security_titles.csv", ["title", "outlet", "outlet_domain", "role", "matched_in", "words_matched", "narrow_cybersecurity"], body) # uk_top_outlets_royal_cyber_tech.csv: the 25 largest outlets by # UK-located rows, whatever country the outlet itself is in. body = [] for rank, key, c in table_a: body.append([rank, oname(D, key), odomain(D, key), D["outlets"][key]["country"], D["outlets"][key]["country_code"], D["outlets"][key]["type"], c["uk_rows"], c["royal"], c["cyber"], c["cyber_wide"], c["tech"]]) w("uk_top_outlets_royal_cyber_tech.csv", ["rank_by_uk_rows", "outlet", "outlet_domain", "outlet_country", "outlet_country_code", "outlet_type", "uk_rows", "royal_titles", "cybersecurity_titles", "security_titles_widened", "technology_titles"], body) # uk_country_outlets_royal_cyber_tech.csv: the 20 largest UK-country # outlets by UK-located rows. body = [] for rank, key, c in table_b: body.append([rank, oname(D, key), odomain(D, key), D["outlets"][key]["country"], D["outlets"][key]["country_code"], D["outlets"][key]["type"], c["uk_rows"], c["royal"], c["cyber"], c["cyber_wide"], c["tech"]]) w("uk_country_outlets_royal_cyber_tech.csv", ["rank_by_uk_rows_uk_country_outlets", "outlet", "outlet_domain", "outlet_country", "outlet_country_code", "outlet_type", "uk_rows", "royal_titles", "cybersecurity_titles", "security_titles_widened", "technology_titles"], body) # world_cybersecurity_titles.csv body = [] for r in sorted(world_cyber, key=lambda r: (oname(D, r["outlet"]).lower(), norm(r["title"]))): body.append([norm(r["title"]), oname(D, r["outlet"]), odomain(D, r["outlet"]), r["role"], r["country_code"], E.COUNTRIES.get(r["country_code"], "")]) w("world_cybersecurity_titles.csv", ["title", "outlet", "outlet_domain", "role", "country_code", "country"], body) # top50_newsrooms_cyber.csv rows_by_outlet = collections.Counter(r["outlet"] for r in rows) gb_by_outlet = D["gb_by_outlet"] body = [] for rank, key in enumerate(D["top50"], start=1): rs = [r for r in rows if r["outlet"] == key] body.append([rank, oname(D, key), odomain(D, key), D["outlets"][key]["country"], D["outlets"][key]["type"], rows_by_outlet[key], gb_by_outlet[key], top50_cyber[key], sum(1 for r in rs if wide_words(r["title"])), sum(1 for r in rs if ROYAL_START.search(r["title"]) and not royal_reason(r["title"])), sum(1 for r in rs if TECH_RE.search(C._norm(r["title"])))]) w("top50_newsrooms_cyber.csv", ["rank_by_editorial_rows", "outlet", "outlet_domain", "outlet_country", "outlet_type", "editorial_rows", "uk_located_rows", "cybersecurity_titles", "security_titles_widened", "royal_titles", "technology_titles"], body) # beat_title_counts.csv body = [] for name, world, uk, pattern, source in comp_rows: body.append([name, world, round(1000.0 * world / N, 2), N, uk, round(1000.0 * uk / NGB, 2), NGB, pattern, source]) w("beat_title_counts.csv", ["beat", "world_titles", "world_per_1000_rows", "world_denominator", "uk_titles", "uk_per_1000_rows", "uk_denominator", "regex", "regex_source"], body) # cybersecurity_reconciliation.csv: the 5 titles the classifier's own beat # regex reaches and this report's narrow rule does not. body = [] for r in sorted(cls_extra, key=lambda r: norm(r["title"])): body.append([norm(r["title"]), oname(D, r["outlet"]), odomain(D, r["outlet"]), r["role"], r["country_code"], E.COUNTRIES.get(r["country_code"], ""), "; ".join(wide_words(r["title"]))]) w("cybersecurity_reconciliation.csv", ["title", "outlet", "outlet_domain", "role", "country_code", "country", "words_matched"], body) # lexicon.txt with open(os.path.join(OUT, "lexicon.txt"), "w", encoding="utf-8") as fh: fh.write("Every regex used in this dataset. All are matched " "case-insensitively.\n\n") fh.write("PROVENANCE\n") fh.write(" This report's own regexes, written for it and not used " "anywhere else on\n the site: %s.\n" % ", ".join(OWN_REGEXES)) fh.write(" The site classifier's own regexes, imported from " "data/classify.py and not\n retyped, so they match the " "/beats/ and /journalists/ pages: %s.\n\n" % ", ".join(CLASSIFIER_REGEXES)) fh.write("ROYAL, word-start (the rule the report uses)\n %s\n\n" % ROYAL_START.pattern) fh.write("ROYAL, whole word (the scout's rule, printed beside it)\n" " %s\n\n" % ROYAL_WORD.pattern) fh.write("CYBERSECURITY, narrow\n %s\n\n" % CYBER_NARROW.pattern) fh.write("SECURITY, widened. One regex per word so a match can name " "the word that matched.\n") for word, pattern in WIDE: fh.write(" %-22s %s\n" % (word, pattern)) fh.write("\nBARE CYBER, the two forms\n") fh.write(" used here %s\n" % BARE_CYBER_LOOKAHEAD.pattern) fh.write(" plain %s\n" % BARE_CYBER_PLAIN.pattern) fh.write(" The lookahead is the one in the lexicon. It keeps a title " "that spells the\n word with a space (Cyber Security " "Specialist) out of the bare-cyber count,\n because the " "narrow cybersecurity rule already counts it. Over the job\n" " title the lookahead matches 5 rows and the plain form 6; " "the one row of\n difference is that title. Both counts are " "in facts_console.txt.\n") fh.write("\nCYBERSECURITY, the site classifier's own beat regex " "(data/classify.py),\nprinted so the number on " "/journalists/cybersecurity/ and the number in this\nreport " "reconcile\n %s\n" % CYBER_CLASSIFIER.pattern) fh.write(" Over the job title it matches 28 rows worldwide and 2 UK " "rows, against\n 23 and 2 for this report's narrow rule. " "The 5 extra titles are in\n cybersecurity_reconciliation." "csv.\n") fh.write("\nNOT in the widened lexicon:\n national security %s\n" % NATIONAL_SECURITY.pattern) fh.write(" It names a defence and intelligence round, not a computer-" "security round.\n") fh.write("\nTITLE NOUN, first match wins in this order\n") for label, rx in NOUN_ORDER: fh.write(" %-16s %s\n" % (label, rx.pattern)) fh.write(" other no match above\n") fh.write("\nBEAT LEXICONS, copied from data/classify.py\n") for name, rx in C.BEATS_RE: fh.write(" %-16s %s\n" % (name, rx.pattern)) fh.write("\nHAND-REMOVAL LIST for the royal set. A lowercased title " "containing the\nstring on the left is removed for the reason " "on the right.\n") for needle, reason in ROYAL_REMOVE: fh.write(" %-26s %s\n" % (needle, reason)) fh.write(" The list is applied the same way to titles and to the " "headline-inclusive\n blob, so a learned society is removed " "wherever it appears: the Royal\n Society, the Royal Society " "of Arts, the Royal Meteorological Society and\n the Royal " "Geographical Society are all removals for the same reason.\n") # The script itself, shipped with the data it wrote. import shutil shutil.copyfile(os.path.abspath(__file__), os.path.join(OUT, "report_royal_vs_cyber.py")) # ── sanity check against a public list ─────────────────────────────── SANITY_URLS = [ ("wikipedia-bbc-news", "https://en.wikipedia.org/wiki/BBC_News"), ("wikipedia-bbc-newsreaders-reporters", "https://en.wikipedia.org/wiki/List_of_BBC_newsreaders_and_reporters"), ("wikipedia-cat-royal-correspondents", "https://en.wikipedia.org/wiki/Category:Royal_correspondents"), ("wikipedia-royal-correspondent", "https://en.wikipedia.org/wiki/Royal_correspondent"), ("wikipedia-cat-cybersecurity-journalists", "https://en.wikipedia.org/wiki/Category:Cybersecurity_journalists"), ("wikipedia-cat-computer-security-writers", "https://en.wikipedia.org/wiki/Category:Computer_security_writers"), ("bbc-correspondents-index", "https://www.bbc.co.uk/news/correspondents"), ("pressgazette-search-royal", "https://pressgazette.co.uk/?s=royal+correspondent"), ] # Pages whose current revision id is recorded so every quoted figure has an # oldid permalink rather than a moving URL. REVID_TITLES = ["BBC_News", "List_of_BBC_newsreaders_and_reporters", "Category:Royal_correspondents"] REVID_API = ("https://en.wikipedia.org/w/api.php?action=query&prop=revisions" "&titles=%s&rvprop=ids&format=json") def sanity(): """Fetched through the Webshare rotating gateway, never from this IP. Prints the raw evidence; the counts and what they mean are written up in the facts file. Every URL tried is listed, including the ones that 404. """ import html as htmlmod sys.path.insert(0, os.path.join(os.path.dirname(ROOT), "ai_sending_tool_v2")) from proxy_fetch import fetch_response_with_fallback import json as jsonmod print("=== revision ids, for oldid permalinks") for title in REVID_TITLES: url = REVID_API % title.replace(":", "%3A") try: r = fetch_response_with_fallback(url, "journalistlabs_royal_sanity", timeout=45) pages = jsonmod.loads(r.text)["query"]["pages"] for pid, page in pages.items(): rev = page.get("revisions", [{}])[0].get("revid") print(" %-42s pageid %s revid %s" % (title, pid, rev)) print(" permalink https://en.wikipedia.org/w/index.php" "?title=%s&oldid=%s" % (title, rev)) except Exception as exc: # noqa: BLE001 print(" %s FAILED %s" % (title, exc)) for name, url in SANITY_URLS: try: r = fetch_response_with_fallback( url, "journalistlabs_royal_sanity", timeout=45) except Exception as exc: # noqa: BLE001 print("=== %s FAILED %s\n %s" % (name, url, exc)) continue txt = re.sub(r"|", "", r.text, flags=re.S) txt = re.sub(r"\s+", " ", htmlmod.unescape(re.sub(r"<[^>]+>", " ", txt))) print("=== %s HTTP %d %d chars %s" % (name, r.status_code, len(txt), url)) if name == "wikipedia-bbc-news": box = re.search(r']+>", " ", box.group(0)))) emp = re.search(r"Number of employees (.{0,60})", flat) print(" infobox, number of employees: %s" % (emp.group(1).strip() if emp else "not found")) for m in re.findall(r"[^.]{0,160}5,500[^.]{0,160}", txt)[:4]: print(" body: " + m.strip()[:300]) block = re.search(r"mw-category(.*?)printfooter", r.text, re.S) if block: members = [t for _, t in re.findall( r'