{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Replay a World Leaders Daily research snapshot\n",
        "\n",
        "Choose a saved manifest or an explicit public versioned manifest URL. The current manifest pins one immutable version during this run. This notebook does not refresh source feeds, execute scheduled jobs or read private quarantine/submissions. Save the cited manifest and release files because public retention can expire older versions.\n",
        "\n",
        "The cells use only Python's standard library. Hashes confirm bytes against the publisher's manifest; they do not independently establish the truth of source claims."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "MANIFEST = \"site/downloads/manifest.json\"\n",
        "# Example remote input after replacing VERSION with a cited release:\n",
        "# MANIFEST = \"https://worldleaders.website/downloads/releases/VERSION/manifest.json\"\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "\"\"\"Replay a cited World Leaders Daily release using only the Python stdlib.\n",
        "\n",
        "python notebooks/replay_snapshot.py --manifest site/downloads/manifest.json\n",
        "python notebooks/replay_snapshot.py --manifest https://worldleaders.website/downloads/releases/VERSION/manifest.json\n",
        "\n",
        "Checks every declared byte count and SHA-256 before interpreting a file, then\n",
        "combines ordered JSON parts and checks dataset totals. No source refresh occurs.\n",
        "\"\"\"\n",
        "from __future__ import annotations\n",
        "\n",
        "import argparse\n",
        "import gzip\n",
        "import hashlib\n",
        "import json\n",
        "from pathlib import Path, PurePosixPath\n",
        "from urllib.parse import urljoin, urlparse\n",
        "from urllib.request import Request, urlopen\n",
        "\n",
        "MAX_FILE_BYTES = 24 * 1024 * 1024\n",
        "\n",
        "\n",
        "def _read(location):\n",
        "    value = str(location)\n",
        "    if urlparse(value).scheme in {\"https\", \"http\"}:\n",
        "        with urlopen(Request(value, headers={\"User-Agent\": \"WorldLeadersDaily-ResearchReplay/1.0\"}), timeout=45) as response:\n",
        "            content = response.read(MAX_FILE_BYTES + 1)\n",
        "    else:\n",
        "        content = Path(value).read_bytes()\n",
        "    if len(content) > MAX_FILE_BYTES:\n",
        "        raise ValueError(\"Download exceeds the advertised release file limit\")\n",
        "    return content\n",
        "\n",
        "\n",
        "def replay(manifest_location):\n",
        "    location = str(manifest_location)\n",
        "    manifest = json.loads(_read(location))\n",
        "    version = manifest[\"version\"]\n",
        "    if not isinstance(version, str) or not version or any(c not in \"0123456789TZ-abcdefundated\" for c in version):\n",
        "        raise ValueError(\"Invalid snapshot version\")\n",
        "    prefix = f\"downloads/releases/{version}/\"\n",
        "    remote = urlparse(location).scheme in {\"https\", \"http\"}\n",
        "    if remote:\n",
        "        parsed = urlparse(location)\n",
        "        base = f\"{parsed.scheme}://{parsed.netloc}/\"\n",
        "    else:\n",
        "        path = Path(location).resolve()\n",
        "        # A saved immutable release lives below site/downloads/releases/VERSION;\n",
        "        # a current manifest lives directly below site/downloads.\n",
        "        if path.parent.name == version and path.parent.parent.name == \"releases\":\n",
        "            base = path.parent.parent.parent.parent\n",
        "        elif path.parent.name in {\"downloads\", \"dataset\"}:\n",
        "            base = path.parent.parent\n",
        "            if path.parent.name == \"dataset\":\n",
        "                base = base / \"site\"\n",
        "        else:\n",
        "            raise ValueError(\"Place a local manifest under downloads/ or downloads/releases/VERSION/\")\n",
        "    files = {}\n",
        "    for entry in manifest[\"files\"]:\n",
        "        relative = entry[\"path\"]\n",
        "        path = PurePosixPath(relative)\n",
        "        if not relative.startswith(prefix) or path.is_absolute() or \"..\" in path.parts or path.name != relative[len(prefix):]:\n",
        "            raise ValueError(\"Manifest refers outside the pinned flat release\")\n",
        "        if relative in files:\n",
        "            raise ValueError(\"Manifest lists a file more than once\")\n",
        "        source = urljoin(base, relative) if remote else Path(base) / relative\n",
        "        content = _read(source)\n",
        "        if len(content) != entry[\"bytes\"] or hashlib.sha256(content).hexdigest() != entry[\"sha256\"]:\n",
        "            raise ValueError(f\"Snapshot integrity check failed: {path.name}\")\n",
        "        decoded = gzip.decompress(content) if entry.get(\"compression\") == \"gzip\" else content\n",
        "        files[relative] = (entry, decoded)\n",
        "    datasets = {}\n",
        "    for name, metadata in manifest[\"datasets\"].items():\n",
        "        rows = []\n",
        "        paths = metadata[\"formats\"].get(\"json\")\n",
        "        if not paths:\n",
        "            raise ValueError(f\"No reproducible JSON parts for {name}\")\n",
        "        for number, relative in enumerate(paths, 1):\n",
        "            if relative not in files:\n",
        "                raise ValueError(\"Dataset part is absent from verified files\")\n",
        "            entry, content = files[relative]\n",
        "            part = json.loads(content)\n",
        "            if entry.get(\"dataset\") != name or entry.get(\"part\") != number or entry.get(\"parts\") != len(paths):\n",
        "                raise ValueError(\"Dataset part order/identity does not match the manifest\")\n",
        "            if part.get(\"generated_at\") != manifest.get(\"generated_at\") or len(part[name]) != entry[\"rows\"]:\n",
        "                raise ValueError(\"Dataset part rows/timestamp do not match the manifest\")\n",
        "            rows.extend(part[name])\n",
        "        if len(rows) != metadata[\"rows\"]:\n",
        "            raise ValueError(\"Dataset totals do not match the manifest\")\n",
        "        datasets[name] = rows\n",
        "    events = datasets.get(\"events\", [])\n",
        "    identifiers = [row[\"item_id\"] for row in events if row.get(\"item_id\")]\n",
        "    if len(identifiers) != len(set(identifiers)):\n",
        "        raise ValueError(\"Snapshot contains repeated event identities\")\n",
        "    summary = {\"version\": version, \"generated_at\": manifest.get(\"generated_at\"),\n",
        "               \"verified_files\": len(files), \"dataset_rows\": {name: len(rows) for name, rows in datasets.items()},\n",
        "               \"event_dates_with_evidence\": sum(bool(row.get(\"event_date\") and row.get(\"date_evidence\")) for row in events),\n",
        "               \"confirmed_meetings_and_calls\": len(datasets.get(\"meetings\", [])),\n",
        "               \"limits\": \"Counts describe collected publications and evidenced records; source coverage changes are not changes in activity.\"}\n",
        "    return manifest, datasets, summary\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "manifest, datasets, summary = replay(MANIFEST)\n",
        "print(json.dumps(summary, ensure_ascii=False, indent=2))\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Publications, occurrences and coverage\n",
        "\n",
        "Publication counts describe collected source material. An occurrence count requires an event date and its supporting evidence. Current officeholder labels cannot identify historical participants: use person_id/person_name with tenure_status. Comparison across sources is affected by diary products, outages and coverage changes."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "from collections import Counter\n",
        "events = datasets.get(\"events\", [])\n",
        "print(\"Record kinds:\", dict(Counter(row.get(\"kind\", \"unknown\") for row in events)))\n",
        "print(\"Source states:\", dict(Counter(row.get(\"source_status\", \"unknown\") for row in events)))\n",
        "occurrences = [row for row in events if row.get(\"event_date\") and row.get(\"date_evidence\")]\n",
        "print(\"Evidenced occurrence records:\", len(occurrences))\n",
        "print(\"Verified named attribution:\", sum(bool(row.get(\"person_id\")) for row in events))\n",
        "# A false/empty location does not prove the officeholder stayed at home.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Preserve a reproducible citation\n",
        "\n",
        "The URL and version below identify the release, not a mutable current alias. Cite the original publisher as well when reporting an individual claim. No independent human review is assumed."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "citation = {\"title\": \"World Leaders Daily research dataset\",\n",
        "            \"version\": manifest[\"version\"], \"snapshot_generated_at\": manifest.get(\"generated_at\"),\n",
        "            \"manifest_url\": \"https://worldleaders.website/downloads/releases/\" + manifest[\"version\"] + \"/manifest.json\",\n",
        "            \"license\": \"CC BY 4.0 compilation; source content retains original rights\"}\n",
        "print(json.dumps(citation, ensure_ascii=False, indent=2))\n"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}