#!/usr/bin/env python3
"""
wayback_fixer.py — Replace dead links to a defunct site with Wayback Machine snapshot links.
For every HTML file in a folder, the script:
1. Finds every href/src that points to the dead domain (default: baptistbiblebelievers.com).
2. Looks up each unique URL once in the Wayback Machine (availability API, then the CDX API
as a fallback), finding the most recent successful snapshot.
3. Writes a corrected copy of each changed file to an output folder (originals are never touched).
4. Writes a CSV report of every dead URL: how many times you link to it, which files,
whether a snapshot was found, and the snapshot link used.
Lookups are cached in wayback_cache.json, so if the run is interrupted (or you run it again
on more files) it picks up where it left off without re-querying.
USAGE
Preview only (no files written, report still produced):
python wayback_fixer.py "C:\\path\\to\\pages"
Write corrected copies to an output folder:
python wayback_fixer.py "C:\\path\\to\\pages" --apply
Options:
--out FOLDER where corrected copies go (default: <pages folder>_fixed)
--domain NAME dead domain to fix (default: baptistbiblebelievers.com)
--report FILE CSV report name (default: wayback_report.csv)
--delay SECONDS pause between Wayback lookups (default: 1.5) — be polite to archive.org
--ext .html,.htm file extensions to scan
No extra packages needed — uses only the Python standard library (Python 3.8+).
"""
import argparse
import csv
import html
import json
import os
import re
import sys
import time
import urllib.parse
import urllib.request
from collections import defaultdict
USER_AGENT = "PreceptAustin-WaybackFixer/1.0 (link repair for preceptaustin.org)"
CACHE_FILE = "wayback_cache.json"
# ---------------------------------------------------------------- link finding
def build_link_regex(domain):
"""Match href="..." or src='...' attribute values that point at the dead domain."""
dom = re.escape(domain)
# Group 1: attribute + opening quote; group 2: the URL; group 3: closing quote.
return re.compile(
r'''((?:href|src)\s*=\s*["'])'''
r'''((?:https?:)?//(?:www\.)?''' + dom + r'''[^"'\s>]*)'''
r'''(["'])''',
re.IGNORECASE,
)
def split_fragment(url):
"""Wayback ignores #anchors, so look up without it and re-attach it afterwards."""
if "#" in url:
base, frag = url.split("#", 1)
return base, "#" + frag
return url, ""
def normalize(url):
"""Give protocol-relative links a scheme so they can be looked up."""
if url.startswith("//"):
return "http:" + url
return url
# ---------------------------------------------------------------- Wayback lookups
def http_get_json(url, retries=3, timeout=30):
last_err = None
for attempt in range(retries):
try:
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
with urllib.request.urlopen(req, timeout=timeout) as resp:
body = resp.read().decode("utf-8", errors="replace").strip()
return json.loads(body) if body else None
except Exception as e: # network hiccup, 429, 5xx, bad JSON...
last_err = e
time.sleep(5 * (attempt + 1))
raise last_err
def lookup_availability(url):
api = "https://archive.org/wayback/available?url=" + urllib.parse.quote(url, safe="")
data = http_get_json(api)
snap = (data or {}).get("archived_snapshots", {}).get("closest")
if snap and snap.get("available") and str(snap.get("status", "200")).startswith("2"):
return snap["timestamp"]
return None
def lookup_cdx(url):
"""Fallback: ask the CDX index for the newest successful (200) capture."""
api = ("https://web.archive.org/cdx/search/cdx?url=" + urllib.parse.quote(url, safe="")
+ "&output=json&filter=statuscode:200&fl=timestamp&limit=-1")
data = http_get_json(api)
# Response is [["timestamp"], ["20190412..."]] — header row then data rows.
if data and len(data) > 1 and data[-1]:
return data[-1][0]
return None
def find_snapshot(url, delay):
"""Return (timestamp or None, status string)."""
try:
ts = lookup_availability(url)
time.sleep(delay)
if ts:
return ts, "archived"
ts = lookup_cdx(url)
time.sleep(delay)
if ts:
return ts, "archived (cdx)"
return None, "NOT ARCHIVED"
except Exception as e:
return None, f"ERROR: {e}"
def snapshot_link(timestamp, url):
return f"https://web.archive.org/web/{timestamp}/{url}"
# ---------------------------------------------------------------- cache
def load_cache(path):
if os.path.exists(path):
with open(path, encoding="utf-8") as f:
return json.load(f)
return {}
def save_cache(cache, path):
tmp = path + ".tmp"
with open(tmp, "w", encoding="utf-8") as f:
json.dump(cache, f, indent=1)
os.replace(tmp, path)
# ---------------------------------------------------------------- main
def read_text(path):
with open(path, "rb") as f:
raw = f.read()
for enc in ("utf-8", "cp1252", "latin-1"):
try:
return raw.decode(enc), enc
except UnicodeDecodeError:
continue
return raw.decode("utf-8", errors="replace"), "utf-8"
def main():
ap = argparse.ArgumentParser(description="Replace dead links with Wayback Machine snapshots.")
ap.add_argument("folder", help="folder containing your saved HTML pages")
ap.add_argument("--apply", action="store_true", help="write corrected copies (default: preview only)")
ap.add_argument("--out", help="output folder for corrected copies")
ap.add_argument("--domain", default="baptistbiblebelievers.com")
ap.add_argument("--report", default="wayback_report.csv")
ap.add_argument("--delay", type=float, default=1.5)
ap.add_argument("--ext", default=".html,.htm")
args = ap.parse_args()
folder = os.path.abspath(args.folder)
if not os.path.isdir(folder):
sys.exit(f"Folder not found: {folder}")
out_dir = os.path.abspath(args.out or folder.rstrip("\\/") + "_fixed")
exts = tuple(e.strip().lower() for e in args.ext.split(","))
link_re = build_link_regex(args.domain)
# Pass 1: find every dead link in every file.
files = []
for root, _, names in os.walk(folder):
if os.path.abspath(root).startswith(out_dir):
continue
for n in names:
if n.lower().endswith(exts):
files.append(os.path.join(root, n))
files.sort()
usage = defaultdict(lambda: {"count": 0, "files": set()})
for path in files:
text, _ = read_text(path)
for m in link_re.finditer(text):
base, _ = split_fragment(normalize(m.group(2)))
key = html.unescape(base) # & in page source -> & for the lookup
usage[key]["count"] += 1
usage[key]["files"].add(os.path.relpath(path, folder))
print(f"Scanned {len(files)} files; found {sum(u['count'] for u in usage.values())} links "
f"to {args.domain} ({len(usage)} unique URLs).")
if not usage:
return
# Pass 2: look each unique URL up once (cached).
cache = load_cache(CACHE_FILE)
todo = [u for u in usage if u not in cache or cache[u]["status"].startswith("ERROR")]
for i, url in enumerate(todo, 1):
ts, status = find_snapshot(url, args.delay)
cache[url] = {"timestamp": ts, "status": status}
print(f" [{i}/{len(todo)}] {status:15} {url}")
if i % 10 == 0:
save_cache(cache, CACHE_FILE)
save_cache(cache, CACHE_FILE)
# Pass 3: rewrite files.
changed_files = 0
replaced = 0
for path in files:
text, enc = read_text(path)
def repl(m):
nonlocal replaced
base, frag = split_fragment(normalize(m.group(2)))
info = cache.get(html.unescape(base), {})
if not info.get("timestamp"):
return m.group(0) # leave unarchived links alone; they're in the report
replaced += 1
return m.group(1) + snapshot_link(info["timestamp"], base) + frag + m.group(3)
new_text = link_re.sub(repl, text)
if new_text != text:
changed_files += 1
if args.apply:
dest = os.path.join(out_dir, os.path.relpath(path, folder))
os.makedirs(os.path.dirname(dest), exist_ok=True)
with open(dest, "w", encoding=enc, newline="") as f:
f.write(new_text)
# Report, most-linked first.
with open(args.report, "w", encoding="utf-8-sig", newline="") as f:
w = csv.writer(f)
w.writerow(["status", "times_linked", "dead_url", "snapshot_url", "pages_linking"])
for url, u in sorted(usage.items(), key=lambda kv: (-kv[1]["count"], kv[0])):
info = cache.get(url, {})
snap = snapshot_link(info["timestamp"], url) if info.get("timestamp") else ""
w.writerow([info.get("status", ""), u["count"], url, snap, "; ".join(sorted(u["files"]))])
missing = [u for u in usage if not cache.get(u, {}).get("timestamp")]
print()
print(f"Links replaced: {replaced} | files changed: {changed_files} | "
f"URLs with no snapshot: {len(missing)}")
print(f"Report: {os.path.abspath(args.report)}")
if args.apply:
print(f"Corrected copies: {out_dir}")
else:
print("Preview only — nothing written. Run again with --apply to write corrected copies.")
if __name__ == "__main__":
main()