diff --git a/domains/updater/update_streaming_domains.py b/domains/updater/update_streaming_domains.py new file mode 100755 index 0000000..9078e57 --- /dev/null +++ b/domains/updater/update_streaming_domains.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +""" +Update streaming_domains_whitelist.txt with domains scraped from known sources. +Always backs up the previous file before overwriting. +""" +import os +import shutil +import urllib.request +import urllib.error +from datetime import datetime + +DOMAINS_FILE = os.path.join(os.path.dirname(__file__), "..", "streaming_domains_whitelist.txt") +BACKUP_DIR = os.path.join(os.environ.get("HOME", "/tmp"), ".hermes", "backups", "domains_backup") +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +PARENT_DIR = os.path.dirname(BASE_DIR) +DOMAINS_FILE = os.path.join(PARENT_DIR, "streaming_domains_whitelist.txt") + +SOURCES = { + "Netflix": "https://raw.githubusercontent.com/wickedyoda/public-setupfiles/main/domains/streaming_domains_whitelist.txt", + "Crunchyroll": "https://gist.githubusercontent.com/NicmeisteR/cdc4867cf256c568b6a7f1844ce229f2/raw", + "Peacock": "https://raw.githubusercontent.com/lit-bg/Peacock/refs/heads/main/filterlist.txt", + "PiHole Streaming List": "https://raw.githubusercontent.com/ozankiratli/801ba17705e7f2a904d2e443af5a64f8/raw/436fa9f0c151afc15a601edf77c796b2a6de9be4/PiHoleStreamingLists.md", +} + +def fetch_text(url): + try: + req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) + with urllib.request.urlopen(req, timeout=30) as resp: + return resp.read().decode("utf-8", errors="replace") + except (urllib.error.URLError, Exception) as e: + print(f"WARN: fetch failed for {url}: {e}") + return "" + +def backup_file(path): + os.makedirs(BACKUP_DIR, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + backup_path = os.path.join(BACKUP_DIR, f"streaming_domains_whitelist.txt.{timestamp}") + shutil.copy2(path, backup_path) + print(f"Backup saved to {backup_path}") + +def main(): + domains = set() + comments = [] + + for name, url in SOURCES.items(): + print(f"Fetching {name}...") + text = fetch_text(url) + if not text: + continue + + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + # Extract domain-like tokens + parts = line.replace("^", "").replace("|", "").replace("/", "").split() + for part in parts: + if "." in part and "/" not in part and " " not in part: + clean = part.lstrip("*") + if clean and not clean[0].isdigit: + domains.add(clean) + + # Hardcoded core domains to always keep + core_domains = [ + "netflix.com", "nflxext.com", "nflximg.com", "nflxso.net", "nflxvideo.net", + "netflix.net", "hbomax.com", "hbo.com", "warnermediacdn.com", "wbdstreaming.com", + "max.com", "wmcdp.io", "wmcdp.com", "hulu.com", "hulustream.com", "huluim.com", + "huluad.com", "vortex.hulu.com", "doppler.hulu.com", "conviva.com", + "disneyplus.com", "disney.com", "bamgrid.com", "disneystreaming.com", "dssott.com", + "paramountplus.com", "cbsivideo.com", "cbsaavideo.com", "cbs.com", "viacomcbs.com", + "peacocktv.com", "nbc.com", "nbcuni.com", "nbcucdn.com", + "tubitv.com", "fox.com", "foxsports.com", "fubo.tv", "pluto.tv", + "roku.com", "rokutime.com", "therokuchannel.roku.com", + "crunchyroll.com", "vrv.co", "funimation.com", + "aiv-cdn.net", "primevideo.com", "amazon.com", "amazonaws.com", + "cloudfront.net", "s3.amazonaws.com", "atv-ext.amazon.com", + "imdb.com", "imdbtv.com", "freevee.com", + "apple.com", "apple-dns.net", "mzstatic.com", + "apple-tv-plus.com", "tv.apple.com", + "vudu.com", "moviefone.com", "mubi.com", "shudder.com", + "britbox.com", "acorn.tv", "sling.com", "philo.com", "xumo.com", + "discoveryplus.com", "dplay.com", + "espn.com", "espncdn.com", "espn.net", + "mlb.com", "mlbstatic.com", "nba.com", "nfl.com", "nhl.com", "nhlstatic.com", + "spotify.com", "scdn.co", "pandora.com", "soundcloud.com", + "deezer.com", "tidal.com", + ] + domains.update(core_domains) + + # Always back up before writing + if os.path.exists(DOMAINS_FILE): + backup_file(DOMAINS_FILE) + + sorted_domains = sorted(d for d in domains if not d.startswith("1") and not d.startswith("2") and not d.startswith("3") and not d.startswith("4") and not d.startswith("5") and not d.startswith("6") and not d.startswith("7") and not d.startswith("8") and not d.startswith("9")) + ip_cidrs = sorted(d for d in domains if any(c.isdigit() for c in d)) + + with open(DOMAINS_FILE, "w") as f: + for domain in sorted_domains: + f.write(domain + "\n") + f.write("\n# IP Ranges / CIDRs\n") + for ip in ip_cidrs: + f.write(ip + "\n") + + print(f"Domains updated: {len(sorted_domains)} domains + {len(ip_cidrs)} IPs") + +if __name__ == "__main__": + main()