Add domains/updater/update_streaming_domains.py script (#108)
* Update streaming_domains_whitelist.txt with additional domains from research * Add domains/updater/update_streaming_domains.py script --------- Co-authored-by: Jarvis <jarvis@wickedyoda.local>
This commit is contained in:
1 parent
956350c64e
commit
37ddee3579
1 file changed
+107
Executable
+107
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Update streaming_domains_whitelist.txt with domains scraped from known sources.
|
||||
Always backs up the previous file before overwriting.
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from datetime import datetime
|
||||
|
||||
DOMAINS_FILE = os.path.join(os.path.dirname(__file__), "..", "streaming_domains_whitelist.txt")
|
||||
BACKUP_DIR = os.path.join(os.environ.get("HOME", "/tmp"), ".hermes", "backups", "domains_backup")
|
||||
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
PARENT_DIR = os.path.dirname(BASE_DIR)
|
||||
DOMAINS_FILE = os.path.join(PARENT_DIR, "streaming_domains_whitelist.txt")
|
||||
|
||||
SOURCES = {
|
||||
"Netflix": "https://raw.githubusercontent.com/wickedyoda/public-setupfiles/main/domains/streaming_domains_whitelist.txt",
|
||||
"Crunchyroll": "https://gist.githubusercontent.com/NicmeisteR/cdc4867cf256c568b6a7f1844ce229f2/raw",
|
||||
"Peacock": "https://raw.githubusercontent.com/lit-bg/Peacock/refs/heads/main/filterlist.txt",
|
||||
"PiHole Streaming List": "https://raw.githubusercontent.com/ozankiratli/801ba17705e7f2a904d2e443af5a64f8/raw/436fa9f0c151afc15a601edf77c796b2a6de9be4/PiHoleStreamingLists.md",
|
||||
}
|
||||
|
||||
def fetch_text(url):
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
return resp.read().decode("utf-8", errors="replace")
|
||||
except (urllib.error.URLError, Exception) as e:
|
||||
print(f"WARN: fetch failed for {url}: {e}")
|
||||
return ""
|
||||
|
||||
def backup_file(path):
|
||||
os.makedirs(BACKUP_DIR, exist_ok=True)
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
backup_path = os.path.join(BACKUP_DIR, f"streaming_domains_whitelist.txt.{timestamp}")
|
||||
shutil.copy2(path, backup_path)
|
||||
print(f"Backup saved to {backup_path}")
|
||||
|
||||
def main():
|
||||
domains = set()
|
||||
comments = []
|
||||
|
||||
for name, url in SOURCES.items():
|
||||
print(f"Fetching {name}...")
|
||||
text = fetch_text(url)
|
||||
if not text:
|
||||
continue
|
||||
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
# Extract domain-like tokens
|
||||
parts = line.replace("^", "").replace("|", "").replace("/", "").split()
|
||||
for part in parts:
|
||||
if "." in part and "/" not in part and " " not in part:
|
||||
clean = part.lstrip("*")
|
||||
if clean and not clean[0].isdigit:
|
||||
domains.add(clean)
|
||||
|
||||
# Hardcoded core domains to always keep
|
||||
core_domains = [
|
||||
"netflix.com", "nflxext.com", "nflximg.com", "nflxso.net", "nflxvideo.net",
|
||||
"netflix.net", "hbomax.com", "hbo.com", "warnermediacdn.com", "wbdstreaming.com",
|
||||
"max.com", "wmcdp.io", "wmcdp.com", "hulu.com", "hulustream.com", "huluim.com",
|
||||
"huluad.com", "vortex.hulu.com", "doppler.hulu.com", "conviva.com",
|
||||
"disneyplus.com", "disney.com", "bamgrid.com", "disneystreaming.com", "dssott.com",
|
||||
"paramountplus.com", "cbsivideo.com", "cbsaavideo.com", "cbs.com", "viacomcbs.com",
|
||||
"peacocktv.com", "nbc.com", "nbcuni.com", "nbcucdn.com",
|
||||
"tubitv.com", "fox.com", "foxsports.com", "fubo.tv", "pluto.tv",
|
||||
"roku.com", "rokutime.com", "therokuchannel.roku.com",
|
||||
"crunchyroll.com", "vrv.co", "funimation.com",
|
||||
"aiv-cdn.net", "primevideo.com", "amazon.com", "amazonaws.com",
|
||||
"cloudfront.net", "s3.amazonaws.com", "atv-ext.amazon.com",
|
||||
"imdb.com", "imdbtv.com", "freevee.com",
|
||||
"apple.com", "apple-dns.net", "mzstatic.com",
|
||||
"apple-tv-plus.com", "tv.apple.com",
|
||||
"vudu.com", "moviefone.com", "mubi.com", "shudder.com",
|
||||
"britbox.com", "acorn.tv", "sling.com", "philo.com", "xumo.com",
|
||||
"discoveryplus.com", "dplay.com",
|
||||
"espn.com", "espncdn.com", "espn.net",
|
||||
"mlb.com", "mlbstatic.com", "nba.com", "nfl.com", "nhl.com", "nhlstatic.com",
|
||||
"spotify.com", "scdn.co", "pandora.com", "soundcloud.com",
|
||||
"deezer.com", "tidal.com",
|
||||
]
|
||||
domains.update(core_domains)
|
||||
|
||||
# Always back up before writing
|
||||
if os.path.exists(DOMAINS_FILE):
|
||||
backup_file(DOMAINS_FILE)
|
||||
|
||||
sorted_domains = sorted(d for d in domains if not d.startswith("1") and not d.startswith("2") and not d.startswith("3") and not d.startswith("4") and not d.startswith("5") and not d.startswith("6") and not d.startswith("7") and not d.startswith("8") and not d.startswith("9"))
|
||||
ip_cidrs = sorted(d for d in domains if any(c.isdigit() for c in d))
|
||||
|
||||
with open(DOMAINS_FILE, "w") as f:
|
||||
for domain in sorted_domains:
|
||||
f.write(domain + "\n")
|
||||
f.write("\n# IP Ranges / CIDRs\n")
|
||||
for ip in ip_cidrs:
|
||||
f.write(ip + "\n")
|
||||
|
||||
print(f"Domains updated: {len(sorted_domains)} domains + {len(ip_cidrs)} IPs")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user