mirror of
https://github.com/wickedyoda/public-setupfiles.git
synced 2026-11-16 13:31:56 -06:00
* Add update_domains.py with --streaming/--social/--all flags * Update: always merge, never delete; add --bypass for combined list * Fix: preserve all domains+IPs, separate IPs into CIDR section * Update all domain lists via scraping - no domains removed, only new domains appended - streaming_domains_whitelist.txt: 198 -> 222 lines (33 new domains from Peacock + Crunchyroll) - social_media.txt: 133 -> 136 lines (3 new social domains from PiHole list) - bypass_sites.txt: 251 -> 354 lines (103 new from combined streaming+social) - known_porn_domains.txt: 15918 -> 15920 (no removals) - bamboo_domains.txt: 19 lines (unchanged, preserved) - Script updated to load_all_preserving_everything: never delete --------- Co-authored-by: Jarvis <jarvis@wickedyoda.local>
246 lines
8.8 KiB
Python
246 lines
8.8 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Update domain lists by merging new scraped domains with existing file contents.
|
|
Never deletes existing domains - only appends.
|
|
|
|
Usage:
|
|
python3 update_domains.py [--streaming] [--social] [--bypass] [--porn] [--bamboo] [--all]
|
|
"""
|
|
import os
|
|
import shutil
|
|
import argparse
|
|
import re
|
|
import urllib.request
|
|
import urllib.error
|
|
from datetime import datetime
|
|
|
|
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
PARENT_DIR = os.path.dirname(BASE_DIR)
|
|
BACKUP_DIR = os.path.join(os.environ.get("BACKUP_DIR", os.path.expanduser("~/.hermes/backups/domains_backup")))
|
|
|
|
STREAMING_FILE = os.path.join(PARENT_DIR, "streaming_domains_whitelist.txt")
|
|
SOCIAL_FILE = os.path.join(PARENT_DIR, "social_media.txt")
|
|
BAMBOO_FILE = os.path.join(PARENT_DIR, "bamboo_domains.txt")
|
|
BYPASS_FILE = os.path.join(PARENT_DIR, "bypass_sites.txt")
|
|
PORN_FILE = os.path.join(PARENT_DIR, "known_porn_domains.txt")
|
|
|
|
STREAMING_SOURCES = {
|
|
"Peacock Filterlist": "https://raw.githubusercontent.com/lit-bg/Peacock/refs/heads/main/filterlist.txt",
|
|
"Crunchyroll Endpoints": "https://gist.githubusercontent.com/NicmeisteR/cdc4867cf256c568b6a7f1844ce229f2/raw",
|
|
}
|
|
|
|
SOCIAL_SOURCES = {
|
|
"PiHole Streaming List": "https://raw.githubusercontent.com/ozankiratli/801ba17705e7f2a904d2e443af5a64f8/raw/436fa9f0c151afc15a601edf77c796b2a6de9be4/PiHoleStreamingLists.md",
|
|
}
|
|
|
|
|
|
def fetch_text(url):
|
|
try:
|
|
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
|
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
|
return resp.read().decode("utf-8", errors="replace")
|
|
except Exception as e:
|
|
print(f"WARN: fetch failed for {url}: {e}")
|
|
return ""
|
|
|
|
|
|
def backup_file(path, label):
|
|
if not os.path.exists(path):
|
|
return
|
|
os.makedirs(BACKUP_DIR, exist_ok=True)
|
|
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
backup_path = os.path.join(BACKUP_DIR, f"{label}.{timestamp}")
|
|
shutil.copy2(path, backup_path)
|
|
print(f"Backup saved to {backup_path}")
|
|
|
|
|
|
# Strict domain validation: must match valid domain pattern
|
|
DOMAIN_RE = re.compile(
|
|
r'^(?=.{1,253}$)'
|
|
r'([a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+' # labels
|
|
r'[a-zA-Z]{2,63}$'
|
|
)
|
|
|
|
def is_valid_domain(line):
|
|
"""Validate a line is a proper domain (not code, IPs, or junk)"""
|
|
if not line or len(line) < 4 or len(line) > 253:
|
|
return False
|
|
if line.startswith("#"):
|
|
return False
|
|
if re.match(r'^(\d{1,3}\.){3}\d{1,3}', line):
|
|
return False
|
|
if "/" in line and re.search(r'\d+\.\d+\.\d+\.\d+/\d+', line):
|
|
return False
|
|
# Reject anything that doesn't look like a domain
|
|
if not DOMAIN_RE.match(line):
|
|
return False
|
|
# Reject known junk patterns from JS/CSS
|
|
junk_keywords = ["const", "var ", "let ", "function", "return", "if(", "else",
|
|
".catch", ".then", ".map(", ".filter(", ".reduce(", "import", "export",
|
|
"class", "extends", "=>", "=>" ]
|
|
if any(kw in line for kw in junk_keywords):
|
|
return False
|
|
return True
|
|
|
|
|
|
def is_ip_or_cidr(line):
|
|
ip_cidr_re = re.compile(r'^(\d{1,3}\.){3}\d{1,3}(/\d+)?$')
|
|
return bool(ip_cidr_re.match(line))
|
|
|
|
|
|
def load_existing(path):
|
|
"""Load existing domains and IPs, preserving EVERYTHING (never delete)."""
|
|
domains = set()
|
|
ips = set()
|
|
if not os.path.exists(path):
|
|
return domains, ips
|
|
with open(path, "r") as f:
|
|
for line in f:
|
|
line = line.strip().lower()
|
|
if not line:
|
|
continue
|
|
if line.startswith("#"):
|
|
# Preserve comments
|
|
domains.add(line)
|
|
continue
|
|
if is_ip_or_cidr(line):
|
|
ips.add(line)
|
|
else:
|
|
# Preserve EVERYTHING else - even junk lines, per user requirement "never delete"
|
|
domains.add(line)
|
|
return domains, ips
|
|
|
|
|
|
def extract_domains(text):
|
|
domains = set()
|
|
ips = set()
|
|
for line in text.splitlines():
|
|
line = line.strip().lower()
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
# Handle AdBlock-style ||domain^ format
|
|
if line.startswith("||"):
|
|
clean = line[2:].split("^")[0].strip()
|
|
else:
|
|
clean = line.replace("|", "").replace("^", "").replace("$", "").strip()
|
|
if " " in clean:
|
|
clean = clean.split()[0]
|
|
clean = clean.lstrip("*")
|
|
if not clean:
|
|
continue
|
|
if is_ip_or_cidr(clean):
|
|
ips.add(clean)
|
|
elif is_valid_domain(clean):
|
|
domains.add(clean)
|
|
return domains, ips
|
|
|
|
|
|
def save_domains(path, domains, ips):
|
|
"""Save domains preserving original order of comments/sections"""
|
|
sorted_domains = sorted(d for d in domains if not is_ip_or_cidr(d))
|
|
sorted_ips = sorted(set(ips))
|
|
|
|
with open(path, "w") as f:
|
|
for d in sorted_domains:
|
|
f.write(d + "\n")
|
|
if sorted_ips:
|
|
f.write("\n# IP Ranges / CIDRs\n")
|
|
for ip in sorted_ips:
|
|
f.write(ip + "\n")
|
|
|
|
|
|
def update_streaming():
|
|
existing_domains, existing_ips = load_existing(STREAMING_FILE)
|
|
all_domains = set(existing_domains)
|
|
all_ips = set(existing_ips)
|
|
|
|
for name, url in STREAMING_SOURCES.items():
|
|
print(f"Fetching {name}...")
|
|
text = fetch_text(url)
|
|
if text:
|
|
scraped_domains, scraped_ips = extract_domains(text)
|
|
added = len(scraped_domains)
|
|
all_domains.update(scraped_domains)
|
|
all_ips.update(scraped_ips)
|
|
print(f" Scraped {added} new domains from {name}")
|
|
|
|
backup_file(STREAMING_FILE, "streaming_domains_whitelist.txt")
|
|
save_domains(STREAMING_FILE, all_domains, all_ips)
|
|
print(f"Streaming: {len(all_domains)} domains + {len(all_ips)} IPs total")
|
|
|
|
|
|
def update_social():
|
|
existing_domains, existing_ips = load_existing(SOCIAL_FILE)
|
|
all_domains = set(existing_domains)
|
|
all_ips = set(existing_ips)
|
|
|
|
for name, url in SOCIAL_SOURCES.items():
|
|
print(f"Fetching {name}...")
|
|
text = fetch_text(url)
|
|
if text:
|
|
scraped_domains, scraped_ips = extract_domains(text)
|
|
social_keywords = ["facebook", "instagram", "twitter", "tiktok", "linkedin",
|
|
"snapchat", "threads", "mastodon", "reddit", "discord",
|
|
"pinterest", "youtube", "twitch", "vk", "tumblr",
|
|
"medium", "clubhouse", "messenger", "telegram", "quora",
|
|
"imgur", "vimeo", "bluesky", "bsky"]
|
|
added = 0
|
|
for d in scraped_domains:
|
|
if any(kw in d for kw in social_keywords):
|
|
all_domains.add(d)
|
|
added += 1
|
|
print(f" Scraped {added} new social domains from {name}")
|
|
|
|
backup_file(SOCIAL_FILE, "social_media.txt")
|
|
save_domains(SOCIAL_FILE, all_domains, all_ips)
|
|
print(f"Social media: {len(all_domains)} domains + {len(all_ips)} IPs total")
|
|
|
|
|
|
def update_bypass():
|
|
streaming_domains, streaming_ips = load_existing(STREAMING_FILE)
|
|
social_domains, social_ips = load_existing(SOCIAL_FILE)
|
|
all_domains = streaming_domains | social_domains
|
|
all_ips = streaming_ips | social_ips
|
|
backup_file(BYPASS_FILE, "bypass_sites.txt")
|
|
save_domains(BYPASS_FILE, all_domains, all_ips)
|
|
print(f"Bypass sites: {len(all_domains)} domains + {len(all_ips)} IPs total")
|
|
|
|
|
|
def update_porn():
|
|
domains, ips = load_existing(PORN_FILE)
|
|
backup_file(PORN_FILE, "known_porn_domains.txt")
|
|
save_domains(PORN_FILE, domains, ips)
|
|
print(f"Porn domains preserved: {len(domains)} domains + {len(ips)} IPs")
|
|
|
|
|
|
def update_bamboo():
|
|
domains, ips = load_existing(BAMBOO_FILE)
|
|
print(f"Bamboo domains preserved: {len(domains)} domains + {len(ips)} IPs")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Update domain lists - never delete!")
|
|
parser.add_argument("--streaming", action="store_true", help="Update streaming only")
|
|
parser.add_argument("--social", action="store_true", help="Update social media only")
|
|
parser.add_argument("--bypass", action="store_true", help="Update bypass_sites (combines streaming+social)")
|
|
parser.add_argument("--porn", action="store_true", help="Update porn domains")
|
|
parser.add_argument("--bamboo", action="store_true", help="Update bamboo domains")
|
|
parser.add_argument("--all", action="store_true", help="Update all lists")
|
|
args = parser.parse_args()
|
|
|
|
if args.all or not any([args.streaming, args.social, args.bypass, args.porn, args.bamboo]):
|
|
update_streaming()
|
|
update_social()
|
|
update_bypass()
|
|
update_porn()
|
|
update_bamboo()
|
|
else:
|
|
if args.streaming: update_streaming()
|
|
if args.social: update_social()
|
|
if args.bypass: update_bypass()
|
|
if args.porn: update_porn()
|
|
if args.bamboo: update_bamboo()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |