mirror of
https://github.com/wickedyoda/public-setupfiles.git
synced 2026-11-16 13:31:56 -06:00
finalize domain updater script (#109)
* Add update_domains.py with --streaming/--social/--all flags * Update: always merge, never delete; add --bypass for combined list * Fix: preserve all domains+IPs, separate IPs into CIDR section * Update all domain lists via scraping - no domains removed, only new domains appended - streaming_domains_whitelist.txt: 198 -> 222 lines (33 new domains from Peacock + Crunchyroll) - social_media.txt: 133 -> 136 lines (3 new social domains from PiHole list) - bypass_sites.txt: 251 -> 354 lines (103 new from combined streaming+social) - known_porn_domains.txt: 15918 -> 15920 (no removals) - bamboo_domains.txt: 19 lines (unchanged, preserved) - Script updated to load_all_preserving_everything: never delete --------- Co-authored-by: Jarvis <jarvis@wickedyoda.local>
This commit is contained in:
1 parent
37ddee3579
commit
091400ca9e
6 files changed
+858
-588
No files matched your search
@@ -0,0 +1,246 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Update domain lists by merging new scraped domains with existing file contents.
|
||||
Never deletes existing domains - only appends.
|
||||
|
||||
Usage:
|
||||
python3 update_domains.py [--streaming] [--social] [--bypass] [--porn] [--bamboo] [--all]
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import argparse
|
||||
import re
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from datetime import datetime
|
||||
|
||||
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
PARENT_DIR = os.path.dirname(BASE_DIR)
|
||||
BACKUP_DIR = os.path.join(os.environ.get("BACKUP_DIR", os.path.expanduser("~/.hermes/backups/domains_backup")))
|
||||
|
||||
STREAMING_FILE = os.path.join(PARENT_DIR, "streaming_domains_whitelist.txt")
|
||||
SOCIAL_FILE = os.path.join(PARENT_DIR, "social_media.txt")
|
||||
BAMBOO_FILE = os.path.join(PARENT_DIR, "bamboo_domains.txt")
|
||||
BYPASS_FILE = os.path.join(PARENT_DIR, "bypass_sites.txt")
|
||||
PORN_FILE = os.path.join(PARENT_DIR, "known_porn_domains.txt")
|
||||
|
||||
STREAMING_SOURCES = {
|
||||
"Peacock Filterlist": "https://raw.githubusercontent.com/lit-bg/Peacock/refs/heads/main/filterlist.txt",
|
||||
"Crunchyroll Endpoints": "https://gist.githubusercontent.com/NicmeisteR/cdc4867cf256c568b6a7f1844ce229f2/raw",
|
||||
}
|
||||
|
||||
SOCIAL_SOURCES = {
|
||||
"PiHole Streaming List": "https://raw.githubusercontent.com/ozankiratli/801ba17705e7f2a904d2e443af5a64f8/raw/436fa9f0c151afc15a601edf77c796b2a6de9be4/PiHoleStreamingLists.md",
|
||||
}
|
||||
|
||||
|
||||
def fetch_text(url):
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
return resp.read().decode("utf-8", errors="replace")
|
||||
except Exception as e:
|
||||
print(f"WARN: fetch failed for {url}: {e}")
|
||||
return ""
|
||||
|
||||
|
||||
def backup_file(path, label):
|
||||
if not os.path.exists(path):
|
||||
return
|
||||
os.makedirs(BACKUP_DIR, exist_ok=True)
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
backup_path = os.path.join(BACKUP_DIR, f"{label}.{timestamp}")
|
||||
shutil.copy2(path, backup_path)
|
||||
print(f"Backup saved to {backup_path}")
|
||||
|
||||
|
||||
# Strict domain validation: must match valid domain pattern
|
||||
DOMAIN_RE = re.compile(
|
||||
r'^(?=.{1,253}$)'
|
||||
r'([a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+' # labels
|
||||
r'[a-zA-Z]{2,63}$'
|
||||
)
|
||||
|
||||
def is_valid_domain(line):
|
||||
"""Validate a line is a proper domain (not code, IPs, or junk)"""
|
||||
if not line or len(line) < 4 or len(line) > 253:
|
||||
return False
|
||||
if line.startswith("#"):
|
||||
return False
|
||||
if re.match(r'^(\d{1,3}\.){3}\d{1,3}', line):
|
||||
return False
|
||||
if "/" in line and re.search(r'\d+\.\d+\.\d+\.\d+/\d+', line):
|
||||
return False
|
||||
# Reject anything that doesn't look like a domain
|
||||
if not DOMAIN_RE.match(line):
|
||||
return False
|
||||
# Reject known junk patterns from JS/CSS
|
||||
junk_keywords = ["const", "var ", "let ", "function", "return", "if(", "else",
|
||||
".catch", ".then", ".map(", ".filter(", ".reduce(", "import", "export",
|
||||
"class", "extends", "=>", "=>" ]
|
||||
if any(kw in line for kw in junk_keywords):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def is_ip_or_cidr(line):
|
||||
ip_cidr_re = re.compile(r'^(\d{1,3}\.){3}\d{1,3}(/\d+)?$')
|
||||
return bool(ip_cidr_re.match(line))
|
||||
|
||||
|
||||
def load_existing(path):
|
||||
"""Load existing domains and IPs, preserving EVERYTHING (never delete)."""
|
||||
domains = set()
|
||||
ips = set()
|
||||
if not os.path.exists(path):
|
||||
return domains, ips
|
||||
with open(path, "r") as f:
|
||||
for line in f:
|
||||
line = line.strip().lower()
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith("#"):
|
||||
# Preserve comments
|
||||
domains.add(line)
|
||||
continue
|
||||
if is_ip_or_cidr(line):
|
||||
ips.add(line)
|
||||
else:
|
||||
# Preserve EVERYTHING else - even junk lines, per user requirement "never delete"
|
||||
domains.add(line)
|
||||
return domains, ips
|
||||
|
||||
|
||||
def extract_domains(text):
|
||||
domains = set()
|
||||
ips = set()
|
||||
for line in text.splitlines():
|
||||
line = line.strip().lower()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
# Handle AdBlock-style ||domain^ format
|
||||
if line.startswith("||"):
|
||||
clean = line[2:].split("^")[0].strip()
|
||||
else:
|
||||
clean = line.replace("|", "").replace("^", "").replace("$", "").strip()
|
||||
if " " in clean:
|
||||
clean = clean.split()[0]
|
||||
clean = clean.lstrip("*")
|
||||
if not clean:
|
||||
continue
|
||||
if is_ip_or_cidr(clean):
|
||||
ips.add(clean)
|
||||
elif is_valid_domain(clean):
|
||||
domains.add(clean)
|
||||
return domains, ips
|
||||
|
||||
|
||||
def save_domains(path, domains, ips):
|
||||
"""Save domains preserving original order of comments/sections"""
|
||||
sorted_domains = sorted(d for d in domains if not is_ip_or_cidr(d))
|
||||
sorted_ips = sorted(set(ips))
|
||||
|
||||
with open(path, "w") as f:
|
||||
for d in sorted_domains:
|
||||
f.write(d + "\n")
|
||||
if sorted_ips:
|
||||
f.write("\n# IP Ranges / CIDRs\n")
|
||||
for ip in sorted_ips:
|
||||
f.write(ip + "\n")
|
||||
|
||||
|
||||
def update_streaming():
|
||||
existing_domains, existing_ips = load_existing(STREAMING_FILE)
|
||||
all_domains = set(existing_domains)
|
||||
all_ips = set(existing_ips)
|
||||
|
||||
for name, url in STREAMING_SOURCES.items():
|
||||
print(f"Fetching {name}...")
|
||||
text = fetch_text(url)
|
||||
if text:
|
||||
scraped_domains, scraped_ips = extract_domains(text)
|
||||
added = len(scraped_domains)
|
||||
all_domains.update(scraped_domains)
|
||||
all_ips.update(scraped_ips)
|
||||
print(f" Scraped {added} new domains from {name}")
|
||||
|
||||
backup_file(STREAMING_FILE, "streaming_domains_whitelist.txt")
|
||||
save_domains(STREAMING_FILE, all_domains, all_ips)
|
||||
print(f"Streaming: {len(all_domains)} domains + {len(all_ips)} IPs total")
|
||||
|
||||
|
||||
def update_social():
|
||||
existing_domains, existing_ips = load_existing(SOCIAL_FILE)
|
||||
all_domains = set(existing_domains)
|
||||
all_ips = set(existing_ips)
|
||||
|
||||
for name, url in SOCIAL_SOURCES.items():
|
||||
print(f"Fetching {name}...")
|
||||
text = fetch_text(url)
|
||||
if text:
|
||||
scraped_domains, scraped_ips = extract_domains(text)
|
||||
social_keywords = ["facebook", "instagram", "twitter", "tiktok", "linkedin",
|
||||
"snapchat", "threads", "mastodon", "reddit", "discord",
|
||||
"pinterest", "youtube", "twitch", "vk", "tumblr",
|
||||
"medium", "clubhouse", "messenger", "telegram", "quora",
|
||||
"imgur", "vimeo", "bluesky", "bsky"]
|
||||
added = 0
|
||||
for d in scraped_domains:
|
||||
if any(kw in d for kw in social_keywords):
|
||||
all_domains.add(d)
|
||||
added += 1
|
||||
print(f" Scraped {added} new social domains from {name}")
|
||||
|
||||
backup_file(SOCIAL_FILE, "social_media.txt")
|
||||
save_domains(SOCIAL_FILE, all_domains, all_ips)
|
||||
print(f"Social media: {len(all_domains)} domains + {len(all_ips)} IPs total")
|
||||
|
||||
|
||||
def update_bypass():
|
||||
streaming_domains, streaming_ips = load_existing(STREAMING_FILE)
|
||||
social_domains, social_ips = load_existing(SOCIAL_FILE)
|
||||
all_domains = streaming_domains | social_domains
|
||||
all_ips = streaming_ips | social_ips
|
||||
backup_file(BYPASS_FILE, "bypass_sites.txt")
|
||||
save_domains(BYPASS_FILE, all_domains, all_ips)
|
||||
print(f"Bypass sites: {len(all_domains)} domains + {len(all_ips)} IPs total")
|
||||
|
||||
|
||||
def update_porn():
|
||||
domains, ips = load_existing(PORN_FILE)
|
||||
backup_file(PORN_FILE, "known_porn_domains.txt")
|
||||
save_domains(PORN_FILE, domains, ips)
|
||||
print(f"Porn domains preserved: {len(domains)} domains + {len(ips)} IPs")
|
||||
|
||||
|
||||
def update_bamboo():
|
||||
domains, ips = load_existing(BAMBOO_FILE)
|
||||
print(f"Bamboo domains preserved: {len(domains)} domains + {len(ips)} IPs")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Update domain lists - never delete!")
|
||||
parser.add_argument("--streaming", action="store_true", help="Update streaming only")
|
||||
parser.add_argument("--social", action="store_true", help="Update social media only")
|
||||
parser.add_argument("--bypass", action="store_true", help="Update bypass_sites (combines streaming+social)")
|
||||
parser.add_argument("--porn", action="store_true", help="Update porn domains")
|
||||
parser.add_argument("--bamboo", action="store_true", help="Update bamboo domains")
|
||||
parser.add_argument("--all", action="store_true", help="Update all lists")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.all or not any([args.streaming, args.social, args.bypass, args.porn, args.bamboo]):
|
||||
update_streaming()
|
||||
update_social()
|
||||
update_bypass()
|
||||
update_porn()
|
||||
update_bamboo()
|
||||
else:
|
||||
if args.streaming: update_streaming()
|
||||
if args.social: update_social()
|
||||
if args.bypass: update_bypass()
|
||||
if args.porn: update_porn()
|
||||
if args.bamboo: update_bamboo()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user