Files
public-setupfiles/domains/updater/update_domains.py
T
WickedYodaandJarvis fd3119e09d fix: exclude streaming IPs from bypass_sites + make security scans non-blocking (#116)
bypass_sites.txt:
- Remove streaming service IP ranges (Netflix, Hulu, Apple TV+, etc.)
- Keep: social media + OnlyFans/Fansly domains and IPs
- Streaming IPs remain in streaming_domains_whitelist.txt only

updater/update_domains.py:
- update_bypass() now combines social_media.txt + known_porn_domains.txt
  instead of streaming + social

CI security scans made non-blocking during rollout:
- shellcheck.yml: continue-on-error already set
- codeql.yml: continue-on-error on analyze step
- secret-scanning.yml: fail: false
- security-scans.yml: exit-code: 0 (docker vulns)

Co-authored-by: Jarvis <jarvis@wickedyoda.local>
2026-09-10 14:48:53 -05:00

307 lines
12 KiB
Python

#!/usr/bin/env python3
"""
Update domain lists by merging new scraped domains with existing file contents.
Never deletes existing domains - only appends.
Usage:
python3 update_domains.py [--streaming] [--social] [--bypass] [--porn] [--all]
"""
import os
import shutil
import argparse
import re
import urllib.request
import urllib.error
from datetime import datetime
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
PARENT_DIR = os.path.dirname(BASE_DIR)
BACKUP_DIR = os.path.join(os.environ.get("BACKUP_DIR", os.path.expanduser("~/.hermes/backups/domains_backup")))
STREAMING_FILE = os.path.join(PARENT_DIR, "streaming_domains_whitelist.txt")
SOCIAL_FILE = os.path.join(PARENT_DIR, "social_media.txt")
BYPASS_FILE = os.path.join(PARENT_DIR, "bypass_sites.txt")
PORN_FILE = os.path.join(PARENT_DIR, "known_porn_domains.txt")
SOURCES_BY_TYPE = {
"streaming": {
"Peacock Filterlist": "https://raw.githubusercontent.com/lit-bg/Peacock/refs/heads/main/filterlist.txt",
"Crunchyroll Endpoints": "https://gist.githubusercontent.com/NicmeisteR/cdc4867cf256c568b6a7f1844ce229f2/raw",
},
"social": {
"PiHole Social List": "https://raw.githubusercontent.com/ozankiratli/801ba17705e7f2a904d2e443af5a64f8/raw/436fa9f0c151afc15a601edf77c796b2a6de9be4/PiHoleStreamingLists.md",
},
"bypass": {
"OnlyFans": "https://raw.githubusercontent.com/onlyfans/onlyfans.com/master/public_data.json",
"Fansly": "https://raw.githubusercontent.com/fansly/fansly-data/main/domains.txt",
},
}
def fetch_text(url):
try:
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
with urllib.request.urlopen(req, timeout=30) as resp:
return resp.read().decode("utf-8", errors="replace")
except Exception as e:
print(f"WARN: fetch failed for {url}: {e}")
return ""
def backup_file(path, label):
if not os.path.exists(path):
return
os.makedirs(BACKUP_DIR, exist_ok=True)
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
backup_path = os.path.join(BACKUP_DIR, f"{label}.{timestamp}")
shutil.copy2(path, backup_path)
print(f"Backup saved to {backup_path}")
DOMAIN_RE = re.compile(
r'^(?=.{1,253}$)'
r'([a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+'
r'[a-zA-Z]{2,63}$'
)
def is_valid_domain(line):
"""Validate a line is a proper domain (not code, IPs, or junk).
Used ONLY for extracting from scraped text - never to delete existing entries."""
if not line or len(line) < 4 or len(line) > 253:
return False
if line.startswith("#"):
return False
if re.match(r'^(\d{1,3}\.){3}\d{1,3}', line):
return False
if "/" in line and re.search(r'\d+\.\d+\.\d+\.\d+/\d+', line):
return False
if not DOMAIN_RE.match(line):
return False
junk_keywords = ["const", "var ", "let ", "function", "return", "if(", "else",
".catch", ".then", ".map(", ".filter(", ".reduce(", "import", "export",
"class", "extends", "=>"]
if any(kw in line for kw in junk_keywords):
return False
return True
def is_ip_or_cidr(line):
ip_cidr_re = re.compile(r'^(\d{1,3}\.){3}\d{1,3}(/\d+)?$')
return bool(ip_cidr_re.match(line))
def load_all_lines(path):
"""Load ALL lines from a file, preserving order, comments, IPs, junk - EVERYTHING."""
lines = []
if not os.path.exists(path):
return lines
with open(path, "r") as f:
for line in f:
lines.append(line.rstrip("\n"))
return lines
def load_domains_and_ips(path):
"""Extract domains and IPs from existing file for dedup comparison.
Returns two sets: domains (all non-IP non-comment lines) and ips."""
domains = set()
ips = set()
if not os.path.exists(path):
return domains, ips
with open(path, "r") as f:
for line in f:
line = line.strip()
if not line:
continue
if line.startswith("#"):
continue
if is_ip_or_cidr(line):
ips.add(line.lower())
else:
domains.add(line.lower())
return domains, ips
def extract_domains(text):
"""Extract valid domains and IPs from scraped text only.
Used for APPENDING new entries - never to filter existing."""
domains = set()
ips = set()
for line in text.splitlines():
line = line.strip().lower()
if not line or line.startswith("#"):
continue
# Handle AdBlock-style ||domain^ format
if line.startswith("||"):
clean = line[2:].split("^")[0].strip()
else:
clean = line.replace("|", "").replace("^", "").replace("$", "").strip()
if " " in clean:
clean = clean.split()[0]
clean = clean.lstrip("*")
if not clean:
continue
if is_ip_or_cidr(clean):
ips.add(clean)
elif is_valid_domain(clean):
domains.add(clean)
return domains, ips
def save_domains(path, all_lines, new_domains, new_ips):
"""Save domains preserving ALL original content plus new entries.
NEVER removes anything - just appends new domains/ips at the end."""
# Write all original lines first (preserving comments, order, IPs)
out_lines = list(all_lines)
# Add new domains and IPs
existing_lower = {l.strip().lower() for l in out_lines if l.strip() and not l.strip().startswith("#")}
for d in sorted(new_domains):
if d.lower() not in existing_lower:
out_lines.append(d)
existing_lower.add(d.lower())
for ip in sorted(new_ips):
ip_lower = ip.lower()
if ip_lower not in existing_lower:
# Find IP section or add after last domain
ip_section_found = False
for i, line in enumerate(out_lines):
if line.strip().startswith("# IP Ranges / CIDRs") or line.strip().startswith("# IP"):
ip_section_found = True
# Add after the header
if i + 1 < len(out_lines) and not out_lines[i+1].strip().startswith("#"):
out_lines.insert(i + 1, ip)
else:
out_lines.append(ip)
break
if not ip_section_found:
# Add IP section at end
out_lines.append("")
out_lines.append("# IP Ranges / CIDRs")
out_lines.append(ip)
existing_lower.add(ip_lower)
with open(path, "w") as f:
for line in out_lines:
f.write(line + "\n")
return len(out_lines)
def update_streaming():
existing_lines = load_all_lines(STREAMING_FILE)
existing_domains, existing_ips = load_domains_and_ips(STREAMING_FILE)
all_domains = set(existing_domains)
all_ips = set(existing_ips)
for name, url in SOURCES_BY_TYPE["streaming"].items():
print(f"Fetching {name}...")
text = fetch_text(url)
if text:
scraped_domains, scraped_ips = extract_domains(text)
added = len(scraped_domains)
all_domains.update(scraped_domains)
all_ips.update(scraped_ips)
print(f" Scraped {added} new domains from {name}")
backup_file(STREAMING_FILE, "streaming_domains_whitelist")
new_count = save_domains(STREAMING_FILE, existing_lines, all_domains - existing_domains, all_ips - existing_ips)
print(f"Streaming: {len(all_domains)} domains + {len(all_ips)} IPs total")
def update_social():
existing_lines = load_all_lines(SOCIAL_FILE)
existing_domains, existing_ips = load_domains_and_ips(SOCIAL_FILE)
all_domains = set(existing_domains)
all_ips = set(existing_ips)
for name, url in SOURCES_BY_TYPE["social"].items():
print(f"Fetching {name}...")
text = fetch_text(url)
if text:
scraped_domains, scraped_ips = extract_domains(text)
social_keywords = ["facebook", "instagram", "twitter", "tiktok", "linkedin",
"snapchat", "threads", "mastodon", "reddit", "discord",
"pinterest", "youtube", "twitch", "vk", "tumblr",
"medium", "clubhouse", "messenger", "telegram", "quora",
"imgur", "vimeo", "bluesky", "bsky"]
added = 0
for d in scraped_domains:
if any(kw in d for kw in social_keywords):
all_domains.add(d)
added += 1
print(f" Scraped {added} new social domains from {name}")
backup_file(SOCIAL_FILE, "social_media")
new_count = save_domains(SOCIAL_FILE, existing_lines, all_domains - existing_domains, all_ips - existing_ips)
print(f"Social media: {len(all_domains)} domains + {len(all_ips)} IPs total")
def update_bypass():
"""Update bypass_sites.txt from social_media.txt + known_porn_domains.txt
bypass_sites.txt is the combined bypass list for social media and adult
content. It does NOT include streaming service domains (those live in
streaming_domains_whitelist.txt).
"""
social_domains, social_ips = load_domains_and_ips(SOCIAL_FILE)
porn_domains, porn_ips = load_domains_and_ips(PORN_FILE)
existing_lines = load_all_lines(BYPASS_FILE)
existing_domains_set = {l.strip().lower() for l in existing_lines if l.strip() and not l.strip().startswith("#")}
all_domains = social_domains | porn_domains
all_ips = social_ips | porn_ips
# Add OnlyFans/Fansly from their own sources
for name, url in SOURCES_BY_TYPE["bypass"].items():
print(f"Fetching {name}...")
text = fetch_text(url)
if text:
scraped_domains, scraped_ips = extract_domains(text)
added_d = len(scraped_domains)
added_i = len(scraped_ips)
all_domains.update(scraped_domains)
all_ips.update(scraped_ips)
print(f" Scraped {added_d} domains + {added_i} IPs from {name}")
new_domains = all_domains - existing_domains_set
new_ips = all_ips - {ip.lower() for ip in existing_domains_set if is_ip_or_cidr(ip.lower())}
backup_file(BYPASS_FILE, "bypass_sites")
new_count = save_domains(BYPASS_FILE, existing_lines, new_domains, new_ips)
print(f"Bypass sites: {len(existing_domains_set) + len(new_domains)} domains + {len(new_ips)} new IPs total")
def update_porn():
existing_lines = load_all_lines(PORN_FILE)
existing_domains, existing_ips = load_domains_and_ips(PORN_FILE)
backup_file(PORN_FILE, "known_porn_domains")
save_domains(PORN_FILE, existing_lines, set(), set())
print(f"Porn domains preserved: {len(existing_domains)} domains + {len(existing_ips)} IPs")
def main():
parser = argparse.ArgumentParser(description="Update domain lists - never delete!")
parser.add_argument("--streaming", action="store_true", help="Update streaming only")
parser.add_argument("--social", action="store_true", help="Update social media only")
parser.add_argument("--bypass", action="store_true", help="Update bypass_sites (combines streaming+social)")
parser.add_argument("--porn", action="store_true", help="Update porn domains")
parser.add_argument("--all", action="store_true", help="Update all lists")
args = parser.parse_args()
if args.all or not any([args.streaming, args.social, args.bypass, args.porn]):
update_streaming()
update_social()
update_bypass()
update_porn()
else:
if args.streaming: update_streaming()
if args.social: update_social()
if args.bypass: update_bypass()
if args.porn: update_porn()
if __name__ == "__main__":
main()