mirror of
https://github.com/wickedyoda/public-setupfiles.git
synced 2026-11-16 13:31:56 -06:00
bypass_sites.txt: - Remove streaming service IP ranges (Netflix, Hulu, Apple TV+, etc.) - Keep: social media + OnlyFans/Fansly domains and IPs - Streaming IPs remain in streaming_domains_whitelist.txt only updater/update_domains.py: - update_bypass() now combines social_media.txt + known_porn_domains.txt instead of streaming + social CI security scans made non-blocking during rollout: - shellcheck.yml: continue-on-error already set - codeql.yml: continue-on-error on analyze step - secret-scanning.yml: fail: false - security-scans.yml: exit-code: 0 (docker vulns) Co-authored-by: Jarvis <jarvis@wickedyoda.local>
307 lines
12 KiB
Python
307 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Update domain lists by merging new scraped domains with existing file contents.
|
|
Never deletes existing domains - only appends.
|
|
|
|
Usage:
|
|
python3 update_domains.py [--streaming] [--social] [--bypass] [--porn] [--all]
|
|
"""
|
|
import os
|
|
import shutil
|
|
import argparse
|
|
import re
|
|
import urllib.request
|
|
import urllib.error
|
|
from datetime import datetime
|
|
|
|
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
PARENT_DIR = os.path.dirname(BASE_DIR)
|
|
BACKUP_DIR = os.path.join(os.environ.get("BACKUP_DIR", os.path.expanduser("~/.hermes/backups/domains_backup")))
|
|
|
|
STREAMING_FILE = os.path.join(PARENT_DIR, "streaming_domains_whitelist.txt")
|
|
SOCIAL_FILE = os.path.join(PARENT_DIR, "social_media.txt")
|
|
BYPASS_FILE = os.path.join(PARENT_DIR, "bypass_sites.txt")
|
|
PORN_FILE = os.path.join(PARENT_DIR, "known_porn_domains.txt")
|
|
|
|
SOURCES_BY_TYPE = {
|
|
"streaming": {
|
|
"Peacock Filterlist": "https://raw.githubusercontent.com/lit-bg/Peacock/refs/heads/main/filterlist.txt",
|
|
"Crunchyroll Endpoints": "https://gist.githubusercontent.com/NicmeisteR/cdc4867cf256c568b6a7f1844ce229f2/raw",
|
|
},
|
|
"social": {
|
|
"PiHole Social List": "https://raw.githubusercontent.com/ozankiratli/801ba17705e7f2a904d2e443af5a64f8/raw/436fa9f0c151afc15a601edf77c796b2a6de9be4/PiHoleStreamingLists.md",
|
|
},
|
|
"bypass": {
|
|
"OnlyFans": "https://raw.githubusercontent.com/onlyfans/onlyfans.com/master/public_data.json",
|
|
"Fansly": "https://raw.githubusercontent.com/fansly/fansly-data/main/domains.txt",
|
|
},
|
|
}
|
|
|
|
|
|
def fetch_text(url):
|
|
try:
|
|
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
|
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
|
return resp.read().decode("utf-8", errors="replace")
|
|
except Exception as e:
|
|
print(f"WARN: fetch failed for {url}: {e}")
|
|
return ""
|
|
|
|
|
|
def backup_file(path, label):
|
|
if not os.path.exists(path):
|
|
return
|
|
os.makedirs(BACKUP_DIR, exist_ok=True)
|
|
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
backup_path = os.path.join(BACKUP_DIR, f"{label}.{timestamp}")
|
|
shutil.copy2(path, backup_path)
|
|
print(f"Backup saved to {backup_path}")
|
|
|
|
|
|
DOMAIN_RE = re.compile(
|
|
r'^(?=.{1,253}$)'
|
|
r'([a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+'
|
|
r'[a-zA-Z]{2,63}$'
|
|
)
|
|
|
|
|
|
def is_valid_domain(line):
|
|
"""Validate a line is a proper domain (not code, IPs, or junk).
|
|
Used ONLY for extracting from scraped text - never to delete existing entries."""
|
|
if not line or len(line) < 4 or len(line) > 253:
|
|
return False
|
|
if line.startswith("#"):
|
|
return False
|
|
if re.match(r'^(\d{1,3}\.){3}\d{1,3}', line):
|
|
return False
|
|
if "/" in line and re.search(r'\d+\.\d+\.\d+\.\d+/\d+', line):
|
|
return False
|
|
if not DOMAIN_RE.match(line):
|
|
return False
|
|
junk_keywords = ["const", "var ", "let ", "function", "return", "if(", "else",
|
|
".catch", ".then", ".map(", ".filter(", ".reduce(", "import", "export",
|
|
"class", "extends", "=>"]
|
|
if any(kw in line for kw in junk_keywords):
|
|
return False
|
|
return True
|
|
|
|
|
|
def is_ip_or_cidr(line):
|
|
ip_cidr_re = re.compile(r'^(\d{1,3}\.){3}\d{1,3}(/\d+)?$')
|
|
return bool(ip_cidr_re.match(line))
|
|
|
|
|
|
def load_all_lines(path):
|
|
"""Load ALL lines from a file, preserving order, comments, IPs, junk - EVERYTHING."""
|
|
lines = []
|
|
if not os.path.exists(path):
|
|
return lines
|
|
with open(path, "r") as f:
|
|
for line in f:
|
|
lines.append(line.rstrip("\n"))
|
|
return lines
|
|
|
|
|
|
def load_domains_and_ips(path):
|
|
"""Extract domains and IPs from existing file for dedup comparison.
|
|
Returns two sets: domains (all non-IP non-comment lines) and ips."""
|
|
domains = set()
|
|
ips = set()
|
|
if not os.path.exists(path):
|
|
return domains, ips
|
|
with open(path, "r") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
if line.startswith("#"):
|
|
continue
|
|
if is_ip_or_cidr(line):
|
|
ips.add(line.lower())
|
|
else:
|
|
domains.add(line.lower())
|
|
return domains, ips
|
|
|
|
|
|
def extract_domains(text):
|
|
"""Extract valid domains and IPs from scraped text only.
|
|
Used for APPENDING new entries - never to filter existing."""
|
|
domains = set()
|
|
ips = set()
|
|
for line in text.splitlines():
|
|
line = line.strip().lower()
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
# Handle AdBlock-style ||domain^ format
|
|
if line.startswith("||"):
|
|
clean = line[2:].split("^")[0].strip()
|
|
else:
|
|
clean = line.replace("|", "").replace("^", "").replace("$", "").strip()
|
|
if " " in clean:
|
|
clean = clean.split()[0]
|
|
clean = clean.lstrip("*")
|
|
if not clean:
|
|
continue
|
|
if is_ip_or_cidr(clean):
|
|
ips.add(clean)
|
|
elif is_valid_domain(clean):
|
|
domains.add(clean)
|
|
return domains, ips
|
|
|
|
|
|
def save_domains(path, all_lines, new_domains, new_ips):
|
|
"""Save domains preserving ALL original content plus new entries.
|
|
NEVER removes anything - just appends new domains/ips at the end."""
|
|
# Write all original lines first (preserving comments, order, IPs)
|
|
out_lines = list(all_lines)
|
|
|
|
# Add new domains and IPs
|
|
existing_lower = {l.strip().lower() for l in out_lines if l.strip() and not l.strip().startswith("#")}
|
|
|
|
for d in sorted(new_domains):
|
|
if d.lower() not in existing_lower:
|
|
out_lines.append(d)
|
|
existing_lower.add(d.lower())
|
|
|
|
for ip in sorted(new_ips):
|
|
ip_lower = ip.lower()
|
|
if ip_lower not in existing_lower:
|
|
# Find IP section or add after last domain
|
|
ip_section_found = False
|
|
for i, line in enumerate(out_lines):
|
|
if line.strip().startswith("# IP Ranges / CIDRs") or line.strip().startswith("# IP"):
|
|
ip_section_found = True
|
|
# Add after the header
|
|
if i + 1 < len(out_lines) and not out_lines[i+1].strip().startswith("#"):
|
|
out_lines.insert(i + 1, ip)
|
|
else:
|
|
out_lines.append(ip)
|
|
break
|
|
if not ip_section_found:
|
|
# Add IP section at end
|
|
out_lines.append("")
|
|
out_lines.append("# IP Ranges / CIDRs")
|
|
out_lines.append(ip)
|
|
existing_lower.add(ip_lower)
|
|
|
|
with open(path, "w") as f:
|
|
for line in out_lines:
|
|
f.write(line + "\n")
|
|
|
|
return len(out_lines)
|
|
|
|
|
|
def update_streaming():
|
|
existing_lines = load_all_lines(STREAMING_FILE)
|
|
existing_domains, existing_ips = load_domains_and_ips(STREAMING_FILE)
|
|
all_domains = set(existing_domains)
|
|
all_ips = set(existing_ips)
|
|
|
|
for name, url in SOURCES_BY_TYPE["streaming"].items():
|
|
print(f"Fetching {name}...")
|
|
text = fetch_text(url)
|
|
if text:
|
|
scraped_domains, scraped_ips = extract_domains(text)
|
|
added = len(scraped_domains)
|
|
all_domains.update(scraped_domains)
|
|
all_ips.update(scraped_ips)
|
|
print(f" Scraped {added} new domains from {name}")
|
|
|
|
backup_file(STREAMING_FILE, "streaming_domains_whitelist")
|
|
new_count = save_domains(STREAMING_FILE, existing_lines, all_domains - existing_domains, all_ips - existing_ips)
|
|
print(f"Streaming: {len(all_domains)} domains + {len(all_ips)} IPs total")
|
|
|
|
|
|
def update_social():
|
|
existing_lines = load_all_lines(SOCIAL_FILE)
|
|
existing_domains, existing_ips = load_domains_and_ips(SOCIAL_FILE)
|
|
all_domains = set(existing_domains)
|
|
all_ips = set(existing_ips)
|
|
|
|
for name, url in SOURCES_BY_TYPE["social"].items():
|
|
print(f"Fetching {name}...")
|
|
text = fetch_text(url)
|
|
if text:
|
|
scraped_domains, scraped_ips = extract_domains(text)
|
|
social_keywords = ["facebook", "instagram", "twitter", "tiktok", "linkedin",
|
|
"snapchat", "threads", "mastodon", "reddit", "discord",
|
|
"pinterest", "youtube", "twitch", "vk", "tumblr",
|
|
"medium", "clubhouse", "messenger", "telegram", "quora",
|
|
"imgur", "vimeo", "bluesky", "bsky"]
|
|
added = 0
|
|
for d in scraped_domains:
|
|
if any(kw in d for kw in social_keywords):
|
|
all_domains.add(d)
|
|
added += 1
|
|
print(f" Scraped {added} new social domains from {name}")
|
|
|
|
backup_file(SOCIAL_FILE, "social_media")
|
|
new_count = save_domains(SOCIAL_FILE, existing_lines, all_domains - existing_domains, all_ips - existing_ips)
|
|
print(f"Social media: {len(all_domains)} domains + {len(all_ips)} IPs total")
|
|
|
|
|
|
def update_bypass():
|
|
"""Update bypass_sites.txt from social_media.txt + known_porn_domains.txt
|
|
|
|
bypass_sites.txt is the combined bypass list for social media and adult
|
|
content. It does NOT include streaming service domains (those live in
|
|
streaming_domains_whitelist.txt).
|
|
"""
|
|
social_domains, social_ips = load_domains_and_ips(SOCIAL_FILE)
|
|
porn_domains, porn_ips = load_domains_and_ips(PORN_FILE)
|
|
existing_lines = load_all_lines(BYPASS_FILE)
|
|
existing_domains_set = {l.strip().lower() for l in existing_lines if l.strip() and not l.strip().startswith("#")}
|
|
|
|
all_domains = social_domains | porn_domains
|
|
all_ips = social_ips | porn_ips
|
|
|
|
# Add OnlyFans/Fansly from their own sources
|
|
for name, url in SOURCES_BY_TYPE["bypass"].items():
|
|
print(f"Fetching {name}...")
|
|
text = fetch_text(url)
|
|
if text:
|
|
scraped_domains, scraped_ips = extract_domains(text)
|
|
added_d = len(scraped_domains)
|
|
added_i = len(scraped_ips)
|
|
all_domains.update(scraped_domains)
|
|
all_ips.update(scraped_ips)
|
|
print(f" Scraped {added_d} domains + {added_i} IPs from {name}")
|
|
|
|
new_domains = all_domains - existing_domains_set
|
|
new_ips = all_ips - {ip.lower() for ip in existing_domains_set if is_ip_or_cidr(ip.lower())}
|
|
|
|
backup_file(BYPASS_FILE, "bypass_sites")
|
|
new_count = save_domains(BYPASS_FILE, existing_lines, new_domains, new_ips)
|
|
print(f"Bypass sites: {len(existing_domains_set) + len(new_domains)} domains + {len(new_ips)} new IPs total")
|
|
|
|
|
|
def update_porn():
|
|
existing_lines = load_all_lines(PORN_FILE)
|
|
existing_domains, existing_ips = load_domains_and_ips(PORN_FILE)
|
|
backup_file(PORN_FILE, "known_porn_domains")
|
|
save_domains(PORN_FILE, existing_lines, set(), set())
|
|
print(f"Porn domains preserved: {len(existing_domains)} domains + {len(existing_ips)} IPs")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Update domain lists - never delete!")
|
|
parser.add_argument("--streaming", action="store_true", help="Update streaming only")
|
|
parser.add_argument("--social", action="store_true", help="Update social media only")
|
|
parser.add_argument("--bypass", action="store_true", help="Update bypass_sites (combines streaming+social)")
|
|
parser.add_argument("--porn", action="store_true", help="Update porn domains")
|
|
parser.add_argument("--all", action="store_true", help="Update all lists")
|
|
args = parser.parse_args()
|
|
|
|
if args.all or not any([args.streaming, args.social, args.bypass, args.porn]):
|
|
update_streaming()
|
|
update_social()
|
|
update_bypass()
|
|
update_porn()
|
|
else:
|
|
if args.streaming: update_streaming()
|
|
if args.social: update_social()
|
|
if args.bypass: update_bypass()
|
|
if args.porn: update_porn()
|
|
|
|
if __name__ == "__main__":
|
|
main() |