Merge branch 'main' into domains

This commit is contained in:
WickedYoda authored and GitHub committed 2026-09-26 21:55:46 -05:00
commit 598b4d0809
16 files changed
+16267 -90

No files matched your search

+7 -8
View File
@@ -11,29 +11,28 @@ permissions:
contents: read
security-events: write
strategy:
matrix:
language: ['python']
jobs:
analyze:
name: CodeQL Analysis
runs-on: ubuntu-latest
strategy:
matrix:
language: ['python']
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
uses: github/codeql-action/init@v4
with:
languages: ${{ matrix.language }}
upload: true
- name: Autobuild
uses: github/codeql-action/autobuild@v3
uses: github/codeql-action/autobuild@v4
- name: Perform CodeQL Analysis
# Non-blocking during initial rollout phase
continue-on-error: true
uses: github/codeql-action/analyze@v3
uses: github/codeql-action/analyze@v4
+21 -19
View File
@@ -7,7 +7,7 @@ on:
branches: [main]
# Run weekly on schedule for ongoing monitoring
schedule:
- cron: ['0 6 * * 1'] # Mondays at 06:00 UTC
- cron: '0 6 * * 1' # Mondays at 06:00 UTC
permissions:
contents: read
@@ -24,24 +24,26 @@ jobs:
# Fetch full history for thorough scanning
fetch-depth: 0
- name: Run Trufflehog
uses: trufflesecurity/trufflehog@main
with:
# Scan the full repository path
path: ./
# Output results in SARIF format for GitHub code scanning
format: sarif
output: results.sarif
# Fail the workflow if secrets are found
# Non-blocking during initial rollout phase — set to true to make it gating
fail: false
# Use entropy-based detection in addition to regex patterns
only-failed: false
- name: Install TruffleHog
run: |
# The trufflesecurity/trufflehog GitHub Action uses a Docker image
# whose :latest CLI (v3.97.4) removed the `scan` subcommand and the
# --format/--output flags that the action's run.sh still passes.
# Install the binary directly instead.
curl -sSL https://github.com/trufflesecurity/trufflehog/releases/download/v3.97.4/trufflehog_3.97.4_linux_amd64.tar.gz | tar -xz -C /usr/local/bin/
trufflehog --help | head -1
- name: Upload Trufflehog results to GitHub Security tab
if: always()
uses: github/codeql-action/upload-security-sarif@v3
- name: Run TruffleHog
# --sarif outputs SARIF to stdout; redirect to file (no --output flag)
# --fail exits 183 if secrets found; continue-on-error makes non-blocking
run: |
trufflehog filesystem . --sarif --fail --no-update > results.sarif
continue-on-error: true
- name: Upload TruffleHog results to GitHub Security tab
# Only upload if the SARIF file was created
if: always() && hashFiles('results.sarif') != ''
uses: github/codeql-action/upload-sarif@v4
with:
sarif_file: results.sarif
# Don't fail the build on upload issues (scanning might have passed)
checkout_path: ./
checkout_path: .
+5 -5
View File
@@ -18,14 +18,14 @@ jobs:
uses: actions/checkout@v4
- name: Run ShellCheck on all shell scripts
# Non-blocking during initial rollout phase — set to false to make it gating
continue-on-error: true
uses: ludeeus/action-shellcheck@2.0.0
with:
# Check all .sh files, excluding .git and vendor directories
files: |
*.sh
**/*.sh
# Only fail on error-level issues (not warnings, info, or style)
severity: error
# Exclude directories we don't want to lint
exclude: |
ignore_paths: |
.git
.idea
.vscode
Regular → Executable
+9 -3
View File
@@ -9,13 +9,19 @@
### Clone and Set Up
```bash
# Automatic setup (installs git if needed)
curl -s https://raw.githubusercontent.com/wickedyoda/public-setupfiles/refs/heads/main/update-from_repo.sh | sudo bash
# Automatic setup (installs git if needed, handles existing clones)
curl -fsSL https://raw.githubusercontent.com/wickedyoda/public-setupfiles/refs/heads/main/update-from_repo.sh | sudo bash
# Or manual clone
# Or manual clone / update
# If you haven't cloned yet:
git clone https://github.com/wickedyoda/public-setupfiles.git
cd public-setupfiles
chmod -R 755 .
# If you already have it cloned, just pull latest:
cd public-setupfiles
git pull
chmod -R 755 .
```
### Update Scripts
+16130 -37
View File
File diff suppressed because it is too large. Load diff
+74
View File
@@ -0,0 +1,74 @@
# bypass_sites.txt Index
> Index of categorized domains in [`bypass_sites.txt`](./bypass_sites.txt)
> This file is regenerated by `update_domains.py --bypass` (combines
> social+porn, no streaming services).
## Structure
The file contains three sections, in order:
1. **Bypass Sites (18 services)** — Manually curated list with structured headers for:
- Facebook/Meta, Amazon, Apple, TikTok/Bytedance, Instagram, Duck.com, ESPN,
Flickr, Meetup, Zoom, Discord, Pinterest, Quora, Snapchat, OnlyFans, Fansly,
WhatsApp — and their known IP ranges.
2. **Porn Sites** — Bulk-merged from `known_porn_domains.txt` and
`known_porn_domains_v2.txt` with all duplicates removed against Section 1.
## Section 1 — Bypass Services
| # | Service Group | Example Domains | IP Ranges |
|---|--------------|-----------------|-----------|
| 1 | Facebook / Meta | facebook.com, fb.com, instagram.com, meta.com | 33 |
| 2 | Amazon | amazon.com, amazonaws.com, s3.amazonaws.com | 0 |
| 3 | Apple | apple.com, apple-dns.net, mzstatic.com, icloud.com | 3 |
| 4 | TikTok / Bytedance | tiktok.com, bytedance.com, musical.ly | 15 |
| 5 | Instagram | instagram.com, api.instagram.com, cdninstagram.com | 0 |
| 6 | Duck.com | duck.com, duckduckgo.com, ddg.gg | 0 |
| 7 | ESPN | espn.com, espn.net, espncdn.com | 0 |
| 8 | Flickr | flickr.com, staticflickr.com | 0 |
| 9 | Meetup | meetup.com, meetupstatic.com | 0 |
| 10 | Zoom | zoom.us, zoom.com, zoomgov.com | 0 |
| 11 | Discord | discord.com, discordapp.com, discord.media | 0 |
| 12 | Messenger | messenger.com | 0 |
| 13 | Pinterest | pinterest.com, pinimg.com, pinterestcdn.com | 0 |
| 14 | Quora | quora.com, quoracdn.net | 0 |
| 15 | Snapchat | snapchat.com, snap.com, snapkit.com | 64 |
| 16 | OnlyFans | onlyfans.com, cdn3.onlyfans.com, api.onlyfans.com | 3 |
| 17 | Fansly | fansly.com, cdn3.fansly.com, apiv3.fansly.com | 3 |
| 18 | WhatsApp | whatsapp.com, whatsapp.net, wa.me | 8 |
## Section 2 — Porn Sites (merged)
| Source File | Unique Domains Added |
|------------|---------------------|
| known_porn_domains.txt | ~15,907 |
| known_porn_domains_v2.txt | ~1,933 (unique, not in v1) |
| **Total porn entries** | **16,117** |
## Section 3 — IP Ranges (deduplicated)
Combined from all service sources. Total IP/CIDR entries: 111.
## Services NOT Included (by design)
- Netflix, Hulu, Disney+, Amazon Prime Video — streaming services
- Spotify, Deezer, Tidal, Pandora — music/audio streaming
- YouTube, Twitch, Rumble — video platforms
- Twitter/X, Reddit, VK, Tumblr, Weibo — social media (not requested)
- Imgur, 4chan, Sina — non-requested platforms
- Telegram — not in user's requested list
## IP Range Notes
- Meta/Facebook: AS32934, AS2603, AS131772 (224.0.0.0/4 excluded — multicast)
- OnlyFans/Fansly: Cloudflare AS13335 (shared ranges deduplicated)
- WhatsApp: Meta AS32934
## Validation
- ✅ Zero streaming domains in file
- ✅ Zero non-approved social media domains
- ✅ Zero duplicate entries (domains or IPs)
- ✅ Zero digit-leading domains (filtered for DNS tool compatibility — Pi-hole, AdGuard, etc.)
- ✅ All 18 service groups present with headers
- ✅ All associated domains included (fb.com, m.me, wa.me, cloudfront.net, etc.)
- ✅ known_porn_domains.txt and _v2.txt merged with deduplication
- ✅ DOMAIN_RE in update_domains.py updated to match stricter validation (letter-first label)
+2 -2
View File
@@ -60,7 +60,7 @@ def backup_file(path, label):
DOMAIN_RE = re.compile(
r'^(?=.{1,253}$)'
r'([a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+'
r'([a-zA-Z]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+'
r'[a-zA-Z]{2,63}$'
)
@@ -287,7 +287,7 @@ def main():
parser = argparse.ArgumentParser(description="Update domain lists - never delete!")
parser.add_argument("--streaming", action="store_true", help="Update streaming only")
parser.add_argument("--social", action="store_true", help="Update social media only")
parser.add_argument("--bypass", action="store_true", help="Update bypass_sites (combines streaming+social)")
parser.add_argument("--bypass", action="store_true", help="Update bypass_sites (combines social+porn+OnlyFans/Fansly), no streaming")
parser.add_argument("--porn", action="store_true", help="Update porn domains")
parser.add_argument("--all", action="store_true", help="Update all lists")
args = parser.parse_args()
+1
View File
@@ -1,4 +1,5 @@
#!/bin/zsh
# shellcheck shell=sh
echo "=== Updating Homebrew ==="
brew update && brew upgrade
+1
View File
@@ -1,4 +1,5 @@
#!/bin/zsh
# shellcheck shell=sh
echo "=== Updating macOS Software ==="
softwareupdate -i -a
@@ -16,7 +16,7 @@ if [ -z "$PKG_MANAGERS" ] || [ ${#PKG_MANAGERS[@]} -eq 0 ]; then
echo "Error! No package managers specified"
exit 1
else
for pkg in ${PKG_MANAGERS[@]}; do
for pkg in "${PKG_MANAGERS[@]}"; do
TEST=`which $pkg`
if [ $? -eq 0 ]; then
PKG_MAN=$pkg
@@ -51,7 +51,7 @@ if [ -z "$OBSERVIUM_HOST" ]; then
fi
if [ -z "$SVN_USER" ] || [ -z "$SVN_PASS" ]; then
$ENTERPRISE = false
ENTERPRISE=false
fi
if [ "$ENTERPRISE" = true ]; then
@@ -127,7 +127,7 @@ mkdir -p /usr/lib/observium_agent/local
if [ -z "$MODULES" ] || [ ${#MODULES[@]} -eq 0 ]; then
echo "No modules specified skipping..."
else
for mod in ${MODULES[@]}; do
for mod in "${MODULES[@]}"; do
cp observium/scripts/agent-local/$mod /usr/lib/observium_agent/local/
done
fi
+1 -2
View File
@@ -1,7 +1,6 @@
#!/bin/bash
!/bin/bash
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# | Add repos |
# ----------------------------------------------------------------------
+1
View File
@@ -0,0 +1 @@
#!/bin/bash
+2 -6
View File
@@ -74,20 +74,16 @@ def main():
# Check if directory already exists
if os.path.isdir(local_dir) and os.listdir(local_dir):
print(f"Directory '{local_dir}' exists. Pulling latest changes...")
os.chdir(local_dir)
result = run(["git", "pull", "origin", "main"])
result = run(["git", "pull", "origin", "main"], cwd=local_dir)
if result.returncode != 0:
print("Git pull failed, re-cloning...")
os.chdir("..")
shutil.rmtree(local_dir)
run(["git", "clone", remote_repo, local_dir])
else:
os.chdir(local_dir)
else:
print("Cloning repository...")
run(["git", "clone", remote_repo, local_dir])
os.chdir(local_dir)
os.chdir(local_dir)
subprocess.run(["chmod", "-R", "755", "."], check=True)
print("Done!")
return 0
+1 -3
View File
@@ -46,10 +46,8 @@ fi
# Check if directory already exists
if [ -d "$LOCAL_DIR" ] && [ "$(ls -A "$LOCAL_DIR" 2>/dev/null)" ]; then
echo "Existing directory found. Pulling latest changes..."
cd "$LOCAL_DIR" || exit 1
if ! git pull origin main; then
if ! (cd "$LOCAL_DIR" && git pull origin main); then
echo "Git pull failed, re-cloning..."
cd ..
rm -rf "$LOCAL_DIR"
git clone "$REMOTE_REPO" "$LOCAL_DIR"
fi
+7 -1
View File
@@ -31,9 +31,15 @@ if [ -s /tmp/influxdata-archive.key ]; then
echo "deb [signed-by=/usr/share/keyrings/influxdata-archive.gpg] https://repos.influxdata.com/debian stable main" \
| sudo tee /etc/apt/sources.list.d/influxdata.list >/dev/null
else
# Fallback: use trusted=yes (signature verification disabled)
# Fallback: use existing keyring if already present
if [ -s /usr/share/keyrings/influxdata-archive.gpg ]; then
echo "deb [signed-by=/usr/share/keyrings/influxdata-archive.gpg] https://repos.influxdata.com/debian stable main" \
| sudo tee /etc/apt/sources.list.d/influxdata.list >/dev/null
else
# Last resort: trusted=yes (signature verification disabled)
echo "deb [trusted=yes] https://repos.influxdata.com/debian stable main" \
| sudo tee /etc/apt/sources.list.d/influxdata.list >/dev/null
fi
fi
echo "==> Fixing Sury PHP repo key"
Regular → Executable
+2 -1
View File
@@ -1,5 +1,6 @@
#!/bin/bash
sudo ./fix_apt_influx.sh
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
sudo "$SCRIPT_DIR/fix_apt_influx.sh"
sudo apt update
sudo apt upgrade -y