Merge branch 'main' into domains
This commit is contained in:
16 files changed
+16269
-92
No files matched your search
@@ -11,29 +11,28 @@ permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
language: ['python']
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: CodeQL Analysis
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
language: ['python']
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v3
|
||||
uses: github/codeql-action/init@v4
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
upload: true
|
||||
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@v3
|
||||
uses: github/codeql-action/autobuild@v4
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
# Non-blocking during initial rollout phase
|
||||
continue-on-error: true
|
||||
uses: github/codeql-action/analyze@v3
|
||||
uses: github/codeql-action/analyze@v4
|
||||
@@ -7,7 +7,7 @@ on:
|
||||
branches: [main]
|
||||
# Run weekly on schedule for ongoing monitoring
|
||||
schedule:
|
||||
- cron: ['0 6 * * 1'] # Mondays at 06:00 UTC
|
||||
- cron: '0 6 * * 1' # Mondays at 06:00 UTC
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -24,24 +24,26 @@ jobs:
|
||||
# Fetch full history for thorough scanning
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Run Trufflehog
|
||||
uses: trufflesecurity/trufflehog@main
|
||||
with:
|
||||
# Scan the full repository path
|
||||
path: ./
|
||||
# Output results in SARIF format for GitHub code scanning
|
||||
format: sarif
|
||||
output: results.sarif
|
||||
# Fail the workflow if secrets are found
|
||||
# Non-blocking during initial rollout phase — set to true to make it gating
|
||||
fail: false
|
||||
# Use entropy-based detection in addition to regex patterns
|
||||
only-failed: false
|
||||
- name: Install TruffleHog
|
||||
run: |
|
||||
# The trufflesecurity/trufflehog GitHub Action uses a Docker image
|
||||
# whose :latest CLI (v3.97.4) removed the `scan` subcommand and the
|
||||
# --format/--output flags that the action's run.sh still passes.
|
||||
# Install the binary directly instead.
|
||||
curl -sSL https://github.com/trufflesecurity/trufflehog/releases/download/v3.97.4/trufflehog_3.97.4_linux_amd64.tar.gz | tar -xz -C /usr/local/bin/
|
||||
trufflehog --help | head -1
|
||||
|
||||
- name: Upload Trufflehog results to GitHub Security tab
|
||||
if: always()
|
||||
uses: github/codeql-action/upload-security-sarif@v3
|
||||
- name: Run TruffleHog
|
||||
# --sarif outputs SARIF to stdout; redirect to file (no --output flag)
|
||||
# --fail exits 183 if secrets found; continue-on-error makes non-blocking
|
||||
run: |
|
||||
trufflehog filesystem . --sarif --fail --no-update > results.sarif
|
||||
continue-on-error: true
|
||||
|
||||
- name: Upload TruffleHog results to GitHub Security tab
|
||||
# Only upload if the SARIF file was created
|
||||
if: always() && hashFiles('results.sarif') != ''
|
||||
uses: github/codeql-action/upload-sarif@v4
|
||||
with:
|
||||
sarif_file: results.sarif
|
||||
# Don't fail the build on upload issues (scanning might have passed)
|
||||
checkout_path: ./
|
||||
checkout_path: .
|
||||
@@ -18,14 +18,14 @@ jobs:
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Run ShellCheck on all shell scripts
|
||||
# Non-blocking during initial rollout phase — set to false to make it gating
|
||||
continue-on-error: true
|
||||
uses: ludeeus/action-shellcheck@2.0.0
|
||||
with:
|
||||
# Check all .sh files, excluding .git and vendor directories
|
||||
files: |
|
||||
*.sh
|
||||
**/*.sh
|
||||
# Only fail on error-level issues (not warnings, info, or style)
|
||||
severity: error
|
||||
# Exclude directories we don't want to lint
|
||||
exclude: |
|
||||
ignore_paths: |
|
||||
.git
|
||||
.idea
|
||||
.vscode
|
||||
@@ -9,13 +9,19 @@
|
||||
### Clone and Set Up
|
||||
|
||||
```bash
|
||||
# Automatic setup (installs git if needed)
|
||||
curl -s https://raw.githubusercontent.com/wickedyoda/public-setupfiles/refs/heads/main/update-from_repo.sh | sudo bash
|
||||
# Automatic setup (installs git if needed, handles existing clones)
|
||||
curl -fsSL https://raw.githubusercontent.com/wickedyoda/public-setupfiles/refs/heads/main/update-from_repo.sh | sudo bash
|
||||
|
||||
# Or manual clone
|
||||
# Or manual clone / update
|
||||
# If you haven't cloned yet:
|
||||
git clone https://github.com/wickedyoda/public-setupfiles.git
|
||||
cd public-setupfiles
|
||||
chmod -R 755 .
|
||||
|
||||
# If you already have it cloned, just pull latest:
|
||||
cd public-setupfiles
|
||||
git pull
|
||||
chmod -R 755 .
|
||||
```
|
||||
|
||||
### Update Scripts
|
||||
|
||||
+16130
-37
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,74 @@
|
||||
# bypass_sites.txt Index
|
||||
|
||||
> Index of categorized domains in [`bypass_sites.txt`](./bypass_sites.txt)
|
||||
> This file is regenerated by `update_domains.py --bypass` (combines
|
||||
> social+porn, no streaming services).
|
||||
|
||||
## Structure
|
||||
|
||||
The file contains three sections, in order:
|
||||
|
||||
1. **Bypass Sites (18 services)** — Manually curated list with structured headers for:
|
||||
- Facebook/Meta, Amazon, Apple, TikTok/Bytedance, Instagram, Duck.com, ESPN,
|
||||
Flickr, Meetup, Zoom, Discord, Pinterest, Quora, Snapchat, OnlyFans, Fansly,
|
||||
WhatsApp — and their known IP ranges.
|
||||
|
||||
2. **Porn Sites** — Bulk-merged from `known_porn_domains.txt` and
|
||||
`known_porn_domains_v2.txt` with all duplicates removed against Section 1.
|
||||
|
||||
## Section 1 — Bypass Services
|
||||
|
||||
| # | Service Group | Example Domains | IP Ranges |
|
||||
|---|--------------|-----------------|-----------|
|
||||
| 1 | Facebook / Meta | facebook.com, fb.com, instagram.com, meta.com | 33 |
|
||||
| 2 | Amazon | amazon.com, amazonaws.com, s3.amazonaws.com | 0 |
|
||||
| 3 | Apple | apple.com, apple-dns.net, mzstatic.com, icloud.com | 3 |
|
||||
| 4 | TikTok / Bytedance | tiktok.com, bytedance.com, musical.ly | 15 |
|
||||
| 5 | Instagram | instagram.com, api.instagram.com, cdninstagram.com | 0 |
|
||||
| 6 | Duck.com | duck.com, duckduckgo.com, ddg.gg | 0 |
|
||||
| 7 | ESPN | espn.com, espn.net, espncdn.com | 0 |
|
||||
| 8 | Flickr | flickr.com, staticflickr.com | 0 |
|
||||
| 9 | Meetup | meetup.com, meetupstatic.com | 0 |
|
||||
| 10 | Zoom | zoom.us, zoom.com, zoomgov.com | 0 |
|
||||
| 11 | Discord | discord.com, discordapp.com, discord.media | 0 |
|
||||
| 12 | Messenger | messenger.com | 0 |
|
||||
| 13 | Pinterest | pinterest.com, pinimg.com, pinterestcdn.com | 0 |
|
||||
| 14 | Quora | quora.com, quoracdn.net | 0 |
|
||||
| 15 | Snapchat | snapchat.com, snap.com, snapkit.com | 64 |
|
||||
| 16 | OnlyFans | onlyfans.com, cdn3.onlyfans.com, api.onlyfans.com | 3 |
|
||||
| 17 | Fansly | fansly.com, cdn3.fansly.com, apiv3.fansly.com | 3 |
|
||||
| 18 | WhatsApp | whatsapp.com, whatsapp.net, wa.me | 8 |
|
||||
|
||||
## Section 2 — Porn Sites (merged)
|
||||
|
||||
| Source File | Unique Domains Added |
|
||||
|------------|---------------------|
|
||||
| known_porn_domains.txt | ~15,907 |
|
||||
| known_porn_domains_v2.txt | ~1,933 (unique, not in v1) |
|
||||
| **Total porn entries** | **16,117** |
|
||||
|
||||
## Section 3 — IP Ranges (deduplicated)
|
||||
Combined from all service sources. Total IP/CIDR entries: 111.
|
||||
|
||||
## Services NOT Included (by design)
|
||||
- Netflix, Hulu, Disney+, Amazon Prime Video — streaming services
|
||||
- Spotify, Deezer, Tidal, Pandora — music/audio streaming
|
||||
- YouTube, Twitch, Rumble — video platforms
|
||||
- Twitter/X, Reddit, VK, Tumblr, Weibo — social media (not requested)
|
||||
- Imgur, 4chan, Sina — non-requested platforms
|
||||
- Telegram — not in user's requested list
|
||||
|
||||
## IP Range Notes
|
||||
- Meta/Facebook: AS32934, AS2603, AS131772 (224.0.0.0/4 excluded — multicast)
|
||||
- OnlyFans/Fansly: Cloudflare AS13335 (shared ranges deduplicated)
|
||||
- WhatsApp: Meta AS32934
|
||||
|
||||
## Validation
|
||||
- ✅ Zero streaming domains in file
|
||||
- ✅ Zero non-approved social media domains
|
||||
- ✅ Zero duplicate entries (domains or IPs)
|
||||
- ✅ Zero digit-leading domains (filtered for DNS tool compatibility — Pi-hole, AdGuard, etc.)
|
||||
- ✅ All 18 service groups present with headers
|
||||
- ✅ All associated domains included (fb.com, m.me, wa.me, cloudfront.net, etc.)
|
||||
- ✅ known_porn_domains.txt and _v2.txt merged with deduplication
|
||||
- ✅ DOMAIN_RE in update_domains.py updated to match stricter validation (letter-first label)
|
||||
@@ -60,7 +60,7 @@ def backup_file(path, label):
|
||||
|
||||
DOMAIN_RE = re.compile(
|
||||
r'^(?=.{1,253}$)'
|
||||
r'([a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+'
|
||||
r'([a-zA-Z]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+'
|
||||
r'[a-zA-Z]{2,63}$'
|
||||
)
|
||||
|
||||
@@ -287,7 +287,7 @@ def main():
|
||||
parser = argparse.ArgumentParser(description="Update domain lists - never delete!")
|
||||
parser.add_argument("--streaming", action="store_true", help="Update streaming only")
|
||||
parser.add_argument("--social", action="store_true", help="Update social media only")
|
||||
parser.add_argument("--bypass", action="store_true", help="Update bypass_sites (combines streaming+social)")
|
||||
parser.add_argument("--bypass", action="store_true", help="Update bypass_sites (combines social+porn+OnlyFans/Fansly), no streaming")
|
||||
parser.add_argument("--porn", action="store_true", help="Update porn domains")
|
||||
parser.add_argument("--all", action="store_true", help="Update all lists")
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#!/bin/zsh
|
||||
# shellcheck shell=sh
|
||||
|
||||
echo "=== Updating Homebrew ==="
|
||||
brew update && brew upgrade
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#!/bin/zsh
|
||||
# shellcheck shell=sh
|
||||
|
||||
echo "=== Updating macOS Software ==="
|
||||
softwareupdate -i -a
|
||||
|
||||
@@ -16,7 +16,7 @@ if [ -z "$PKG_MANAGERS" ] || [ ${#PKG_MANAGERS[@]} -eq 0 ]; then
|
||||
echo "Error! No package managers specified"
|
||||
exit 1
|
||||
else
|
||||
for pkg in ${PKG_MANAGERS[@]}; do
|
||||
for pkg in "${PKG_MANAGERS[@]}"; do
|
||||
TEST=`which $pkg`
|
||||
if [ $? -eq 0 ]; then
|
||||
PKG_MAN=$pkg
|
||||
@@ -51,7 +51,7 @@ if [ -z "$OBSERVIUM_HOST" ]; then
|
||||
fi
|
||||
|
||||
if [ -z "$SVN_USER" ] || [ -z "$SVN_PASS" ]; then
|
||||
$ENTERPRISE = false
|
||||
ENTERPRISE=false
|
||||
fi
|
||||
|
||||
if [ "$ENTERPRISE" = true ]; then
|
||||
@@ -127,7 +127,7 @@ mkdir -p /usr/lib/observium_agent/local
|
||||
if [ -z "$MODULES" ] || [ ${#MODULES[@]} -eq 0 ]; then
|
||||
echo "No modules specified skipping..."
|
||||
else
|
||||
for mod in ${MODULES[@]}; do
|
||||
for mod in "${MODULES[@]}"; do
|
||||
cp observium/scripts/agent-local/$mod /usr/lib/observium_agent/local/
|
||||
done
|
||||
fi
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
#!/bin/bash
|
||||
!/bin/bash
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# ----------------------------------------------------------------------
|
||||
# | Add repos |
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
#!/bin/bash
|
||||
+2
-6
@@ -74,20 +74,16 @@ def main():
|
||||
# Check if directory already exists
|
||||
if os.path.isdir(local_dir) and os.listdir(local_dir):
|
||||
print(f"Directory '{local_dir}' exists. Pulling latest changes...")
|
||||
os.chdir(local_dir)
|
||||
result = run(["git", "pull", "origin", "main"])
|
||||
result = run(["git", "pull", "origin", "main"], cwd=local_dir)
|
||||
if result.returncode != 0:
|
||||
print("Git pull failed, re-cloning...")
|
||||
os.chdir("..")
|
||||
shutil.rmtree(local_dir)
|
||||
run(["git", "clone", remote_repo, local_dir])
|
||||
else:
|
||||
os.chdir(local_dir)
|
||||
else:
|
||||
print("Cloning repository...")
|
||||
run(["git", "clone", remote_repo, local_dir])
|
||||
os.chdir(local_dir)
|
||||
|
||||
os.chdir(local_dir)
|
||||
subprocess.run(["chmod", "-R", "755", "."], check=True)
|
||||
print("Done!")
|
||||
return 0
|
||||
|
||||
+1
-3
@@ -46,10 +46,8 @@ fi
|
||||
# Check if directory already exists
|
||||
if [ -d "$LOCAL_DIR" ] && [ "$(ls -A "$LOCAL_DIR" 2>/dev/null)" ]; then
|
||||
echo "Existing directory found. Pulling latest changes..."
|
||||
cd "$LOCAL_DIR" || exit 1
|
||||
if ! git pull origin main; then
|
||||
if ! (cd "$LOCAL_DIR" && git pull origin main); then
|
||||
echo "Git pull failed, re-cloning..."
|
||||
cd ..
|
||||
rm -rf "$LOCAL_DIR"
|
||||
git clone "$REMOTE_REPO" "$LOCAL_DIR"
|
||||
fi
|
||||
|
||||
Regular → Executable
+9
-3
@@ -31,9 +31,15 @@ if [ -s /tmp/influxdata-archive.key ]; then
|
||||
echo "deb [signed-by=/usr/share/keyrings/influxdata-archive.gpg] https://repos.influxdata.com/debian stable main" \
|
||||
| sudo tee /etc/apt/sources.list.d/influxdata.list >/dev/null
|
||||
else
|
||||
# Fallback: use trusted=yes (signature verification disabled)
|
||||
echo "deb [trusted=yes] https://repos.influxdata.com/debian stable main" \
|
||||
| sudo tee /etc/apt/sources.list.d/influxdata.list >/dev/null
|
||||
# Fallback: use existing keyring if already present
|
||||
if [ -s /usr/share/keyrings/influxdata-archive.gpg ]; then
|
||||
echo "deb [signed-by=/usr/share/keyrings/influxdata-archive.gpg] https://repos.influxdata.com/debian stable main" \
|
||||
| sudo tee /etc/apt/sources.list.d/influxdata.list >/dev/null
|
||||
else
|
||||
# Last resort: trusted=yes (signature verification disabled)
|
||||
echo "deb [trusted=yes] https://repos.influxdata.com/debian stable main" \
|
||||
| sudo tee /etc/apt/sources.list.d/influxdata.list >/dev/null
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "==> Fixing Sury PHP repo key"
|
||||
|
||||
Regular → Executable
+2
-1
@@ -1,5 +1,6 @@
|
||||
#!/bin/bash
|
||||
sudo ./fix_apt_influx.sh
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
sudo "$SCRIPT_DIR/fix_apt_influx.sh"
|
||||
|
||||
sudo apt update
|
||||
sudo apt upgrade -y
|
||||
|
||||
Reference in new issue
Block a user