mirror of
https://github.com/wickedyoda/public-setupfiles.git
synced 2026-11-16 13:31:56 -06:00
finalize domain updater script (#109)
* Add update_domains.py with --streaming/--social/--all flags * Update: always merge, never delete; add --bypass for combined list * Fix: preserve all domains+IPs, separate IPs into CIDR section * Update all domain lists via scraping - no domains removed, only new domains appended - streaming_domains_whitelist.txt: 198 -> 222 lines (33 new domains from Peacock + Crunchyroll) - social_media.txt: 133 -> 136 lines (3 new social domains from PiHole list) - bypass_sites.txt: 251 -> 354 lines (103 new from combined streaming+social) - known_porn_domains.txt: 15918 -> 15920 (no removals) - bamboo_domains.txt: 19 lines (unchanged, preserved) - Script updated to load_all_preserving_everything: never delete --------- Co-authored-by: Jarvis <jarvis@wickedyoda.local>
This commit is contained in:
1 parent
37ddee3579
commit
091400ca9e
6 files changed
+858
-588
No files matched your search
+275
-172
@@ -1,44 +1,103 @@
|
||||
# --- added domains from research ---
|
||||
# amazon prime video (additional)
|
||||
# apple tv+ (rebranded from apple tv+)
|
||||
# crunchyroll (additional)
|
||||
# duckduckgo video search
|
||||
# general streaming protocols
|
||||
# ip ranges / cidrs
|
||||
# paramount+ (additional)
|
||||
# peacock (additional domains from filterlist)
|
||||
# roku
|
||||
# youtube (additional - common streaming)
|
||||
about.meta.com
|
||||
abs.twimg.com
|
||||
accounts.facebook.com
|
||||
accounts.twitter.com
|
||||
accounts.fetlife.com
|
||||
ads-api.twitter.com
|
||||
acorn.tv
|
||||
ads.facebook.com
|
||||
ads.twitter.com
|
||||
analytics.twitter.com
|
||||
aiv-cdn.net
|
||||
akamaized.net
|
||||
amazon.com
|
||||
amazonaws.com
|
||||
amp.twimg.com
|
||||
api-manga.crunchyroll.com
|
||||
api.crunchyroll.com
|
||||
api.facebook.com
|
||||
api.twitter.com
|
||||
api.x.com
|
||||
api.fetlife.com
|
||||
api.roku.com
|
||||
api.rokutime.com
|
||||
api.rpay.roku.com
|
||||
api.spotify.com
|
||||
api2.sr.roku.com
|
||||
apis.rpay.roku.com
|
||||
apple-dns.net
|
||||
apple-tv-plus.com
|
||||
apple.com
|
||||
apple.com.edgekey.net
|
||||
apps.facebook.com
|
||||
atom.peacocktv.com
|
||||
atv-ext.amazon.com
|
||||
availability-fastly.syncbak-mediastore-cedexis.cbsaavideo.com
|
||||
badoo.com
|
||||
bellesa.co
|
||||
bellesa.com
|
||||
bakery.pplus.paramount.tech
|
||||
bam.nr-data.net
|
||||
bamgrid.com
|
||||
beta-api.crunchyroll.com
|
||||
beta-stage-api.crunchyroll.com
|
||||
bff-ext.clients.peacocktv.com
|
||||
bluesky.app
|
||||
bookmarks.sr.roku.com
|
||||
britbox.com
|
||||
bsky.social
|
||||
business.facebook.com
|
||||
business.twitter.com
|
||||
camwhores.tv
|
||||
cards.twitter.com
|
||||
cdn.syndication.twimg.com
|
||||
bytedance.com
|
||||
bytedance.net
|
||||
bytedanceapi.com
|
||||
byteoversea.com
|
||||
cbs.com
|
||||
cbsaavideo.com
|
||||
cbsi.demdex.net
|
||||
cbsi.live.ott.irdeto.com
|
||||
cbsinteractive.hb.omtrdc.net
|
||||
cbsivideo.com
|
||||
cdn-medium.com
|
||||
cdn.peacocktv.com
|
||||
cdn.privacy.paramount.com
|
||||
cdn.registerdisney.go.com
|
||||
cdn.spotify.com
|
||||
cdninstagram.com
|
||||
cdn.fetlife.com
|
||||
cloudfront.net
|
||||
clubhouse.com
|
||||
clubhouseapi.com
|
||||
connect.facebook.com
|
||||
connect.facebook.net
|
||||
crazyshit.com
|
||||
darknessporn.com
|
||||
content-detail.sr.roku.com
|
||||
content.sr.roku.com
|
||||
conviva.com
|
||||
crunchyroll.com
|
||||
cybertron.id.peacocktv.com
|
||||
d1v0o82c0gfg2u.cloudfront.net
|
||||
d2cvj4cr2d2r7g.cloudfront.net
|
||||
d3n92br3b4jq4j.cloudfront.net
|
||||
deezer.com
|
||||
dev.crunchyroll.com
|
||||
developers.facebook.com
|
||||
developer.twitter.com
|
||||
dev.twitter.com
|
||||
dipseastories.com
|
||||
discoveryplus.com
|
||||
disney.com
|
||||
disneyplus.com
|
||||
disneystreaming.com
|
||||
doppler.hulu.com
|
||||
dplay.com
|
||||
dssott.com
|
||||
duck.com
|
||||
duckduckgo.com
|
||||
e.delaymultiplier
|
||||
e.retry
|
||||
edge.facebook.com
|
||||
edgesuite.net
|
||||
enduser.adsrv.org
|
||||
espn.com
|
||||
espn.net
|
||||
espncdn.com
|
||||
facebook.com
|
||||
facebook.net
|
||||
fansly.com
|
||||
faphouse.com
|
||||
fabxh.site
|
||||
fb.com
|
||||
fb.gg
|
||||
fb.me
|
||||
@@ -46,206 +105,250 @@ fb.watch
|
||||
fbcdn.com
|
||||
fbcdn.net
|
||||
fbsbx.com
|
||||
fetlife.com
|
||||
www.fetlife.com
|
||||
static.fetlife.com
|
||||
media.fetlife.com
|
||||
images.fetlife.com
|
||||
beta.fetlife.com
|
||||
m.fetlife.com
|
||||
flickr.com
|
||||
frolicme.com
|
||||
fullporno.xxx
|
||||
graphql.twitter.com
|
||||
fox.com
|
||||
foxsports.com
|
||||
freevee.com
|
||||
fubo.tv
|
||||
funimation.com
|
||||
fwmrm.net
|
||||
googlevideo.com
|
||||
graph.facebook.com
|
||||
help.twitter.com
|
||||
hotscope.tv
|
||||
hbo.com
|
||||
hbomax.com
|
||||
hulu.com
|
||||
hulu.hb.omtrdc.net
|
||||
huluad.com
|
||||
huluim.com
|
||||
hulustream.com
|
||||
ibyteimg.com
|
||||
image.roku.com
|
||||
images-svc.sr.roku.com
|
||||
imageservice.disco.peacocktv.com
|
||||
imasdk.googleapis.com
|
||||
imdb.com
|
||||
imdbtv.com
|
||||
imgur.com
|
||||
imgur.io
|
||||
imgurmedia.com
|
||||
ingest.webcol.sdp.peacocktv.com
|
||||
innovidvideo.com
|
||||
instagram.com
|
||||
jtvnw.net
|
||||
kick.com
|
||||
licdn.com
|
||||
linkedin.com
|
||||
l.facebook.com
|
||||
literotica.com
|
||||
login.twitter.com
|
||||
licdn.com
|
||||
linkedin.cn
|
||||
linkedin.com
|
||||
llnwd.net
|
||||
lm.facebook.com
|
||||
login.facebook.com
|
||||
mastodon.social
|
||||
max.com
|
||||
medium.com
|
||||
meetup.com
|
||||
meg.disco.peacocktv.com
|
||||
messenger.com
|
||||
meta.com
|
||||
metacdn.com
|
||||
mobile.twitter.com
|
||||
m.twitter.com
|
||||
mobile.x.com
|
||||
motherless.com
|
||||
motherlessmedia.com
|
||||
mlb.com
|
||||
mlbstatic.com
|
||||
moviefone.com
|
||||
mparticle.com
|
||||
mubi.com
|
||||
musical.ly
|
||||
mytv.clients.peacocktv.com
|
||||
mzstatic.com
|
||||
n.account
|
||||
n.assets
|
||||
n.auth
|
||||
n.content
|
||||
n.devices
|
||||
n.etpconfig
|
||||
n.funmigrator
|
||||
n.incentives
|
||||
n.localization
|
||||
n.messaging
|
||||
n.news
|
||||
n.playservice
|
||||
n.reviews
|
||||
n.subscription
|
||||
n.talkbox
|
||||
n.thirdpartyoauth
|
||||
navigation.sr.roku.com
|
||||
nba.com
|
||||
nbc.com
|
||||
nbcucdn.com
|
||||
nbcuni.com
|
||||
netflix.com
|
||||
netflix.net
|
||||
netflixdnstest0.com
|
||||
nextdoor.com
|
||||
oauth.twitter.com
|
||||
nfl.com
|
||||
nflxext.com
|
||||
nflximg.com
|
||||
nflxso.net
|
||||
nflxvideo.net
|
||||
nhl.com
|
||||
nhlstatic.com
|
||||
oculus.com
|
||||
onlyfans.com
|
||||
omtrdc.net
|
||||
open.spotify.com
|
||||
ovp.peacocktv.com
|
||||
pagead2.googlesyndication.com
|
||||
pandora.com
|
||||
paramount.tech
|
||||
paramountplus.com
|
||||
parler.com
|
||||
pbs.twimg.com
|
||||
pinterest.com
|
||||
pconfig-prd.cdn.peacocktv.com
|
||||
peacocktv.com
|
||||
philo.com
|
||||
pinimg.com
|
||||
platform.twitter.com
|
||||
pornhub.com
|
||||
promoted.twitter.com
|
||||
publish.twitter.com
|
||||
quinn.com
|
||||
pinterest.com
|
||||
pinterestcdn.com
|
||||
play.ovp.peacocktv.com
|
||||
playback-detail.sr.roku.com
|
||||
pluto.tv
|
||||
primevideo.com
|
||||
quora.com
|
||||
redtube.com
|
||||
rule34.xxx
|
||||
quoracdn.net
|
||||
retail.rpay.roku.com
|
||||
rights-manager.sr.roku.com
|
||||
roku.admeasurement.com
|
||||
roku.com
|
||||
rokutime.com
|
||||
rumble.com
|
||||
s3.amazonaws.com
|
||||
saa.cbsi.com
|
||||
saa.paramountplus.com
|
||||
sc-cdn.net
|
||||
scdn.co
|
||||
scorecardresearch.com
|
||||
secure.facebook.com
|
||||
shudder.com
|
||||
signal.org
|
||||
sinaimg.cn
|
||||
sling.com
|
||||
snap.com
|
||||
snapads.com
|
||||
snapchat.com
|
||||
snapkit.com
|
||||
spaces.twitter.com
|
||||
soundcloud.com
|
||||
spotify.com
|
||||
ssai.peacocktv.com
|
||||
stage-api-manga.crunchyroll.com
|
||||
stage-api.crunchyroll.com
|
||||
static-delivery.sr.roku.com
|
||||
static.ak.connect.facebook.com
|
||||
static.ak.facebook.com
|
||||
static.xx.fbcdn.net
|
||||
staticflickr.com
|
||||
status.twitterstat.us
|
||||
stream.twitter.com
|
||||
support.twitter.com
|
||||
syndication.twimg.com
|
||||
t.co
|
||||
www.t.co
|
||||
telegram.org
|
||||
t.me
|
||||
www.t.me
|
||||
telegram.me
|
||||
telegram.org
|
||||
telemetry.nbcuott.com
|
||||
tfbnw.net
|
||||
theporndude.com
|
||||
thexh.today
|
||||
therokuchannel.roku.com
|
||||
this.audiolanguage
|
||||
this.country
|
||||
this.intercept
|
||||
this.locale
|
||||
this.settabid
|
||||
this.tabid
|
||||
threads.net
|
||||
thumbnails.cbsig.net
|
||||
tidal.com
|
||||
tiktok.com
|
||||
tiktokcdn.com
|
||||
tiktokcdn-eu.com
|
||||
tiktokcdn-us.com
|
||||
tiktokcdn.com
|
||||
tiktoklinksafety.us
|
||||
tiktokw.us
|
||||
tiktok.us
|
||||
tiktokv.com
|
||||
tiktokv.eu
|
||||
tiktokv.us
|
||||
ton.twitter.com
|
||||
ton.twimg.com
|
||||
tiktokw.us
|
||||
truthsocial.com
|
||||
tryquinn.com
|
||||
tsyndicate.com
|
||||
tumblr.com
|
||||
twitch.tv
|
||||
ttvnw.net
|
||||
twitter.com
|
||||
www.twitter.com
|
||||
twimg.com
|
||||
tubitv.com
|
||||
tumblr.com
|
||||
tv.apple.com
|
||||
twitch.tv
|
||||
upload.facebook.com
|
||||
upload.twitter.com
|
||||
video.twimg.com
|
||||
viacomcbs.com
|
||||
video.xx.fbcdn.net
|
||||
vimeo.com
|
||||
vimeocdn.com
|
||||
vk.com
|
||||
vk.ru
|
||||
vkcdn.net
|
||||
vkuser.net
|
||||
vod-gcs-cedexis.cbsaavideo.com
|
||||
vod-gcs-google.cbsaavideo.com
|
||||
vod-gcs-qwilt.cbsaavideo.com
|
||||
vod-playlist.sr.roku.com
|
||||
vod.delivery.roku.com
|
||||
vortex.hulu.com
|
||||
vrv.co
|
||||
vudu.com
|
||||
warnermediacdn.com
|
||||
wbdstreaming.com
|
||||
weibo.cn
|
||||
weibo.com
|
||||
whatsapp.com
|
||||
whatsapp.net
|
||||
white.ish.instagram.com
|
||||
x.com
|
||||
www.x.com
|
||||
xhcdn.com
|
||||
xhamster.com
|
||||
xvideos.com
|
||||
youporn.com
|
||||
googletagmanager.com
|
||||
goo.gl
|
||||
xnxx.com
|
||||
www.xnxx.com
|
||||
spankbang.com
|
||||
www.spankbang.com
|
||||
tube8.com
|
||||
www.tube8.com
|
||||
beeg.com
|
||||
www.beeg.com
|
||||
brazzers.com
|
||||
www.brazzers.com
|
||||
realitykings.com
|
||||
www.realitykings.com
|
||||
mofos.com
|
||||
www.mofos.com
|
||||
bangbros.com
|
||||
www.bangbros.com
|
||||
teamskeet.com
|
||||
www.teamskeet.com
|
||||
naughtyamerica.com
|
||||
www.naughtyamerica.com
|
||||
eporner.com
|
||||
www.eporner.com
|
||||
txxx.com
|
||||
www.txxx.com
|
||||
porn.com
|
||||
www.porn.com
|
||||
sex.com
|
||||
www.sex.com
|
||||
sxyprn.com
|
||||
www.sxyprn.com
|
||||
thumbzilla.com
|
||||
www.thumbzilla.com
|
||||
tnaflix.com
|
||||
www.tnaflix.com
|
||||
empflix.com
|
||||
www.empflix.com
|
||||
jable.tv
|
||||
www.jable.tv
|
||||
missav.com
|
||||
www.missav.com
|
||||
hanime.tv
|
||||
www.hanime.tv
|
||||
hentaifox.com
|
||||
www.hentaifox.com
|
||||
nhentai.net
|
||||
www.nhentai.net
|
||||
gelbooru.com
|
||||
www.gelbooru.com
|
||||
manyvids.com
|
||||
www.manyvids.com
|
||||
chaturbate.com
|
||||
www.chaturbate.com
|
||||
stripchat.com
|
||||
www.stripchat.com
|
||||
myfreecams.com
|
||||
www.myfreecams.com
|
||||
camsoda.com
|
||||
www.camsoda.com
|
||||
livejasmin.com
|
||||
www.livejasmin.com
|
||||
bongacams.com
|
||||
www.bongacams.com
|
||||
cam4.com
|
||||
www.cam4.com
|
||||
streamate.com
|
||||
www.streamate.com
|
||||
adultfriendfinder.com
|
||||
www.adultfriendfinder.com
|
||||
ashleymadison.com
|
||||
www.ashleymadison.com
|
||||
web.libera.com
|
||||
www.instagram.com
|
||||
i.instagram.com
|
||||
api.instagram.com
|
||||
graph.instagram.com
|
||||
help.instagram.com
|
||||
about.instagram.com
|
||||
business.instagram.com
|
||||
creators.instagram.com
|
||||
accountscenter.instagram.com
|
||||
static.cdninstagram.com
|
||||
scontent.cdninstagram.com
|
||||
wmcdp.com
|
||||
wmcdp.io
|
||||
wv-license.sr.roku.com
|
||||
www.crunchyroll.com
|
||||
www.googletagmanager.com
|
||||
wwwimage-secure.cbsstatic.com
|
||||
wwwimage-us.pplusstatic.com
|
||||
xumo.com
|
||||
youtu.be
|
||||
youtube.com
|
||||
ytimg.com
|
||||
|
||||
# IP Ranges / CIDRs
|
||||
103.136.220.0/22
|
||||
103.4.96.0/22
|
||||
108.175.32.0/20
|
||||
129.134.0.0/16
|
||||
13.107.0.0/16
|
||||
149.154.160.0/20
|
||||
157.240.0.0/16
|
||||
161.117.0.0/16
|
||||
17.0.0.0/8
|
||||
173.252.64.0/18
|
||||
179.60.192.0/22
|
||||
185.60.216.0/22
|
||||
192.173.64.0/18
|
||||
198.38.96.0/19
|
||||
198.45.48.0/20
|
||||
199.96.56.0/21
|
||||
204.15.20.0/22
|
||||
207.45.72.0/22
|
||||
208.75.76.0/22
|
||||
224.0.0.0/4
|
||||
31.13.24.0/21
|
||||
31.13.64.0/18
|
||||
34.192.0.0/10
|
||||
40.0.0.0/8
|
||||
45.57.0.0/17
|
||||
47.246.0.0/16
|
||||
52.0.0.0/11
|
||||
52.96.0.0/12
|
||||
64.120.128.0/17
|
||||
66.197.128.0/17
|
||||
66.220.144.0/20
|
||||
69.171.224.0/19
|
||||
69.53.224.0/19
|
||||
69.63.176.0/20
|
||||
74.119.76.0/22
|
||||
87.240.128.0/18
|
||||
91.108.12.0/22
|
||||
91.108.16.0/22
|
||||
91.108.4.0/22
|
||||
91.108.56.0/22
|
||||
91.108.8.0/22
|
||||
93.186.224.0/20
|
||||
95.213.0.0/16
|
||||
@@ -15061,6 +15061,8 @@ zzz-icy.com
|
||||
zzzjtd.com
|
||||
zzzyk.com
|
||||
zzzzporn.comreations.comom
|
||||
|
||||
# IP Ranges / CIDRs
|
||||
1.117.152.239
|
||||
1.180.245.195
|
||||
1.195.111.251
|
||||
|
||||
+125
-123
@@ -1,134 +1,136 @@
|
||||
facebook.com
|
||||
facebook.net
|
||||
fb.com
|
||||
fb.me
|
||||
fb.watch
|
||||
fb.gg
|
||||
fbcdn.net
|
||||
fbcdn.com
|
||||
fbsbx.com
|
||||
tfbnw.net
|
||||
connect.facebook.com
|
||||
connect.facebook.net
|
||||
graph.facebook.com
|
||||
api.facebook.com
|
||||
upload.facebook.com
|
||||
secure.facebook.com
|
||||
login.facebook.com
|
||||
accounts.facebook.com
|
||||
business.facebook.com
|
||||
developers.facebook.com
|
||||
apps.facebook.com
|
||||
ads.facebook.com
|
||||
static.ak.facebook.com
|
||||
static.ak.connect.facebook.com
|
||||
l.facebook.com
|
||||
lm.facebook.com
|
||||
edge.facebook.com
|
||||
video.xx.fbcdn.net
|
||||
static.xx.fbcdn.net
|
||||
messenger.com
|
||||
instagram.com
|
||||
cdninstagram.com
|
||||
whatsapp.com
|
||||
whatsapp.net
|
||||
meta.com
|
||||
metacdn.com
|
||||
about.meta.com
|
||||
oculus.com
|
||||
threads.net
|
||||
31.13.24.0/21
|
||||
31.13.64.0/18
|
||||
66.220.144.0/20
|
||||
69.63.176.0/20
|
||||
69.171.224.0/19
|
||||
74.119.76.0/22
|
||||
103.4.96.0/22
|
||||
129.134.0.0/16
|
||||
157.240.0.0/16
|
||||
173.252.64.0/18
|
||||
179.60.192.0/22
|
||||
185.60.216.0/22
|
||||
204.15.20.0/22
|
||||
tiktok.com
|
||||
tiktokcdn.com
|
||||
tiktokcdn-us.com
|
||||
tiktokcdn-eu.com
|
||||
tiktokv.com
|
||||
tiktokv.us
|
||||
tiktokv.eu
|
||||
tiktokw.us
|
||||
tiktoklinksafety.us
|
||||
musical.ly
|
||||
byteoversea.com
|
||||
ibyteimg.com
|
||||
accounts.facebook.com
|
||||
ads.facebook.com
|
||||
api.facebook.com
|
||||
apps.facebook.com
|
||||
badoo.com
|
||||
bluesky.app
|
||||
bsky.social
|
||||
business.facebook.com
|
||||
bytedance.com
|
||||
bytedance.net
|
||||
bytedanceapi.com
|
||||
47.246.0.0/16
|
||||
103.136.220.0/22
|
||||
161.117.0.0/16
|
||||
snapchat.com
|
||||
snap.com
|
||||
sc-cdn.net
|
||||
snapkit.com
|
||||
snapads.com
|
||||
34.192.0.0/10
|
||||
52.0.0.0/11
|
||||
linkedin.com
|
||||
licdn.com
|
||||
linkedin.cn
|
||||
13.107.0.0/16
|
||||
40.0.0.0/8
|
||||
52.96.0.0/12
|
||||
pinterest.com
|
||||
pinimg.com
|
||||
pinterestcdn.com
|
||||
199.96.56.0/21
|
||||
tumblr.com
|
||||
twitch.tv
|
||||
ttvnw.net
|
||||
jtvnw.net
|
||||
kick.com
|
||||
rumble.com
|
||||
vimeo.com
|
||||
vimeocdn.com
|
||||
mastodon.social
|
||||
truthsocial.com
|
||||
bluesky.app
|
||||
bsky.social
|
||||
parler.com
|
||||
weibo.com
|
||||
weibo.cn
|
||||
sinaimg.cn
|
||||
vk.com
|
||||
vk.ru
|
||||
vkuser.net
|
||||
vkcdn.net
|
||||
87.240.128.0/18
|
||||
95.213.0.0/16
|
||||
93.186.224.0/20
|
||||
telegram.org
|
||||
t.me
|
||||
telegram.me
|
||||
91.108.4.0/22
|
||||
91.108.8.0/22
|
||||
91.108.12.0/22
|
||||
91.108.16.0/22
|
||||
91.108.56.0/22
|
||||
149.154.160.0/20
|
||||
signal.org
|
||||
quora.com
|
||||
quoracdn.net
|
||||
medium.com
|
||||
byteoversea.com
|
||||
cdn-medium.com
|
||||
cdninstagram.com
|
||||
clubhouse.com
|
||||
clubhouseapi.com
|
||||
connect.facebook.com
|
||||
connect.facebook.net
|
||||
developers.facebook.com
|
||||
edge.facebook.com
|
||||
facebook.com
|
||||
facebook.net
|
||||
fb.com
|
||||
fb.gg
|
||||
fb.me
|
||||
fb.watch
|
||||
fbcdn.com
|
||||
fbcdn.net
|
||||
fbsbx.com
|
||||
flickr.com
|
||||
staticflickr.com
|
||||
graph.facebook.com
|
||||
ibyteimg.com
|
||||
imgur.com
|
||||
imgur.io
|
||||
imgurmedia.com
|
||||
badoo.com
|
||||
instagram.com
|
||||
jtvnw.net
|
||||
kick.com
|
||||
l.facebook.com
|
||||
licdn.com
|
||||
linkedin.cn
|
||||
linkedin.com
|
||||
lm.facebook.com
|
||||
login.facebook.com
|
||||
mastodon.social
|
||||
medium.com
|
||||
meetup.com
|
||||
messenger.com
|
||||
meta.com
|
||||
metacdn.com
|
||||
musical.ly
|
||||
nextdoor.com
|
||||
clubhouse.com
|
||||
clubhouseapi.com
|
||||
oculus.com
|
||||
parler.com
|
||||
pinimg.com
|
||||
pinterest.com
|
||||
pinterestcdn.com
|
||||
quora.com
|
||||
quoracdn.net
|
||||
rumble.com
|
||||
sc-cdn.net
|
||||
secure.facebook.com
|
||||
signal.org
|
||||
sinaimg.cn
|
||||
snap.com
|
||||
snapads.com
|
||||
snapchat.com
|
||||
snapkit.com
|
||||
static.ak.connect.facebook.com
|
||||
static.ak.facebook.com
|
||||
static.xx.fbcdn.net
|
||||
staticflickr.com
|
||||
t.me
|
||||
telegram.me
|
||||
telegram.org
|
||||
tfbnw.net
|
||||
threads.net
|
||||
tiktok.com
|
||||
tiktokcdn-eu.com
|
||||
tiktokcdn-us.com
|
||||
tiktokcdn.com
|
||||
tiktoklinksafety.us
|
||||
tiktokv.com
|
||||
tiktokv.eu
|
||||
tiktokv.us
|
||||
tiktokw.us
|
||||
truthsocial.com
|
||||
ttvnw.net
|
||||
tumblr.com
|
||||
twitch.tv
|
||||
upload.facebook.com
|
||||
video.xx.fbcdn.net
|
||||
vimeo.com
|
||||
vimeocdn.com
|
||||
vk.com
|
||||
vk.ru
|
||||
vkcdn.net
|
||||
vkuser.net
|
||||
weibo.cn
|
||||
weibo.com
|
||||
whatsapp.com
|
||||
whatsapp.net
|
||||
|
||||
# IP Ranges / CIDRs
|
||||
103.136.220.0/22
|
||||
103.4.96.0/22
|
||||
129.134.0.0/16
|
||||
13.107.0.0/16
|
||||
149.154.160.0/20
|
||||
157.240.0.0/16
|
||||
161.117.0.0/16
|
||||
173.252.64.0/18
|
||||
179.60.192.0/22
|
||||
185.60.216.0/22
|
||||
199.96.56.0/21
|
||||
204.15.20.0/22
|
||||
31.13.24.0/21
|
||||
31.13.64.0/18
|
||||
34.192.0.0/10
|
||||
40.0.0.0/8
|
||||
47.246.0.0/16
|
||||
52.0.0.0/11
|
||||
52.96.0.0/12
|
||||
66.220.144.0/20
|
||||
69.171.224.0/19
|
||||
69.63.176.0/20
|
||||
74.119.76.0/22
|
||||
87.240.128.0/18
|
||||
91.108.12.0/22
|
||||
91.108.16.0/22
|
||||
91.108.4.0/22
|
||||
91.108.56.0/22
|
||||
91.108.8.0/22
|
||||
93.186.224.0/20
|
||||
95.213.0.0/16
|
||||
@@ -1,198 +1,222 @@
|
||||
# --- added domains from research ---
|
||||
# amazon prime video (additional)
|
||||
# apple tv+ (rebranded from apple tv+)
|
||||
# crunchyroll (additional)
|
||||
# duckduckgo video search
|
||||
# general streaming protocols
|
||||
# paramount+ (additional)
|
||||
# peacock (additional domains from filterlist)
|
||||
# roku
|
||||
# youtube (additional - common streaming)
|
||||
acorn.tv
|
||||
aiv-cdn.net
|
||||
akamaized.net
|
||||
amazon.com
|
||||
amazonaws.com
|
||||
amp.twimg.com
|
||||
api-manga.crunchyroll.com
|
||||
api.crunchyroll.com
|
||||
api.roku.com
|
||||
api.rokutime.com
|
||||
api.rpay.roku.com
|
||||
api.spotify.com
|
||||
api2.sr.roku.com
|
||||
apis.rpay.roku.com
|
||||
apple-dns.net
|
||||
apple-tv-plus.com
|
||||
apple.com
|
||||
apple.com.edgekey.net
|
||||
atom.peacocktv.com
|
||||
atv-ext.amazon.com
|
||||
availability-fastly.syncbak-mediastore-cedexis.cbsaavideo.com
|
||||
bakery.pplus.paramount.tech
|
||||
bam.nr-data.net
|
||||
bamgrid.com
|
||||
beta-api.crunchyroll.com
|
||||
beta-stage-api.crunchyroll.com
|
||||
bff-ext.clients.peacocktv.com
|
||||
bookmarks.sr.roku.com
|
||||
britbox.com
|
||||
cbs.com
|
||||
cbsaavideo.com
|
||||
cbsi.demdex.net
|
||||
cbsi.live.ott.irdeto.com
|
||||
cbsinteractive.hb.omtrdc.net
|
||||
cbsivideo.com
|
||||
cdn.peacocktv.com
|
||||
cdn.privacy.paramount.com
|
||||
cdn.registerdisney.go.com
|
||||
cdn.spotify.com
|
||||
cloudfront.net
|
||||
content-detail.sr.roku.com
|
||||
content.sr.roku.com
|
||||
conviva.com
|
||||
crunchyroll.com
|
||||
cybertron.id.peacocktv.com
|
||||
d1v0o82c0gfg2u.cloudfront.net
|
||||
d2cvj4cr2d2r7g.cloudfront.net
|
||||
d3n92br3b4jq4j.cloudfront.net
|
||||
deezer.com
|
||||
dev.crunchyroll.com
|
||||
discoveryplus.com
|
||||
disney.com
|
||||
disneyplus.com
|
||||
disneystreaming.com
|
||||
doppler.hulu.com
|
||||
dplay.com
|
||||
dssott.com
|
||||
duck.com
|
||||
duckduckgo.com
|
||||
e.delaymultiplier
|
||||
e.retry
|
||||
edgesuite.net
|
||||
enduser.adsrv.org
|
||||
espn.com
|
||||
espn.net
|
||||
espncdn.com
|
||||
fox.com
|
||||
foxsports.com
|
||||
freevee.com
|
||||
fubo.tv
|
||||
funimation.com
|
||||
fwmrm.net
|
||||
googlevideo.com
|
||||
hbo.com
|
||||
hbomax.com
|
||||
hulu.com
|
||||
hulu.hb.omtrdc.net
|
||||
huluad.com
|
||||
huluim.com
|
||||
hulustream.com
|
||||
image.roku.com
|
||||
images-svc.sr.roku.com
|
||||
imageservice.disco.peacocktv.com
|
||||
imasdk.googleapis.com
|
||||
imdb.com
|
||||
imdbtv.com
|
||||
ingest.webcol.sdp.peacocktv.com
|
||||
innovidvideo.com
|
||||
llnwd.net
|
||||
max.com
|
||||
meg.disco.peacocktv.com
|
||||
mlb.com
|
||||
mlbstatic.com
|
||||
moviefone.com
|
||||
mparticle.com
|
||||
mubi.com
|
||||
mytv.clients.peacocktv.com
|
||||
mzstatic.com
|
||||
n.account
|
||||
n.assets
|
||||
n.auth
|
||||
n.content
|
||||
n.devices
|
||||
n.etpconfig
|
||||
n.funmigrator
|
||||
n.incentives
|
||||
n.localization
|
||||
n.messaging
|
||||
n.news
|
||||
n.playservice
|
||||
n.reviews
|
||||
n.subscription
|
||||
n.talkbox
|
||||
n.thirdpartyoauth
|
||||
navigation.sr.roku.com
|
||||
nba.com
|
||||
nbc.com
|
||||
nbcucdn.com
|
||||
nbcuni.com
|
||||
netflix.com
|
||||
netflix.net
|
||||
netflixdnstest0.com
|
||||
nfl.com
|
||||
nflxext.com
|
||||
nflximg.com
|
||||
nflxso.net
|
||||
nflxvideo.net
|
||||
netflix.net
|
||||
netflixdnstest0.com
|
||||
31.13.64.0/18
|
||||
45.57.0.0/17
|
||||
64.120.128.0/17
|
||||
66.197.128.0/17
|
||||
69.53.224.0/19
|
||||
nhl.com
|
||||
nhlstatic.com
|
||||
omtrdc.net
|
||||
open.spotify.com
|
||||
ovp.peacocktv.com
|
||||
pagead2.googlesyndication.com
|
||||
pandora.com
|
||||
paramount.tech
|
||||
paramountplus.com
|
||||
pconfig-prd.cdn.peacocktv.com
|
||||
peacocktv.com
|
||||
philo.com
|
||||
play.ovp.peacocktv.com
|
||||
playback-detail.sr.roku.com
|
||||
pluto.tv
|
||||
primevideo.com
|
||||
retail.rpay.roku.com
|
||||
rights-manager.sr.roku.com
|
||||
roku.admeasurement.com
|
||||
roku.com
|
||||
rokutime.com
|
||||
s3.amazonaws.com
|
||||
saa.cbsi.com
|
||||
saa.paramountplus.com
|
||||
scdn.co
|
||||
scorecardresearch.com
|
||||
shudder.com
|
||||
sling.com
|
||||
soundcloud.com
|
||||
spotify.com
|
||||
ssai.peacocktv.com
|
||||
stage-api-manga.crunchyroll.com
|
||||
stage-api.crunchyroll.com
|
||||
static-delivery.sr.roku.com
|
||||
telemetry.nbcuott.com
|
||||
therokuchannel.roku.com
|
||||
this.audiolanguage
|
||||
this.country
|
||||
this.intercept
|
||||
this.locale
|
||||
this.settabid
|
||||
this.tabid
|
||||
thumbnails.cbsig.net
|
||||
tidal.com
|
||||
tubitv.com
|
||||
tv.apple.com
|
||||
viacomcbs.com
|
||||
vod-gcs-cedexis.cbsaavideo.com
|
||||
vod-gcs-google.cbsaavideo.com
|
||||
vod-gcs-qwilt.cbsaavideo.com
|
||||
vod-playlist.sr.roku.com
|
||||
vod.delivery.roku.com
|
||||
vortex.hulu.com
|
||||
vrv.co
|
||||
vudu.com
|
||||
warnermediacdn.com
|
||||
wbdstreaming.com
|
||||
wmcdp.com
|
||||
wmcdp.io
|
||||
wv-license.sr.roku.com
|
||||
www.crunchyroll.com
|
||||
www.googletagmanager.com
|
||||
wwwimage-secure.cbsstatic.com
|
||||
wwwimage-us.pplusstatic.com
|
||||
xumo.com
|
||||
youtu.be
|
||||
youtube.com
|
||||
ytimg.com
|
||||
|
||||
# IP Ranges / CIDRs
|
||||
108.175.32.0/20
|
||||
17.0.0.0/8
|
||||
192.173.64.0/18
|
||||
198.38.96.0/19
|
||||
198.45.48.0/20
|
||||
207.45.72.0/22
|
||||
208.75.76.0/22
|
||||
hbomax.com
|
||||
hbo.com
|
||||
warnermediacdn.com
|
||||
wbdstreaming.com
|
||||
max.com
|
||||
wmcdp.io
|
||||
wmcdp.com
|
||||
34.192.0.0/10
|
||||
52.0.0.0/11
|
||||
hulu.com
|
||||
hulustream.com
|
||||
huluim.com
|
||||
huluad.com
|
||||
vortex.hulu.com
|
||||
doppler.hulu.com
|
||||
conviva.com
|
||||
hulu.hb.omtrdc.net
|
||||
disneyplus.com
|
||||
disney.com
|
||||
bamgrid.com
|
||||
bam.nr-data.net
|
||||
cdn.registerdisney.go.com
|
||||
disneystreaming.com
|
||||
dssott.com
|
||||
paramountplus.com
|
||||
cbsivideo.com
|
||||
cbsaavideo.com
|
||||
cbs.com
|
||||
viacomcbs.com
|
||||
paramount.tech
|
||||
wwwimage-us.pplusstatic.com
|
||||
wwwimage-secure.cbsstatic.com
|
||||
thumbnails.cbsig.net
|
||||
bakery.pplus.paramount.tech
|
||||
cbsi.demdex.net
|
||||
vod-gcs-cedexis.cbsaavideo.com
|
||||
cbsinteractive.hb.omtrdc.net
|
||||
cbsi.live.ott.irdeto.com
|
||||
saa.paramountplus.com
|
||||
saa.cbsi.com
|
||||
peacocktv.com
|
||||
nbc.com
|
||||
nbcuni.com
|
||||
nbcucdn.com
|
||||
telemetry.nbcuott.com
|
||||
tubitv.com
|
||||
fox.com
|
||||
foxsports.com
|
||||
fubo.tv
|
||||
pluto.tv
|
||||
roku.com
|
||||
rokutime.com
|
||||
therokuchannel.roku.com
|
||||
api.roku.com
|
||||
api.rokutime.com
|
||||
roku.admeasurement.com
|
||||
crunchyroll.com
|
||||
beta-api.crunchyroll.com
|
||||
www.crunchyroll.com
|
||||
vrv.co
|
||||
funimation.com
|
||||
aiv-cdn.net
|
||||
primevideo.com
|
||||
amazon.com
|
||||
amazonaws.com
|
||||
cloudfront.net
|
||||
s3.amazonaws.com
|
||||
atv-ext.amazon.com
|
||||
imdb.com
|
||||
imdbtv.com
|
||||
freevee.com
|
||||
apple.com
|
||||
apple-dns.net
|
||||
apple.com.edgekey.net
|
||||
mzstatic.com
|
||||
apple-tv-plus.com
|
||||
tv.apple.com
|
||||
17.0.0.0/8
|
||||
224.0.0.0/4
|
||||
vudu.com
|
||||
moviefone.com
|
||||
mubi.com
|
||||
shudder.com
|
||||
britbox.com
|
||||
acorn.tv
|
||||
sling.com
|
||||
philo.com
|
||||
xumo.com
|
||||
discoveryplus.com
|
||||
dplay.com
|
||||
espn.com
|
||||
espncdn.com
|
||||
espn.net
|
||||
mlb.com
|
||||
mlbstatic.com
|
||||
nba.com
|
||||
nfl.com
|
||||
nhl.com
|
||||
nhlstatic.com
|
||||
spotify.com
|
||||
scdn.co
|
||||
api.spotify.com
|
||||
cdn.spotify.com
|
||||
open.spotify.com
|
||||
pandora.com
|
||||
soundcloud.com
|
||||
deezer.com
|
||||
tidal.com
|
||||
# --- Added domains from research ---
|
||||
# Apple TV+ (rebranded from Apple TV+)
|
||||
tv.apple.com
|
||||
amp.twimg.com
|
||||
# Roku
|
||||
retail.rpay.roku.com
|
||||
api.rpay.roku.com
|
||||
image.roku.com
|
||||
vod.delivery.roku.com
|
||||
vod-playlist.sr.roku.com
|
||||
rights-manager.sr.roku.com
|
||||
wv-license.sr.roku.com
|
||||
static-delivery.sr.roku.com
|
||||
bookmarks.sr.roku.com
|
||||
navigation.sr.roku.com
|
||||
images-svc.sr.roku.com
|
||||
content.sr.roku.com
|
||||
content-detail.sr.roku.com
|
||||
playback-detail.sr.roku.com
|
||||
api2.sr.roku.com
|
||||
apis.rpay.roku.com
|
||||
# Crunchyroll (additional)
|
||||
api.crunchyroll.com
|
||||
api-manga.crunchyroll.com
|
||||
stage-api.crunchyroll.com
|
||||
stage-api-manga.crunchyroll.com
|
||||
beta-stage-api.crunchyroll.com
|
||||
dev.crunchyroll.com
|
||||
# Paramount+ (additional)
|
||||
imasdk.googleapis.com
|
||||
enduser.adsrv.org
|
||||
cdn.privacy.paramount.com
|
||||
www.googletagmanager.com
|
||||
pagead2.googlesyndication.com
|
||||
availability-fastly.syncbak-mediastore-cedexis.cbsaavideo.com
|
||||
vod-gcs-qwilt.cbsaavideo.com
|
||||
vod-gcs-google.cbsaavideo.com
|
||||
# Peacock (additional domains from filterlist)
|
||||
ingest.webcol.sdp.peacocktv.com
|
||||
mparticle.com
|
||||
fwmrm.net
|
||||
scorecardresearch.com
|
||||
omtrdc.net
|
||||
ssai.peacocktv.com
|
||||
mytv.clients.peacocktv.com
|
||||
bff-ext.clients.peacocktv.com
|
||||
imageservice.disco.peacocktv.com
|
||||
play.ovp.peacocktv.com
|
||||
atom.peacocktv.com
|
||||
cybertron.id.peacocktv.com
|
||||
meg.disco.peacocktv.com
|
||||
ovp.peacocktv.com
|
||||
pconfig-prd.cdn.peacocktv.com
|
||||
cdn.peacocktv.com
|
||||
# Amazon Prime Video (additional)
|
||||
aiv-cdn.net
|
||||
d1v0o82c0gfg2u.cloudfront.net
|
||||
d2cvj4cr2d2r7g.cloudfront.net
|
||||
d3n92br3b4jq4j.cloudfront.net
|
||||
# YouTube (additional - common streaming)
|
||||
youtube.com
|
||||
googlevideo.com
|
||||
ytimg.com
|
||||
youtu.be
|
||||
innovidvideo.com
|
||||
# General streaming protocols
|
||||
akamaized.net
|
||||
edgesuite.net
|
||||
llnwd.net
|
||||
# DuckDuckGo video search
|
||||
duckduckgo.com
|
||||
duck.com
|
||||
31.13.64.0/18
|
||||
34.192.0.0/10
|
||||
45.57.0.0/17
|
||||
52.0.0.0/11
|
||||
64.120.128.0/17
|
||||
66.197.128.0/17
|
||||
69.53.224.0/19
|
||||
@@ -0,0 +1,246 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Update domain lists by merging new scraped domains with existing file contents.
|
||||
Never deletes existing domains - only appends.
|
||||
|
||||
Usage:
|
||||
python3 update_domains.py [--streaming] [--social] [--bypass] [--porn] [--bamboo] [--all]
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import argparse
|
||||
import re
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from datetime import datetime
|
||||
|
||||
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
PARENT_DIR = os.path.dirname(BASE_DIR)
|
||||
BACKUP_DIR = os.path.join(os.environ.get("BACKUP_DIR", os.path.expanduser("~/.hermes/backups/domains_backup")))
|
||||
|
||||
STREAMING_FILE = os.path.join(PARENT_DIR, "streaming_domains_whitelist.txt")
|
||||
SOCIAL_FILE = os.path.join(PARENT_DIR, "social_media.txt")
|
||||
BAMBOO_FILE = os.path.join(PARENT_DIR, "bamboo_domains.txt")
|
||||
BYPASS_FILE = os.path.join(PARENT_DIR, "bypass_sites.txt")
|
||||
PORN_FILE = os.path.join(PARENT_DIR, "known_porn_domains.txt")
|
||||
|
||||
STREAMING_SOURCES = {
|
||||
"Peacock Filterlist": "https://raw.githubusercontent.com/lit-bg/Peacock/refs/heads/main/filterlist.txt",
|
||||
"Crunchyroll Endpoints": "https://gist.githubusercontent.com/NicmeisteR/cdc4867cf256c568b6a7f1844ce229f2/raw",
|
||||
}
|
||||
|
||||
SOCIAL_SOURCES = {
|
||||
"PiHole Streaming List": "https://raw.githubusercontent.com/ozankiratli/801ba17705e7f2a904d2e443af5a64f8/raw/436fa9f0c151afc15a601edf77c796b2a6de9be4/PiHoleStreamingLists.md",
|
||||
}
|
||||
|
||||
|
||||
def fetch_text(url):
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
return resp.read().decode("utf-8", errors="replace")
|
||||
except Exception as e:
|
||||
print(f"WARN: fetch failed for {url}: {e}")
|
||||
return ""
|
||||
|
||||
|
||||
def backup_file(path, label):
|
||||
if not os.path.exists(path):
|
||||
return
|
||||
os.makedirs(BACKUP_DIR, exist_ok=True)
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
backup_path = os.path.join(BACKUP_DIR, f"{label}.{timestamp}")
|
||||
shutil.copy2(path, backup_path)
|
||||
print(f"Backup saved to {backup_path}")
|
||||
|
||||
|
||||
# Strict domain validation: must match valid domain pattern
|
||||
DOMAIN_RE = re.compile(
|
||||
r'^(?=.{1,253}$)'
|
||||
r'([a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+' # labels
|
||||
r'[a-zA-Z]{2,63}$'
|
||||
)
|
||||
|
||||
def is_valid_domain(line):
|
||||
"""Validate a line is a proper domain (not code, IPs, or junk)"""
|
||||
if not line or len(line) < 4 or len(line) > 253:
|
||||
return False
|
||||
if line.startswith("#"):
|
||||
return False
|
||||
if re.match(r'^(\d{1,3}\.){3}\d{1,3}', line):
|
||||
return False
|
||||
if "/" in line and re.search(r'\d+\.\d+\.\d+\.\d+/\d+', line):
|
||||
return False
|
||||
# Reject anything that doesn't look like a domain
|
||||
if not DOMAIN_RE.match(line):
|
||||
return False
|
||||
# Reject known junk patterns from JS/CSS
|
||||
junk_keywords = ["const", "var ", "let ", "function", "return", "if(", "else",
|
||||
".catch", ".then", ".map(", ".filter(", ".reduce(", "import", "export",
|
||||
"class", "extends", "=>", "=>" ]
|
||||
if any(kw in line for kw in junk_keywords):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def is_ip_or_cidr(line):
|
||||
ip_cidr_re = re.compile(r'^(\d{1,3}\.){3}\d{1,3}(/\d+)?$')
|
||||
return bool(ip_cidr_re.match(line))
|
||||
|
||||
|
||||
def load_existing(path):
|
||||
"""Load existing domains and IPs, preserving EVERYTHING (never delete)."""
|
||||
domains = set()
|
||||
ips = set()
|
||||
if not os.path.exists(path):
|
||||
return domains, ips
|
||||
with open(path, "r") as f:
|
||||
for line in f:
|
||||
line = line.strip().lower()
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith("#"):
|
||||
# Preserve comments
|
||||
domains.add(line)
|
||||
continue
|
||||
if is_ip_or_cidr(line):
|
||||
ips.add(line)
|
||||
else:
|
||||
# Preserve EVERYTHING else - even junk lines, per user requirement "never delete"
|
||||
domains.add(line)
|
||||
return domains, ips
|
||||
|
||||
|
||||
def extract_domains(text):
|
||||
domains = set()
|
||||
ips = set()
|
||||
for line in text.splitlines():
|
||||
line = line.strip().lower()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
# Handle AdBlock-style ||domain^ format
|
||||
if line.startswith("||"):
|
||||
clean = line[2:].split("^")[0].strip()
|
||||
else:
|
||||
clean = line.replace("|", "").replace("^", "").replace("$", "").strip()
|
||||
if " " in clean:
|
||||
clean = clean.split()[0]
|
||||
clean = clean.lstrip("*")
|
||||
if not clean:
|
||||
continue
|
||||
if is_ip_or_cidr(clean):
|
||||
ips.add(clean)
|
||||
elif is_valid_domain(clean):
|
||||
domains.add(clean)
|
||||
return domains, ips
|
||||
|
||||
|
||||
def save_domains(path, domains, ips):
|
||||
"""Save domains preserving original order of comments/sections"""
|
||||
sorted_domains = sorted(d for d in domains if not is_ip_or_cidr(d))
|
||||
sorted_ips = sorted(set(ips))
|
||||
|
||||
with open(path, "w") as f:
|
||||
for d in sorted_domains:
|
||||
f.write(d + "\n")
|
||||
if sorted_ips:
|
||||
f.write("\n# IP Ranges / CIDRs\n")
|
||||
for ip in sorted_ips:
|
||||
f.write(ip + "\n")
|
||||
|
||||
|
||||
def update_streaming():
|
||||
existing_domains, existing_ips = load_existing(STREAMING_FILE)
|
||||
all_domains = set(existing_domains)
|
||||
all_ips = set(existing_ips)
|
||||
|
||||
for name, url in STREAMING_SOURCES.items():
|
||||
print(f"Fetching {name}...")
|
||||
text = fetch_text(url)
|
||||
if text:
|
||||
scraped_domains, scraped_ips = extract_domains(text)
|
||||
added = len(scraped_domains)
|
||||
all_domains.update(scraped_domains)
|
||||
all_ips.update(scraped_ips)
|
||||
print(f" Scraped {added} new domains from {name}")
|
||||
|
||||
backup_file(STREAMING_FILE, "streaming_domains_whitelist.txt")
|
||||
save_domains(STREAMING_FILE, all_domains, all_ips)
|
||||
print(f"Streaming: {len(all_domains)} domains + {len(all_ips)} IPs total")
|
||||
|
||||
|
||||
def update_social():
|
||||
existing_domains, existing_ips = load_existing(SOCIAL_FILE)
|
||||
all_domains = set(existing_domains)
|
||||
all_ips = set(existing_ips)
|
||||
|
||||
for name, url in SOCIAL_SOURCES.items():
|
||||
print(f"Fetching {name}...")
|
||||
text = fetch_text(url)
|
||||
if text:
|
||||
scraped_domains, scraped_ips = extract_domains(text)
|
||||
social_keywords = ["facebook", "instagram", "twitter", "tiktok", "linkedin",
|
||||
"snapchat", "threads", "mastodon", "reddit", "discord",
|
||||
"pinterest", "youtube", "twitch", "vk", "tumblr",
|
||||
"medium", "clubhouse", "messenger", "telegram", "quora",
|
||||
"imgur", "vimeo", "bluesky", "bsky"]
|
||||
added = 0
|
||||
for d in scraped_domains:
|
||||
if any(kw in d for kw in social_keywords):
|
||||
all_domains.add(d)
|
||||
added += 1
|
||||
print(f" Scraped {added} new social domains from {name}")
|
||||
|
||||
backup_file(SOCIAL_FILE, "social_media.txt")
|
||||
save_domains(SOCIAL_FILE, all_domains, all_ips)
|
||||
print(f"Social media: {len(all_domains)} domains + {len(all_ips)} IPs total")
|
||||
|
||||
|
||||
def update_bypass():
|
||||
streaming_domains, streaming_ips = load_existing(STREAMING_FILE)
|
||||
social_domains, social_ips = load_existing(SOCIAL_FILE)
|
||||
all_domains = streaming_domains | social_domains
|
||||
all_ips = streaming_ips | social_ips
|
||||
backup_file(BYPASS_FILE, "bypass_sites.txt")
|
||||
save_domains(BYPASS_FILE, all_domains, all_ips)
|
||||
print(f"Bypass sites: {len(all_domains)} domains + {len(all_ips)} IPs total")
|
||||
|
||||
|
||||
def update_porn():
|
||||
domains, ips = load_existing(PORN_FILE)
|
||||
backup_file(PORN_FILE, "known_porn_domains.txt")
|
||||
save_domains(PORN_FILE, domains, ips)
|
||||
print(f"Porn domains preserved: {len(domains)} domains + {len(ips)} IPs")
|
||||
|
||||
|
||||
def update_bamboo():
|
||||
domains, ips = load_existing(BAMBOO_FILE)
|
||||
print(f"Bamboo domains preserved: {len(domains)} domains + {len(ips)} IPs")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Update domain lists - never delete!")
|
||||
parser.add_argument("--streaming", action="store_true", help="Update streaming only")
|
||||
parser.add_argument("--social", action="store_true", help="Update social media only")
|
||||
parser.add_argument("--bypass", action="store_true", help="Update bypass_sites (combines streaming+social)")
|
||||
parser.add_argument("--porn", action="store_true", help="Update porn domains")
|
||||
parser.add_argument("--bamboo", action="store_true", help="Update bamboo domains")
|
||||
parser.add_argument("--all", action="store_true", help="Update all lists")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.all or not any([args.streaming, args.social, args.bypass, args.porn, args.bamboo]):
|
||||
update_streaming()
|
||||
update_social()
|
||||
update_bypass()
|
||||
update_porn()
|
||||
update_bamboo()
|
||||
else:
|
||||
if args.streaming: update_streaming()
|
||||
if args.social: update_social()
|
||||
if args.bypass: update_bypass()
|
||||
if args.porn: update_porn()
|
||||
if args.bamboo: update_bamboo()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,107 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Update streaming_domains_whitelist.txt with domains scraped from known sources.
|
||||
Always backs up the previous file before overwriting.
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from datetime import datetime
|
||||
|
||||
DOMAINS_FILE = os.path.join(os.path.dirname(__file__), "..", "streaming_domains_whitelist.txt")
|
||||
BACKUP_DIR = os.path.join(os.environ.get("HOME", "/tmp"), ".hermes", "backups", "domains_backup")
|
||||
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
PARENT_DIR = os.path.dirname(BASE_DIR)
|
||||
DOMAINS_FILE = os.path.join(PARENT_DIR, "streaming_domains_whitelist.txt")
|
||||
|
||||
SOURCES = {
|
||||
"Netflix": "https://raw.githubusercontent.com/wickedyoda/public-setupfiles/main/domains/streaming_domains_whitelist.txt",
|
||||
"Crunchyroll": "https://gist.githubusercontent.com/NicmeisteR/cdc4867cf256c568b6a7f1844ce229f2/raw",
|
||||
"Peacock": "https://raw.githubusercontent.com/lit-bg/Peacock/refs/heads/main/filterlist.txt",
|
||||
"PiHole Streaming List": "https://raw.githubusercontent.com/ozankiratli/801ba17705e7f2a904d2e443af5a64f8/raw/436fa9f0c151afc15a601edf77c796b2a6de9be4/PiHoleStreamingLists.md",
|
||||
}
|
||||
|
||||
def fetch_text(url):
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
return resp.read().decode("utf-8", errors="replace")
|
||||
except (urllib.error.URLError, Exception) as e:
|
||||
print(f"WARN: fetch failed for {url}: {e}")
|
||||
return ""
|
||||
|
||||
def backup_file(path):
|
||||
os.makedirs(BACKUP_DIR, exist_ok=True)
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
backup_path = os.path.join(BACKUP_DIR, f"streaming_domains_whitelist.txt.{timestamp}")
|
||||
shutil.copy2(path, backup_path)
|
||||
print(f"Backup saved to {backup_path}")
|
||||
|
||||
def main():
|
||||
domains = set()
|
||||
comments = []
|
||||
|
||||
for name, url in SOURCES.items():
|
||||
print(f"Fetching {name}...")
|
||||
text = fetch_text(url)
|
||||
if not text:
|
||||
continue
|
||||
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
# Extract domain-like tokens
|
||||
parts = line.replace("^", "").replace("|", "").replace("/", "").split()
|
||||
for part in parts:
|
||||
if "." in part and "/" not in part and " " not in part:
|
||||
clean = part.lstrip("*")
|
||||
if clean and not clean[0].isdigit:
|
||||
domains.add(clean)
|
||||
|
||||
# Hardcoded core domains to always keep
|
||||
core_domains = [
|
||||
"netflix.com", "nflxext.com", "nflximg.com", "nflxso.net", "nflxvideo.net",
|
||||
"netflix.net", "hbomax.com", "hbo.com", "warnermediacdn.com", "wbdstreaming.com",
|
||||
"max.com", "wmcdp.io", "wmcdp.com", "hulu.com", "hulustream.com", "huluim.com",
|
||||
"huluad.com", "vortex.hulu.com", "doppler.hulu.com", "conviva.com",
|
||||
"disneyplus.com", "disney.com", "bamgrid.com", "disneystreaming.com", "dssott.com",
|
||||
"paramountplus.com", "cbsivideo.com", "cbsaavideo.com", "cbs.com", "viacomcbs.com",
|
||||
"peacocktv.com", "nbc.com", "nbcuni.com", "nbcucdn.com",
|
||||
"tubitv.com", "fox.com", "foxsports.com", "fubo.tv", "pluto.tv",
|
||||
"roku.com", "rokutime.com", "therokuchannel.roku.com",
|
||||
"crunchyroll.com", "vrv.co", "funimation.com",
|
||||
"aiv-cdn.net", "primevideo.com", "amazon.com", "amazonaws.com",
|
||||
"cloudfront.net", "s3.amazonaws.com", "atv-ext.amazon.com",
|
||||
"imdb.com", "imdbtv.com", "freevee.com",
|
||||
"apple.com", "apple-dns.net", "mzstatic.com",
|
||||
"apple-tv-plus.com", "tv.apple.com",
|
||||
"vudu.com", "moviefone.com", "mubi.com", "shudder.com",
|
||||
"britbox.com", "acorn.tv", "sling.com", "philo.com", "xumo.com",
|
||||
"discoveryplus.com", "dplay.com",
|
||||
"espn.com", "espncdn.com", "espn.net",
|
||||
"mlb.com", "mlbstatic.com", "nba.com", "nfl.com", "nhl.com", "nhlstatic.com",
|
||||
"spotify.com", "scdn.co", "pandora.com", "soundcloud.com",
|
||||
"deezer.com", "tidal.com",
|
||||
]
|
||||
domains.update(core_domains)
|
||||
|
||||
# Always back up before writing
|
||||
if os.path.exists(DOMAINS_FILE):
|
||||
backup_file(DOMAINS_FILE)
|
||||
|
||||
sorted_domains = sorted(d for d in domains if not d.startswith("1") and not d.startswith("2") and not d.startswith("3") and not d.startswith("4") and not d.startswith("5") and not d.startswith("6") and not d.startswith("7") and not d.startswith("8") and not d.startswith("9"))
|
||||
ip_cidrs = sorted(d for d in domains if any(c.isdigit() for c in d))
|
||||
|
||||
with open(DOMAINS_FILE, "w") as f:
|
||||
for domain in sorted_domains:
|
||||
f.write(domain + "\n")
|
||||
f.write("\n# IP Ranges / CIDRs\n")
|
||||
for ip in ip_cidrs:
|
||||
f.write(ip + "\n")
|
||||
|
||||
print(f"Domains updated: {len(sorted_domains)} domains + {len(ip_cidrs)} IPs")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user