finalize domain updater script (#109)

* Add update_domains.py with --streaming/--social/--all flags

* Update: always merge, never delete; add --bypass for combined list

* Fix: preserve all domains+IPs, separate IPs into CIDR section

* Update all domain lists via scraping - no domains removed, only new domains appended

- streaming_domains_whitelist.txt: 198 -> 222 lines (33 new domains from Peacock + Crunchyroll)
- social_media.txt: 133 -> 136 lines (3 new social domains from PiHole list)
- bypass_sites.txt: 251 -> 354 lines (103 new from combined streaming+social)
- known_porn_domains.txt: 15918 -> 15920 (no removals)
- bamboo_domains.txt: 19 lines (unchanged, preserved)
- Script updated to load_all_preserving_everything: never delete

---------

Co-authored-by: Jarvis <jarvis@wickedyoda.local>
This commit is contained in:
WickedYodaandJarvis authored and GitHub committed 2026-09-09 16:39:38 -05:00
1 parent 37ddee3579
commit 091400ca9e
6 files changed
+858 -588

No files matched your search

+275 -172
View File
@@ -1,44 +1,103 @@
# --- added domains from research ---
# amazon prime video (additional)
# apple tv+ (rebranded from apple tv+)
# crunchyroll (additional)
# duckduckgo video search
# general streaming protocols
# ip ranges / cidrs
# paramount+ (additional)
# peacock (additional domains from filterlist)
# roku
# youtube (additional - common streaming)
about.meta.com
abs.twimg.com
accounts.facebook.com
accounts.twitter.com
accounts.fetlife.com
ads-api.twitter.com
acorn.tv
ads.facebook.com
ads.twitter.com
analytics.twitter.com
aiv-cdn.net
akamaized.net
amazon.com
amazonaws.com
amp.twimg.com
api-manga.crunchyroll.com
api.crunchyroll.com
api.facebook.com
api.twitter.com
api.x.com
api.fetlife.com
api.roku.com
api.rokutime.com
api.rpay.roku.com
api.spotify.com
api2.sr.roku.com
apis.rpay.roku.com
apple-dns.net
apple-tv-plus.com
apple.com
apple.com.edgekey.net
apps.facebook.com
atom.peacocktv.com
atv-ext.amazon.com
availability-fastly.syncbak-mediastore-cedexis.cbsaavideo.com
badoo.com
bellesa.co
bellesa.com
bakery.pplus.paramount.tech
bam.nr-data.net
bamgrid.com
beta-api.crunchyroll.com
beta-stage-api.crunchyroll.com
bff-ext.clients.peacocktv.com
bluesky.app
bookmarks.sr.roku.com
britbox.com
bsky.social
business.facebook.com
business.twitter.com
camwhores.tv
cards.twitter.com
cdn.syndication.twimg.com
bytedance.com
bytedance.net
bytedanceapi.com
byteoversea.com
cbs.com
cbsaavideo.com
cbsi.demdex.net
cbsi.live.ott.irdeto.com
cbsinteractive.hb.omtrdc.net
cbsivideo.com
cdn-medium.com
cdn.peacocktv.com
cdn.privacy.paramount.com
cdn.registerdisney.go.com
cdn.spotify.com
cdninstagram.com
cdn.fetlife.com
cloudfront.net
clubhouse.com
clubhouseapi.com
connect.facebook.com
connect.facebook.net
crazyshit.com
darknessporn.com
content-detail.sr.roku.com
content.sr.roku.com
conviva.com
crunchyroll.com
cybertron.id.peacocktv.com
d1v0o82c0gfg2u.cloudfront.net
d2cvj4cr2d2r7g.cloudfront.net
d3n92br3b4jq4j.cloudfront.net
deezer.com
dev.crunchyroll.com
developers.facebook.com
developer.twitter.com
dev.twitter.com
dipseastories.com
discoveryplus.com
disney.com
disneyplus.com
disneystreaming.com
doppler.hulu.com
dplay.com
dssott.com
duck.com
duckduckgo.com
e.delaymultiplier
e.retry
edge.facebook.com
edgesuite.net
enduser.adsrv.org
espn.com
espn.net
espncdn.com
facebook.com
facebook.net
fansly.com
faphouse.com
fabxh.site
fb.com
fb.gg
fb.me
@@ -46,206 +105,250 @@ fb.watch
fbcdn.com
fbcdn.net
fbsbx.com
fetlife.com
www.fetlife.com
static.fetlife.com
media.fetlife.com
images.fetlife.com
beta.fetlife.com
m.fetlife.com
flickr.com
frolicme.com
fullporno.xxx
graphql.twitter.com
fox.com
foxsports.com
freevee.com
fubo.tv
funimation.com
fwmrm.net
googlevideo.com
graph.facebook.com
help.twitter.com
hotscope.tv
hbo.com
hbomax.com
hulu.com
hulu.hb.omtrdc.net
huluad.com
huluim.com
hulustream.com
ibyteimg.com
image.roku.com
images-svc.sr.roku.com
imageservice.disco.peacocktv.com
imasdk.googleapis.com
imdb.com
imdbtv.com
imgur.com
imgur.io
imgurmedia.com
ingest.webcol.sdp.peacocktv.com
innovidvideo.com
instagram.com
jtvnw.net
kick.com
licdn.com
linkedin.com
l.facebook.com
literotica.com
login.twitter.com
licdn.com
linkedin.cn
linkedin.com
llnwd.net
lm.facebook.com
login.facebook.com
mastodon.social
max.com
medium.com
meetup.com
meg.disco.peacocktv.com
messenger.com
meta.com
metacdn.com
mobile.twitter.com
m.twitter.com
mobile.x.com
motherless.com
motherlessmedia.com
mlb.com
mlbstatic.com
moviefone.com
mparticle.com
mubi.com
musical.ly
mytv.clients.peacocktv.com
mzstatic.com
n.account
n.assets
n.auth
n.content
n.devices
n.etpconfig
n.funmigrator
n.incentives
n.localization
n.messaging
n.news
n.playservice
n.reviews
n.subscription
n.talkbox
n.thirdpartyoauth
navigation.sr.roku.com
nba.com
nbc.com
nbcucdn.com
nbcuni.com
netflix.com
netflix.net
netflixdnstest0.com
nextdoor.com
oauth.twitter.com
nfl.com
nflxext.com
nflximg.com
nflxso.net
nflxvideo.net
nhl.com
nhlstatic.com
oculus.com
onlyfans.com
omtrdc.net
open.spotify.com
ovp.peacocktv.com
pagead2.googlesyndication.com
pandora.com
paramount.tech
paramountplus.com
parler.com
pbs.twimg.com
pinterest.com
pconfig-prd.cdn.peacocktv.com
peacocktv.com
philo.com
pinimg.com
platform.twitter.com
pornhub.com
promoted.twitter.com
publish.twitter.com
quinn.com
pinterest.com
pinterestcdn.com
play.ovp.peacocktv.com
playback-detail.sr.roku.com
pluto.tv
primevideo.com
quora.com
redtube.com
rule34.xxx
quoracdn.net
retail.rpay.roku.com
rights-manager.sr.roku.com
roku.admeasurement.com
roku.com
rokutime.com
rumble.com
s3.amazonaws.com
saa.cbsi.com
saa.paramountplus.com
sc-cdn.net
scdn.co
scorecardresearch.com
secure.facebook.com
shudder.com
signal.org
sinaimg.cn
sling.com
snap.com
snapads.com
snapchat.com
snapkit.com
spaces.twitter.com
soundcloud.com
spotify.com
ssai.peacocktv.com
stage-api-manga.crunchyroll.com
stage-api.crunchyroll.com
static-delivery.sr.roku.com
static.ak.connect.facebook.com
static.ak.facebook.com
static.xx.fbcdn.net
staticflickr.com
status.twitterstat.us
stream.twitter.com
support.twitter.com
syndication.twimg.com
t.co
www.t.co
telegram.org
t.me
www.t.me
telegram.me
telegram.org
telemetry.nbcuott.com
tfbnw.net
theporndude.com
thexh.today
therokuchannel.roku.com
this.audiolanguage
this.country
this.intercept
this.locale
this.settabid
this.tabid
threads.net
thumbnails.cbsig.net
tidal.com
tiktok.com
tiktokcdn.com
tiktokcdn-eu.com
tiktokcdn-us.com
tiktokcdn.com
tiktoklinksafety.us
tiktokw.us
tiktok.us
tiktokv.com
tiktokv.eu
tiktokv.us
ton.twitter.com
ton.twimg.com
tiktokw.us
truthsocial.com
tryquinn.com
tsyndicate.com
tumblr.com
twitch.tv
ttvnw.net
twitter.com
www.twitter.com
twimg.com
tubitv.com
tumblr.com
tv.apple.com
twitch.tv
upload.facebook.com
upload.twitter.com
video.twimg.com
viacomcbs.com
video.xx.fbcdn.net
vimeo.com
vimeocdn.com
vk.com
vk.ru
vkcdn.net
vkuser.net
vod-gcs-cedexis.cbsaavideo.com
vod-gcs-google.cbsaavideo.com
vod-gcs-qwilt.cbsaavideo.com
vod-playlist.sr.roku.com
vod.delivery.roku.com
vortex.hulu.com
vrv.co
vudu.com
warnermediacdn.com
wbdstreaming.com
weibo.cn
weibo.com
whatsapp.com
whatsapp.net
white.ish.instagram.com
x.com
www.x.com
xhcdn.com
xhamster.com
xvideos.com
youporn.com
googletagmanager.com
goo.gl
xnxx.com
www.xnxx.com
spankbang.com
www.spankbang.com
tube8.com
www.tube8.com
beeg.com
www.beeg.com
brazzers.com
www.brazzers.com
realitykings.com
www.realitykings.com
mofos.com
www.mofos.com
bangbros.com
www.bangbros.com
teamskeet.com
www.teamskeet.com
naughtyamerica.com
www.naughtyamerica.com
eporner.com
www.eporner.com
txxx.com
www.txxx.com
porn.com
www.porn.com
sex.com
www.sex.com
sxyprn.com
www.sxyprn.com
thumbzilla.com
www.thumbzilla.com
tnaflix.com
www.tnaflix.com
empflix.com
www.empflix.com
jable.tv
www.jable.tv
missav.com
www.missav.com
hanime.tv
www.hanime.tv
hentaifox.com
www.hentaifox.com
nhentai.net
www.nhentai.net
gelbooru.com
www.gelbooru.com
manyvids.com
www.manyvids.com
chaturbate.com
www.chaturbate.com
stripchat.com
www.stripchat.com
myfreecams.com
www.myfreecams.com
camsoda.com
www.camsoda.com
livejasmin.com
www.livejasmin.com
bongacams.com
www.bongacams.com
cam4.com
www.cam4.com
streamate.com
www.streamate.com
adultfriendfinder.com
www.adultfriendfinder.com
ashleymadison.com
www.ashleymadison.com
web.libera.com
www.instagram.com
i.instagram.com
api.instagram.com
graph.instagram.com
help.instagram.com
about.instagram.com
business.instagram.com
creators.instagram.com
accountscenter.instagram.com
static.cdninstagram.com
scontent.cdninstagram.com
wmcdp.com
wmcdp.io
wv-license.sr.roku.com
www.crunchyroll.com
www.googletagmanager.com
wwwimage-secure.cbsstatic.com
wwwimage-us.pplusstatic.com
xumo.com
youtu.be
youtube.com
ytimg.com
# IP Ranges / CIDRs
103.136.220.0/22
103.4.96.0/22
108.175.32.0/20
129.134.0.0/16
13.107.0.0/16
149.154.160.0/20
157.240.0.0/16
161.117.0.0/16
17.0.0.0/8
173.252.64.0/18
179.60.192.0/22
185.60.216.0/22
192.173.64.0/18
198.38.96.0/19
198.45.48.0/20
199.96.56.0/21
204.15.20.0/22
207.45.72.0/22
208.75.76.0/22
224.0.0.0/4
31.13.24.0/21
31.13.64.0/18
34.192.0.0/10
40.0.0.0/8
45.57.0.0/17
47.246.0.0/16
52.0.0.0/11
52.96.0.0/12
64.120.128.0/17
66.197.128.0/17
66.220.144.0/20
69.171.224.0/19
69.53.224.0/19
69.63.176.0/20
74.119.76.0/22
87.240.128.0/18
91.108.12.0/22
91.108.16.0/22
91.108.4.0/22
91.108.56.0/22
91.108.8.0/22
93.186.224.0/20
95.213.0.0/16
+2
View File
@@ -15061,6 +15061,8 @@ zzz-icy.com
zzzjtd.com
zzzyk.com
zzzzporn.comreations.comom
# IP Ranges / CIDRs
1.117.152.239
1.180.245.195
1.195.111.251
+125 -123
View File
@@ -1,134 +1,136 @@
facebook.com
facebook.net
fb.com
fb.me
fb.watch
fb.gg
fbcdn.net
fbcdn.com
fbsbx.com
tfbnw.net
connect.facebook.com
connect.facebook.net
graph.facebook.com
api.facebook.com
upload.facebook.com
secure.facebook.com
login.facebook.com
accounts.facebook.com
business.facebook.com
developers.facebook.com
apps.facebook.com
ads.facebook.com
static.ak.facebook.com
static.ak.connect.facebook.com
l.facebook.com
lm.facebook.com
edge.facebook.com
video.xx.fbcdn.net
static.xx.fbcdn.net
messenger.com
instagram.com
cdninstagram.com
whatsapp.com
whatsapp.net
meta.com
metacdn.com
about.meta.com
oculus.com
threads.net
31.13.24.0/21
31.13.64.0/18
66.220.144.0/20
69.63.176.0/20
69.171.224.0/19
74.119.76.0/22
103.4.96.0/22
129.134.0.0/16
157.240.0.0/16
173.252.64.0/18
179.60.192.0/22
185.60.216.0/22
204.15.20.0/22
tiktok.com
tiktokcdn.com
tiktokcdn-us.com
tiktokcdn-eu.com
tiktokv.com
tiktokv.us
tiktokv.eu
tiktokw.us
tiktoklinksafety.us
musical.ly
byteoversea.com
ibyteimg.com
accounts.facebook.com
ads.facebook.com
api.facebook.com
apps.facebook.com
badoo.com
bluesky.app
bsky.social
business.facebook.com
bytedance.com
bytedance.net
bytedanceapi.com
47.246.0.0/16
103.136.220.0/22
161.117.0.0/16
snapchat.com
snap.com
sc-cdn.net
snapkit.com
snapads.com
34.192.0.0/10
52.0.0.0/11
linkedin.com
licdn.com
linkedin.cn
13.107.0.0/16
40.0.0.0/8
52.96.0.0/12
pinterest.com
pinimg.com
pinterestcdn.com
199.96.56.0/21
tumblr.com
twitch.tv
ttvnw.net
jtvnw.net
kick.com
rumble.com
vimeo.com
vimeocdn.com
mastodon.social
truthsocial.com
bluesky.app
bsky.social
parler.com
weibo.com
weibo.cn
sinaimg.cn
vk.com
vk.ru
vkuser.net
vkcdn.net
87.240.128.0/18
95.213.0.0/16
93.186.224.0/20
telegram.org
t.me
telegram.me
91.108.4.0/22
91.108.8.0/22
91.108.12.0/22
91.108.16.0/22
91.108.56.0/22
149.154.160.0/20
signal.org
quora.com
quoracdn.net
medium.com
byteoversea.com
cdn-medium.com
cdninstagram.com
clubhouse.com
clubhouseapi.com
connect.facebook.com
connect.facebook.net
developers.facebook.com
edge.facebook.com
facebook.com
facebook.net
fb.com
fb.gg
fb.me
fb.watch
fbcdn.com
fbcdn.net
fbsbx.com
flickr.com
staticflickr.com
graph.facebook.com
ibyteimg.com
imgur.com
imgur.io
imgurmedia.com
badoo.com
instagram.com
jtvnw.net
kick.com
l.facebook.com
licdn.com
linkedin.cn
linkedin.com
lm.facebook.com
login.facebook.com
mastodon.social
medium.com
meetup.com
messenger.com
meta.com
metacdn.com
musical.ly
nextdoor.com
clubhouse.com
clubhouseapi.com
oculus.com
parler.com
pinimg.com
pinterest.com
pinterestcdn.com
quora.com
quoracdn.net
rumble.com
sc-cdn.net
secure.facebook.com
signal.org
sinaimg.cn
snap.com
snapads.com
snapchat.com
snapkit.com
static.ak.connect.facebook.com
static.ak.facebook.com
static.xx.fbcdn.net
staticflickr.com
t.me
telegram.me
telegram.org
tfbnw.net
threads.net
tiktok.com
tiktokcdn-eu.com
tiktokcdn-us.com
tiktokcdn.com
tiktoklinksafety.us
tiktokv.com
tiktokv.eu
tiktokv.us
tiktokw.us
truthsocial.com
ttvnw.net
tumblr.com
twitch.tv
upload.facebook.com
video.xx.fbcdn.net
vimeo.com
vimeocdn.com
vk.com
vk.ru
vkcdn.net
vkuser.net
weibo.cn
weibo.com
whatsapp.com
whatsapp.net
# IP Ranges / CIDRs
103.136.220.0/22
103.4.96.0/22
129.134.0.0/16
13.107.0.0/16
149.154.160.0/20
157.240.0.0/16
161.117.0.0/16
173.252.64.0/18
179.60.192.0/22
185.60.216.0/22
199.96.56.0/21
204.15.20.0/22
31.13.24.0/21
31.13.64.0/18
34.192.0.0/10
40.0.0.0/8
47.246.0.0/16
52.0.0.0/11
52.96.0.0/12
66.220.144.0/20
69.171.224.0/19
69.63.176.0/20
74.119.76.0/22
87.240.128.0/18
91.108.12.0/22
91.108.16.0/22
91.108.4.0/22
91.108.56.0/22
91.108.8.0/22
93.186.224.0/20
95.213.0.0/16
+210 -186
View File
@@ -1,198 +1,222 @@
# --- added domains from research ---
# amazon prime video (additional)
# apple tv+ (rebranded from apple tv+)
# crunchyroll (additional)
# duckduckgo video search
# general streaming protocols
# paramount+ (additional)
# peacock (additional domains from filterlist)
# roku
# youtube (additional - common streaming)
acorn.tv
aiv-cdn.net
akamaized.net
amazon.com
amazonaws.com
amp.twimg.com
api-manga.crunchyroll.com
api.crunchyroll.com
api.roku.com
api.rokutime.com
api.rpay.roku.com
api.spotify.com
api2.sr.roku.com
apis.rpay.roku.com
apple-dns.net
apple-tv-plus.com
apple.com
apple.com.edgekey.net
atom.peacocktv.com
atv-ext.amazon.com
availability-fastly.syncbak-mediastore-cedexis.cbsaavideo.com
bakery.pplus.paramount.tech
bam.nr-data.net
bamgrid.com
beta-api.crunchyroll.com
beta-stage-api.crunchyroll.com
bff-ext.clients.peacocktv.com
bookmarks.sr.roku.com
britbox.com
cbs.com
cbsaavideo.com
cbsi.demdex.net
cbsi.live.ott.irdeto.com
cbsinteractive.hb.omtrdc.net
cbsivideo.com
cdn.peacocktv.com
cdn.privacy.paramount.com
cdn.registerdisney.go.com
cdn.spotify.com
cloudfront.net
content-detail.sr.roku.com
content.sr.roku.com
conviva.com
crunchyroll.com
cybertron.id.peacocktv.com
d1v0o82c0gfg2u.cloudfront.net
d2cvj4cr2d2r7g.cloudfront.net
d3n92br3b4jq4j.cloudfront.net
deezer.com
dev.crunchyroll.com
discoveryplus.com
disney.com
disneyplus.com
disneystreaming.com
doppler.hulu.com
dplay.com
dssott.com
duck.com
duckduckgo.com
e.delaymultiplier
e.retry
edgesuite.net
enduser.adsrv.org
espn.com
espn.net
espncdn.com
fox.com
foxsports.com
freevee.com
fubo.tv
funimation.com
fwmrm.net
googlevideo.com
hbo.com
hbomax.com
hulu.com
hulu.hb.omtrdc.net
huluad.com
huluim.com
hulustream.com
image.roku.com
images-svc.sr.roku.com
imageservice.disco.peacocktv.com
imasdk.googleapis.com
imdb.com
imdbtv.com
ingest.webcol.sdp.peacocktv.com
innovidvideo.com
llnwd.net
max.com
meg.disco.peacocktv.com
mlb.com
mlbstatic.com
moviefone.com
mparticle.com
mubi.com
mytv.clients.peacocktv.com
mzstatic.com
n.account
n.assets
n.auth
n.content
n.devices
n.etpconfig
n.funmigrator
n.incentives
n.localization
n.messaging
n.news
n.playservice
n.reviews
n.subscription
n.talkbox
n.thirdpartyoauth
navigation.sr.roku.com
nba.com
nbc.com
nbcucdn.com
nbcuni.com
netflix.com
netflix.net
netflixdnstest0.com
nfl.com
nflxext.com
nflximg.com
nflxso.net
nflxvideo.net
netflix.net
netflixdnstest0.com
31.13.64.0/18
45.57.0.0/17
64.120.128.0/17
66.197.128.0/17
69.53.224.0/19
nhl.com
nhlstatic.com
omtrdc.net
open.spotify.com
ovp.peacocktv.com
pagead2.googlesyndication.com
pandora.com
paramount.tech
paramountplus.com
pconfig-prd.cdn.peacocktv.com
peacocktv.com
philo.com
play.ovp.peacocktv.com
playback-detail.sr.roku.com
pluto.tv
primevideo.com
retail.rpay.roku.com
rights-manager.sr.roku.com
roku.admeasurement.com
roku.com
rokutime.com
s3.amazonaws.com
saa.cbsi.com
saa.paramountplus.com
scdn.co
scorecardresearch.com
shudder.com
sling.com
soundcloud.com
spotify.com
ssai.peacocktv.com
stage-api-manga.crunchyroll.com
stage-api.crunchyroll.com
static-delivery.sr.roku.com
telemetry.nbcuott.com
therokuchannel.roku.com
this.audiolanguage
this.country
this.intercept
this.locale
this.settabid
this.tabid
thumbnails.cbsig.net
tidal.com
tubitv.com
tv.apple.com
viacomcbs.com
vod-gcs-cedexis.cbsaavideo.com
vod-gcs-google.cbsaavideo.com
vod-gcs-qwilt.cbsaavideo.com
vod-playlist.sr.roku.com
vod.delivery.roku.com
vortex.hulu.com
vrv.co
vudu.com
warnermediacdn.com
wbdstreaming.com
wmcdp.com
wmcdp.io
wv-license.sr.roku.com
www.crunchyroll.com
www.googletagmanager.com
wwwimage-secure.cbsstatic.com
wwwimage-us.pplusstatic.com
xumo.com
youtu.be
youtube.com
ytimg.com
# IP Ranges / CIDRs
108.175.32.0/20
17.0.0.0/8
192.173.64.0/18
198.38.96.0/19
198.45.48.0/20
207.45.72.0/22
208.75.76.0/22
hbomax.com
hbo.com
warnermediacdn.com
wbdstreaming.com
max.com
wmcdp.io
wmcdp.com
34.192.0.0/10
52.0.0.0/11
hulu.com
hulustream.com
huluim.com
huluad.com
vortex.hulu.com
doppler.hulu.com
conviva.com
hulu.hb.omtrdc.net
disneyplus.com
disney.com
bamgrid.com
bam.nr-data.net
cdn.registerdisney.go.com
disneystreaming.com
dssott.com
paramountplus.com
cbsivideo.com
cbsaavideo.com
cbs.com
viacomcbs.com
paramount.tech
wwwimage-us.pplusstatic.com
wwwimage-secure.cbsstatic.com
thumbnails.cbsig.net
bakery.pplus.paramount.tech
cbsi.demdex.net
vod-gcs-cedexis.cbsaavideo.com
cbsinteractive.hb.omtrdc.net
cbsi.live.ott.irdeto.com
saa.paramountplus.com
saa.cbsi.com
peacocktv.com
nbc.com
nbcuni.com
nbcucdn.com
telemetry.nbcuott.com
tubitv.com
fox.com
foxsports.com
fubo.tv
pluto.tv
roku.com
rokutime.com
therokuchannel.roku.com
api.roku.com
api.rokutime.com
roku.admeasurement.com
crunchyroll.com
beta-api.crunchyroll.com
www.crunchyroll.com
vrv.co
funimation.com
aiv-cdn.net
primevideo.com
amazon.com
amazonaws.com
cloudfront.net
s3.amazonaws.com
atv-ext.amazon.com
imdb.com
imdbtv.com
freevee.com
apple.com
apple-dns.net
apple.com.edgekey.net
mzstatic.com
apple-tv-plus.com
tv.apple.com
17.0.0.0/8
224.0.0.0/4
vudu.com
moviefone.com
mubi.com
shudder.com
britbox.com
acorn.tv
sling.com
philo.com
xumo.com
discoveryplus.com
dplay.com
espn.com
espncdn.com
espn.net
mlb.com
mlbstatic.com
nba.com
nfl.com
nhl.com
nhlstatic.com
spotify.com
scdn.co
api.spotify.com
cdn.spotify.com
open.spotify.com
pandora.com
soundcloud.com
deezer.com
tidal.com
# --- Added domains from research ---
# Apple TV+ (rebranded from Apple TV+)
tv.apple.com
amp.twimg.com
# Roku
retail.rpay.roku.com
api.rpay.roku.com
image.roku.com
vod.delivery.roku.com
vod-playlist.sr.roku.com
rights-manager.sr.roku.com
wv-license.sr.roku.com
static-delivery.sr.roku.com
bookmarks.sr.roku.com
navigation.sr.roku.com
images-svc.sr.roku.com
content.sr.roku.com
content-detail.sr.roku.com
playback-detail.sr.roku.com
api2.sr.roku.com
apis.rpay.roku.com
# Crunchyroll (additional)
api.crunchyroll.com
api-manga.crunchyroll.com
stage-api.crunchyroll.com
stage-api-manga.crunchyroll.com
beta-stage-api.crunchyroll.com
dev.crunchyroll.com
# Paramount+ (additional)
imasdk.googleapis.com
enduser.adsrv.org
cdn.privacy.paramount.com
www.googletagmanager.com
pagead2.googlesyndication.com
availability-fastly.syncbak-mediastore-cedexis.cbsaavideo.com
vod-gcs-qwilt.cbsaavideo.com
vod-gcs-google.cbsaavideo.com
# Peacock (additional domains from filterlist)
ingest.webcol.sdp.peacocktv.com
mparticle.com
fwmrm.net
scorecardresearch.com
omtrdc.net
ssai.peacocktv.com
mytv.clients.peacocktv.com
bff-ext.clients.peacocktv.com
imageservice.disco.peacocktv.com
play.ovp.peacocktv.com
atom.peacocktv.com
cybertron.id.peacocktv.com
meg.disco.peacocktv.com
ovp.peacocktv.com
pconfig-prd.cdn.peacocktv.com
cdn.peacocktv.com
# Amazon Prime Video (additional)
aiv-cdn.net
d1v0o82c0gfg2u.cloudfront.net
d2cvj4cr2d2r7g.cloudfront.net
d3n92br3b4jq4j.cloudfront.net
# YouTube (additional - common streaming)
youtube.com
googlevideo.com
ytimg.com
youtu.be
innovidvideo.com
# General streaming protocols
akamaized.net
edgesuite.net
llnwd.net
# DuckDuckGo video search
duckduckgo.com
duck.com
31.13.64.0/18
34.192.0.0/10
45.57.0.0/17
52.0.0.0/11
64.120.128.0/17
66.197.128.0/17
69.53.224.0/19
+246
View File
@@ -0,0 +1,246 @@
#!/usr/bin/env python3
"""
Update domain lists by merging new scraped domains with existing file contents.
Never deletes existing domains - only appends.
Usage:
python3 update_domains.py [--streaming] [--social] [--bypass] [--porn] [--bamboo] [--all]
"""
import os
import shutil
import argparse
import re
import urllib.request
import urllib.error
from datetime import datetime
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
PARENT_DIR = os.path.dirname(BASE_DIR)
BACKUP_DIR = os.path.join(os.environ.get("BACKUP_DIR", os.path.expanduser("~/.hermes/backups/domains_backup")))
STREAMING_FILE = os.path.join(PARENT_DIR, "streaming_domains_whitelist.txt")
SOCIAL_FILE = os.path.join(PARENT_DIR, "social_media.txt")
BAMBOO_FILE = os.path.join(PARENT_DIR, "bamboo_domains.txt")
BYPASS_FILE = os.path.join(PARENT_DIR, "bypass_sites.txt")
PORN_FILE = os.path.join(PARENT_DIR, "known_porn_domains.txt")
STREAMING_SOURCES = {
"Peacock Filterlist": "https://raw.githubusercontent.com/lit-bg/Peacock/refs/heads/main/filterlist.txt",
"Crunchyroll Endpoints": "https://gist.githubusercontent.com/NicmeisteR/cdc4867cf256c568b6a7f1844ce229f2/raw",
}
SOCIAL_SOURCES = {
"PiHole Streaming List": "https://raw.githubusercontent.com/ozankiratli/801ba17705e7f2a904d2e443af5a64f8/raw/436fa9f0c151afc15a601edf77c796b2a6de9be4/PiHoleStreamingLists.md",
}
def fetch_text(url):
try:
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
with urllib.request.urlopen(req, timeout=30) as resp:
return resp.read().decode("utf-8", errors="replace")
except Exception as e:
print(f"WARN: fetch failed for {url}: {e}")
return ""
def backup_file(path, label):
if not os.path.exists(path):
return
os.makedirs(BACKUP_DIR, exist_ok=True)
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
backup_path = os.path.join(BACKUP_DIR, f"{label}.{timestamp}")
shutil.copy2(path, backup_path)
print(f"Backup saved to {backup_path}")
# Strict domain validation: must match valid domain pattern
DOMAIN_RE = re.compile(
r'^(?=.{1,253}$)'
r'([a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+' # labels
r'[a-zA-Z]{2,63}$'
)
def is_valid_domain(line):
"""Validate a line is a proper domain (not code, IPs, or junk)"""
if not line or len(line) < 4 or len(line) > 253:
return False
if line.startswith("#"):
return False
if re.match(r'^(\d{1,3}\.){3}\d{1,3}', line):
return False
if "/" in line and re.search(r'\d+\.\d+\.\d+\.\d+/\d+', line):
return False
# Reject anything that doesn't look like a domain
if not DOMAIN_RE.match(line):
return False
# Reject known junk patterns from JS/CSS
junk_keywords = ["const", "var ", "let ", "function", "return", "if(", "else",
".catch", ".then", ".map(", ".filter(", ".reduce(", "import", "export",
"class", "extends", "=>", "=>" ]
if any(kw in line for kw in junk_keywords):
return False
return True
def is_ip_or_cidr(line):
ip_cidr_re = re.compile(r'^(\d{1,3}\.){3}\d{1,3}(/\d+)?$')
return bool(ip_cidr_re.match(line))
def load_existing(path):
"""Load existing domains and IPs, preserving EVERYTHING (never delete)."""
domains = set()
ips = set()
if not os.path.exists(path):
return domains, ips
with open(path, "r") as f:
for line in f:
line = line.strip().lower()
if not line:
continue
if line.startswith("#"):
# Preserve comments
domains.add(line)
continue
if is_ip_or_cidr(line):
ips.add(line)
else:
# Preserve EVERYTHING else - even junk lines, per user requirement "never delete"
domains.add(line)
return domains, ips
def extract_domains(text):
domains = set()
ips = set()
for line in text.splitlines():
line = line.strip().lower()
if not line or line.startswith("#"):
continue
# Handle AdBlock-style ||domain^ format
if line.startswith("||"):
clean = line[2:].split("^")[0].strip()
else:
clean = line.replace("|", "").replace("^", "").replace("$", "").strip()
if " " in clean:
clean = clean.split()[0]
clean = clean.lstrip("*")
if not clean:
continue
if is_ip_or_cidr(clean):
ips.add(clean)
elif is_valid_domain(clean):
domains.add(clean)
return domains, ips
def save_domains(path, domains, ips):
"""Save domains preserving original order of comments/sections"""
sorted_domains = sorted(d for d in domains if not is_ip_or_cidr(d))
sorted_ips = sorted(set(ips))
with open(path, "w") as f:
for d in sorted_domains:
f.write(d + "\n")
if sorted_ips:
f.write("\n# IP Ranges / CIDRs\n")
for ip in sorted_ips:
f.write(ip + "\n")
def update_streaming():
existing_domains, existing_ips = load_existing(STREAMING_FILE)
all_domains = set(existing_domains)
all_ips = set(existing_ips)
for name, url in STREAMING_SOURCES.items():
print(f"Fetching {name}...")
text = fetch_text(url)
if text:
scraped_domains, scraped_ips = extract_domains(text)
added = len(scraped_domains)
all_domains.update(scraped_domains)
all_ips.update(scraped_ips)
print(f" Scraped {added} new domains from {name}")
backup_file(STREAMING_FILE, "streaming_domains_whitelist.txt")
save_domains(STREAMING_FILE, all_domains, all_ips)
print(f"Streaming: {len(all_domains)} domains + {len(all_ips)} IPs total")
def update_social():
existing_domains, existing_ips = load_existing(SOCIAL_FILE)
all_domains = set(existing_domains)
all_ips = set(existing_ips)
for name, url in SOCIAL_SOURCES.items():
print(f"Fetching {name}...")
text = fetch_text(url)
if text:
scraped_domains, scraped_ips = extract_domains(text)
social_keywords = ["facebook", "instagram", "twitter", "tiktok", "linkedin",
"snapchat", "threads", "mastodon", "reddit", "discord",
"pinterest", "youtube", "twitch", "vk", "tumblr",
"medium", "clubhouse", "messenger", "telegram", "quora",
"imgur", "vimeo", "bluesky", "bsky"]
added = 0
for d in scraped_domains:
if any(kw in d for kw in social_keywords):
all_domains.add(d)
added += 1
print(f" Scraped {added} new social domains from {name}")
backup_file(SOCIAL_FILE, "social_media.txt")
save_domains(SOCIAL_FILE, all_domains, all_ips)
print(f"Social media: {len(all_domains)} domains + {len(all_ips)} IPs total")
def update_bypass():
streaming_domains, streaming_ips = load_existing(STREAMING_FILE)
social_domains, social_ips = load_existing(SOCIAL_FILE)
all_domains = streaming_domains | social_domains
all_ips = streaming_ips | social_ips
backup_file(BYPASS_FILE, "bypass_sites.txt")
save_domains(BYPASS_FILE, all_domains, all_ips)
print(f"Bypass sites: {len(all_domains)} domains + {len(all_ips)} IPs total")
def update_porn():
domains, ips = load_existing(PORN_FILE)
backup_file(PORN_FILE, "known_porn_domains.txt")
save_domains(PORN_FILE, domains, ips)
print(f"Porn domains preserved: {len(domains)} domains + {len(ips)} IPs")
def update_bamboo():
domains, ips = load_existing(BAMBOO_FILE)
print(f"Bamboo domains preserved: {len(domains)} domains + {len(ips)} IPs")
def main():
parser = argparse.ArgumentParser(description="Update domain lists - never delete!")
parser.add_argument("--streaming", action="store_true", help="Update streaming only")
parser.add_argument("--social", action="store_true", help="Update social media only")
parser.add_argument("--bypass", action="store_true", help="Update bypass_sites (combines streaming+social)")
parser.add_argument("--porn", action="store_true", help="Update porn domains")
parser.add_argument("--bamboo", action="store_true", help="Update bamboo domains")
parser.add_argument("--all", action="store_true", help="Update all lists")
args = parser.parse_args()
if args.all or not any([args.streaming, args.social, args.bypass, args.porn, args.bamboo]):
update_streaming()
update_social()
update_bypass()
update_porn()
update_bamboo()
else:
if args.streaming: update_streaming()
if args.social: update_social()
if args.bypass: update_bypass()
if args.porn: update_porn()
if args.bamboo: update_bamboo()
if __name__ == "__main__":
main()
-107
View File
@@ -1,107 +0,0 @@
#!/usr/bin/env python3
"""
Update streaming_domains_whitelist.txt with domains scraped from known sources.
Always backs up the previous file before overwriting.
"""
import os
import shutil
import urllib.request
import urllib.error
from datetime import datetime
DOMAINS_FILE = os.path.join(os.path.dirname(__file__), "..", "streaming_domains_whitelist.txt")
BACKUP_DIR = os.path.join(os.environ.get("HOME", "/tmp"), ".hermes", "backups", "domains_backup")
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
PARENT_DIR = os.path.dirname(BASE_DIR)
DOMAINS_FILE = os.path.join(PARENT_DIR, "streaming_domains_whitelist.txt")
SOURCES = {
"Netflix": "https://raw.githubusercontent.com/wickedyoda/public-setupfiles/main/domains/streaming_domains_whitelist.txt",
"Crunchyroll": "https://gist.githubusercontent.com/NicmeisteR/cdc4867cf256c568b6a7f1844ce229f2/raw",
"Peacock": "https://raw.githubusercontent.com/lit-bg/Peacock/refs/heads/main/filterlist.txt",
"PiHole Streaming List": "https://raw.githubusercontent.com/ozankiratli/801ba17705e7f2a904d2e443af5a64f8/raw/436fa9f0c151afc15a601edf77c796b2a6de9be4/PiHoleStreamingLists.md",
}
def fetch_text(url):
try:
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
with urllib.request.urlopen(req, timeout=30) as resp:
return resp.read().decode("utf-8", errors="replace")
except (urllib.error.URLError, Exception) as e:
print(f"WARN: fetch failed for {url}: {e}")
return ""
def backup_file(path):
os.makedirs(BACKUP_DIR, exist_ok=True)
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
backup_path = os.path.join(BACKUP_DIR, f"streaming_domains_whitelist.txt.{timestamp}")
shutil.copy2(path, backup_path)
print(f"Backup saved to {backup_path}")
def main():
domains = set()
comments = []
for name, url in SOURCES.items():
print(f"Fetching {name}...")
text = fetch_text(url)
if not text:
continue
for line in text.splitlines():
line = line.strip()
if not line or line.startswith("#"):
continue
# Extract domain-like tokens
parts = line.replace("^", "").replace("|", "").replace("/", "").split()
for part in parts:
if "." in part and "/" not in part and " " not in part:
clean = part.lstrip("*")
if clean and not clean[0].isdigit:
domains.add(clean)
# Hardcoded core domains to always keep
core_domains = [
"netflix.com", "nflxext.com", "nflximg.com", "nflxso.net", "nflxvideo.net",
"netflix.net", "hbomax.com", "hbo.com", "warnermediacdn.com", "wbdstreaming.com",
"max.com", "wmcdp.io", "wmcdp.com", "hulu.com", "hulustream.com", "huluim.com",
"huluad.com", "vortex.hulu.com", "doppler.hulu.com", "conviva.com",
"disneyplus.com", "disney.com", "bamgrid.com", "disneystreaming.com", "dssott.com",
"paramountplus.com", "cbsivideo.com", "cbsaavideo.com", "cbs.com", "viacomcbs.com",
"peacocktv.com", "nbc.com", "nbcuni.com", "nbcucdn.com",
"tubitv.com", "fox.com", "foxsports.com", "fubo.tv", "pluto.tv",
"roku.com", "rokutime.com", "therokuchannel.roku.com",
"crunchyroll.com", "vrv.co", "funimation.com",
"aiv-cdn.net", "primevideo.com", "amazon.com", "amazonaws.com",
"cloudfront.net", "s3.amazonaws.com", "atv-ext.amazon.com",
"imdb.com", "imdbtv.com", "freevee.com",
"apple.com", "apple-dns.net", "mzstatic.com",
"apple-tv-plus.com", "tv.apple.com",
"vudu.com", "moviefone.com", "mubi.com", "shudder.com",
"britbox.com", "acorn.tv", "sling.com", "philo.com", "xumo.com",
"discoveryplus.com", "dplay.com",
"espn.com", "espncdn.com", "espn.net",
"mlb.com", "mlbstatic.com", "nba.com", "nfl.com", "nhl.com", "nhlstatic.com",
"spotify.com", "scdn.co", "pandora.com", "soundcloud.com",
"deezer.com", "tidal.com",
]
domains.update(core_domains)
# Always back up before writing
if os.path.exists(DOMAINS_FILE):
backup_file(DOMAINS_FILE)
sorted_domains = sorted(d for d in domains if not d.startswith("1") and not d.startswith("2") and not d.startswith("3") and not d.startswith("4") and not d.startswith("5") and not d.startswith("6") and not d.startswith("7") and not d.startswith("8") and not d.startswith("9"))
ip_cidrs = sorted(d for d in domains if any(c.isdigit() for c in d))
with open(DOMAINS_FILE, "w") as f:
for domain in sorted_domains:
f.write(domain + "\n")
f.write("\n# IP Ranges / CIDRs\n")
for ip in ip_cidrs:
f.write(ip + "\n")
print(f"Domains updated: {len(sorted_domains)} domains + {len(ip_cidrs)} IPs")
if __name__ == "__main__":
main()