Merge branch 'master' into master

README.md

@@ -14,10 +14,9 @@ You can use this list one of two ways:
 There is a cache_domains.json file to define CDNs and additional meta deta with the following structure
 
 - cache_domains: Array of cache_domain object
-	- name: shortname for the cache domain
+	- name: shortname for the cache domain. Should match `^[0-9A-Za-z]$`
 	- description: a longer description to aid others in identifying what this domain does (not all users of this repo will want to enable all caches)
 	- notes: implementation specific notes which may be useful for other users
-	- mixed_content: true if this domain hosts mixed https and http content (a straight dns injection is unlikely to work in this case). Assumed to be false if undefined
 	- domain_files: array of files within the repo assosciated to the cdn. Most cdn's only need one file
 	- Example domain entry for origin
 ```json
@@ -27,7 +26,6 @@ There is a cache_domains.json file to define CDNs and additional meta deta with
 			"name": "origin",
 			"description": "CDN for origin",
 			"notes": "Should be enabled for HTTP traffic only or with a HTTPS proxy else origin client download fails",
-			"mixed_content": true,
 			"domain_files": ["origin.txt"]
 		}
 	]
@@ -43,6 +41,7 @@ There is a separate file for each cacheable service. Some notes on formatting:
   - Wildcards are permitted as per below
   - Lines starting with a # will be treated as a comment.
   - Files must end with an empty newline.
+  - File names must match the regular expression `^[0-9A-Za-z].txt`
 
 #### Wildcards
 
@@ -65,10 +64,20 @@ Please fork this repository and submit pull requests if you have any extra hostn
 
 If the CDN you are adding does not already exist please remember to add an entry within cache_domains.json as well as a new .txt file
 
+### Uncacheable CDNs
+
+Certain CDNs are not suitable for inclusion on this list because the content is entirely delivered over HTTPS and thus cannot be cached.
+
+For a list of currently identified uncacheable CDNs please see these issues: https://github.com/uklans/cache-domains/issues?q=is%3Aissue+is%3Aopen+label%3Ahttps-cantfix
+
 ## Issues and Feedback
 
 Please raise all issues and feedback on GitHub at [uklans/cache-domains](https://github.com/uklans/cache-domains/issues).
 
+## Useful Information
+
+There are a few useful anecdotes on cache domains usage in some old issues https://github.com/uklans/cache-domains/issues?q=is%3Aissue+is%3Aclosed+label%3Afor-reference-only
+
 ## License
 
 The MIT License (MIT)

cache_domains.json

@@ -20,6 +20,11 @@
 			"description": "Daybreak games CDN",
 			"domain_files": ["daybreak.txt"]
 		},
+		{
+			"name": "epicgames",
+			"description": "CDN for Epic Games",
+			"domain_files": ["epicgames.txt"]
+		},
 		{
 			"name": "frontier",
 			"description": "CDN for frontier games",
@@ -36,16 +41,14 @@
 			"domain_files": ["nexusmods.txt"]
 		},
 		{
-			"name": "nintendo",
+			"name": "neverwinter",
-			"description": "CDN for nintendo (swtich)",
+			"description": "Cryptic CDN for Neverwinter",
-			"domain_files": ["nintendo.txt"]
+			"domain_files": ["neverwinter.txt"]
 		},
 		{
-			"name": "origin",
+			"name": "nintendo",
-			"description": "CDN for origin",
+			"description": "CDN for Nintendo consoles and download servers",
-			"notes": "Should be enabled for HTTP traffic only or with a HTTPS proxy else origin client download fails",
+			"domain_files": ["nintendo.txt"]
-			"mixed_content": true,
-			"domain_files": ["origin.txt"]
 		},
 		{
 			"name": "renegadex",
@@ -82,10 +85,20 @@
 			"description": "CDN for uplay downloader",
 			"domain_files": ["uplay.txt"]
 		},
+		{
+			"name": "teso",
+			"description": "CDN for The Elder Scrolls Online",
+			"domain_files": ["teso.txt"]
+		},
 		{
 			"name": "twitch",
 			"description": "CDN for twitch games / mods and addons",
-			"domain_files": ["twitchapp.txt"]
+			"domain_files": ["twitch.txt"]
+		},
+		{
+			"name": "warframe",
+			"description": "CDN for Warframe",
+			"domain_files": ["warframe.txt"]
 		},
 		{
 			"name": "wargaming",
@@ -100,9 +113,7 @@
 		{
 			"name": "xboxlive",
 			"description": "CDN for xboxlive",
-			"domain_files": ["xboxlive.txt"],
+			"domain_files": ["xboxlive.txt"]
-			"mixed_content": true
 		}
 	]
 }
-

epicgames.txt

@@ -0,0 +1,5 @@
+epicgames-download1.akamaized.net
+download.epicgames.com
+download2.epicgames.com
+download3.epicgames.com
+download4.epicgames.com

neverwinter.txt

@@ -0,0 +1 @@
+level3.nwhttppatch.crypticstudios.com

nintendo.txt

@@ -1,8 +1,13 @@
 ccs.cdn.wup.shop.nintendo.com
-pushmo.hac.lp1.eshop.nintendo.net
+ccs.cdn.wup.shop.nintendo.net
+ccs.cdn.wup.shop.nintendo.net.edgesuite.net
+geisha-wup.cdn.nintendo.net
+geisha-wup.cdn.nintendo.net.edgekey.net
+idbe-wup.cdn.nintendo.net
+idbe-wup.cdn.nintendo.net.edgekey.net
 ecs-lp1.hac.shop.nintendo.net
 receive-lp1.dg.srv.nintendo.net
-aqua.hac.lp1.d4c.nintendo.net
+*.wup.shop.nintendo.net
-atum.hac.lp1.d4c.nintendo.net
+*.wup.eshop.nintendo.net
-bugyo.hac.lp1.eshop.nintendo.net
+*.hac.lp1.d4c.nintendo.net
-tagaya.hac.lp1.eshop.nintendo.net
+*.hac.lp1.eshop.nintendo.net

origin.txt

@@ -1,6 +0,0 @@
-# WARNING:  Origin has been seen downloading https client downloads on origin-a.akamaihd.net.  A solution should be in place to forward https to the origin server (eg sniproxy)
-origin-a.akamaihd.net
-akamai.cdn.ea.com
-lvlt.cdn.ea.com
-river.data.ea.com
-origin-a.akamaihd.net.edgesuite.net

riot.txt

@@ -2,4 +2,4 @@ l3cdn.riotgames.com
 worldwide.l3cdn.riotgames.com
 riotgamespatcher-a.akamaihd.net
 riotgamespatcher-a.akamaihd.net.edgesuite.net
-lol.dyn.riotcdn.net
+*.dyn.riotcdn.net

scripts/config.example.json

@@ -1,6 +1,6 @@
 {
 	"ips": {
-		"steam":	"10.10.3.11",
+		"steam":	["10.10.3.10", "10.10.3.11"],
 		"origin":	"10.10.3.12",
 		"blizzard":	"10.10.3.13",
 		"windows":	"10.10.3.14",

scripts/create-dnsmasq.sh

@@ -16,7 +16,7 @@ fi
 cachenamedefault="disabled"
 
 while read line; do
-        ip=$(jq -r ".ips[\"${line}\"]" config.json)
+        ip=$(jq ".ips[\"${line}\"]" config.json)
         declare "cacheip$line"="$ip"
 done <<< $(jq -r '.ips | to_entries[] | .key' config.json)
 
@@ -39,7 +39,7 @@ while read entry; do
                 continue;
         fi
         cacheipname="cacheip${!cachename}"
-        cacheip=${!cacheipname}
+        cacheip=$(jq -r 'if type == "array" then .[] else . end' <<< ${!cacheipname} | xargs)
         while read fileid; do
                 while read filename; do
                         destfilename=$(echo $filename | sed -e 's/txt/conf/')
@@ -54,7 +54,9 @@ while read entry; do
                                 if grep -q "$parsed" $outputfile; then
                                         continue
                                 fi
-                                echo "address=/${parsed}/${cacheip}" >> $outputfile
+                                for i in ${cacheip}; do
+                                        echo "address=/${parsed}/${i}" >> $outputfile
+                                done
                         done <<< $(cat ${basedir}/$filename);
                 done <<< $(jq -r ".cache_domains[$entry].domain_files[$fileid]" $path)
         done <<< $(jq -r ".cache_domains[$entry].domain_files | to_entries[] | .key" $path)

scripts/create-unbound.sh

@@ -16,7 +16,7 @@ fi
 cachenamedefault="disabled"
 
 while read line; do 
-	ip=$(jq -r ".ips[\"${line}\"]" config.json)
+	ip=$(jq ".ips[\"${line}\"]" config.json)
 	declare "cacheip$line"="$ip"
 done <<< $(jq -r '.ips | to_entries[] | .key' config.json)
 
@@ -39,7 +39,7 @@ while read entry; do
 		continue;
 	fi
 	cacheipname="cacheip${!cachename}"
-	cacheip=${!cacheipname}
+	cacheip=$(jq -r 'if type == "array" then .[] else . end' <<< ${!cacheipname} | xargs)
 	while read fileid; do
 		while read filename; do
 			destfilename=$(echo $filename | sed -e 's/txt/conf/')
@@ -56,7 +56,9 @@ while read entry; do
 					continue
 				fi
 				echo "  local-zone: \"${parsed}\" redirect" >> $outputfile
-				echo "  local-data: \"${parsed} 30 IN A ${cacheip}\"" >> $outputfile
+				for i in ${cacheip}; do
+					echo "  local-data: \"${parsed} 30 IN A ${i}\"" >> $outputfile
+				done
 			done <<< $(cat ${basedir}/$filename);
 		done <<< $(jq -r ".cache_domains[$entry].domain_files[$fileid]" $path)
 	done <<< $(jq -r ".cache_domains[$entry].domain_files | to_entries[] | .key" $path)

teso.txt

@@ -0,0 +1 @@
+live.patcher.elderscrollsonline.com

warframe.txt

@@ -0,0 +1 @@
+content.warframe.com

windowsupdates.txt

@@ -6,3 +6,8 @@ dl.delivery.mp.microsoft.com
 *.update.microsoft.com
 *.do.dsp.mp.microsoft.com
 *.microsoft.com.edgesuite.net
+amupdatedl.microsoft.com
+amupdatedl2.microsoft.com
+amupdatedl3.microsoft.com
+amupdatedl4.microsoft.com
+amupdatedl5.microsoft.com