Merge pull request #149 from nightah/fix-generation-scripts-sorting

Fix generation scripts

.gitignore

@@ -1 +1,2 @@
 scripts/output
+scripts/config.json

README.md

@@ -11,13 +11,13 @@ You can use this list one of two ways:
  - Overriding DNS for these hostnames to point to the IP of your cache server.
  - Use them in Squid with WCCP to redirect content to the right cache server.
 
-There is a cache_domains.json file to define CDNs and additional meta deta with the following structure
+There is a cache_domains.json file to define CDNs and additional metadata with the following structure
 
 - cache_domains: Array of cache_domain object
     - name: shortname for the cache domain. Should match `^[0-9A-Za-z]$`
     - description: a longer description to aid others in identifying what this domain does (not all users of this repo will want to enable all caches)
     - notes: implementation specific notes which may be useful for other users
-    - domain_files: array of files within the repo assosciated to the cdn. Most cdn's only need one file
+    - domain_files: array of files within the repo associated to the cdn. Most CDNs only need one file
     - Example domain entry for steam
 ```json
 {
@@ -35,7 +35,7 @@ There is a cache_domains.json file to define CDNs and additional meta deta with
 
 There is a separate file for each cacheable service. Some notes on formatting:
 
-  - Every line should be a seperate hostname for that service.
+  - Every line should be a separate hostname for that service.
   - Only one entry is permitted per line.
   - Wildcards are permitted as per below
   - Lines starting with a # will be treated as a comment.
@@ -46,10 +46,10 @@ There is a separate file for each cacheable service. Some notes on formatting:
 
 The wildcard format shall be defined as per the below
 
-  - Wildcards should be represented with an asterix.
+  - Wildcards should be represented with an asterisk.
   - If a wildcard is used, it should be the first character on the line.
   - Wildcards are not treated as matching null, e.g. `*.example.com` will match `a.example.com` but will not match `example.com`
-  - Only simple domain wildcards will be accepted eg `*.example.com` not `*ww.example.com`
+  - Only simple domain wildcards will be accepted e.g. `*.example.com` not `*ww.example.com`
 
 ##### Notes for Squid users
 

scripts/README.md

@@ -0,0 +1,49 @@
+# DNS Generation Scripts
+
+## Introduction
+
+The respective shell scripts contained within this directory can be utilised to generate application specific compliant
+configuration which can be utilised with:
+
+* Dnsmasq
+* Unbound
+
+## Usage
+
+1. Copy `config.example.json` to `config.json`.
+2. Modify `config.json` to include your Cacheserver's IP(s) and the CDNs you plan to cache.
+   The following example assumes a single shared Cacheserver IP:
+```json
+{
+  "ips": {
+    "generic":	["10.10.10.200"]
+  },
+  "cache_domains": {
+    "blizzard":     "generic",
+    "epicgames":    "generic",
+    "nintendo":     "generic",
+    "origin":       "generic",
+    "riot":         "generic",
+    "sony":         "generic",
+    "steam":        "generic",
+    "uplay":        "generic",
+    "wsus":         "generic"
+  }
+}
+```
+3. Run generation script relative to your DNS implementation: `bash create-dnsmasq.sh`.
+4. Copy files from `output/{dnsmasq,unbound}/*` to the respective locations for Dnsmasq/Unbound.
+5. Restart Dnsmasq or Unbound.
+
+### Notes for Dnsmasq users
+
+**This also applies to users utilising the script alongside Pi-hole.**
+
+If utilising the `create-dnsmasq.sh` the generation script will create a `lancache.conf` which also loads in the respective `*.hosts` files.
+
+The `lancache.conf` should be copied into the `/etc/dnsmasq.d/` location but also will need to be modified to point to the respective location of the `*.hosts` files.
+
+You can copy the `*.hosts` file to any location other than `/etc/dnsmasq.d/` as this location is utilised only for `*.conf` files.
+
+For example if utilising Pi-hole a user can copy the `*.hosts` files to `/etc/pihole/` and modify the `lancache.conf` with the following command, prior to copying it to `/etc/dnsmasq.d/`:
+`sed -i 's/dnsmasq\/hosts/pihole/g' output/dnsmasq/lancache.conf`

scripts/create-dnsmasq.sh

@@ -26,7 +26,7 @@ while read -r line; do
 done <<< $(jq -r '.cache_domains | to_entries[] | .key' config.json)
 
 rm -rf ${outputdir}
-mkdir -p ${outputdir}
+mkdir -p ${outputdir}/hosts
 touch ${outputdir}/lancache.conf
 while read -r entry; do
         unset cacheip
@@ -45,37 +45,45 @@ while read -r entry; do
                 while read -r filename; do
                         destfilename=$(echo $filename | sed -e 's/txt/hosts/')
                         lancacheconf=${outputdir}/lancache.conf
-                        outputfile=${outputdir}/${destfilename}
+                        outputfile=${outputdir}/hosts/${destfilename}
-                        echo "addn-hosts=/etc/dnsmasq.d/${destfilename}" >> ${lancacheconf}
+                        echo "addn-hosts=/etc/dnsmasq/hosts/${destfilename}" >> ${lancacheconf}
                         touch "$outputfile"
                         # Wildcard entries
                         while read -r fileentry; do
-                                # Ignore comments
+                                # Ignore comments and non-wildcards
-                                if [[ $fileentry == \#* ]]; then
+                                if [[ $fileentry == \#* ]] || [[ ! $fileentry =~ ^\*\. ]]; then
                                         continue
                                 fi
-                                wildcard=$(echo $fileentry | grep "*." | sed -e "s/^\*\.//")
+                                wildcard=$(echo $fileentry | sed -e "s/^\*\.//")
-                                if grep -q "$wildcard" "$lancacheconf"; then
+                                if grep -qx "$wildcard" "$lancacheconf"; then
                                         continue
                                 fi
                                 for i in ${cacheip}; do
                                         echo "address=/${wildcard}/${i}" >> "$lancacheconf"
                                 done
-                        done <<< $(cat ${basedir}/$filename);
+                        done <<< $(cat ${basedir}/$filename | sort);
                         # All other entries
                         while read -r fileentry; do
-                                # Ignore comments
+                                # Ignore comments and wildcards
-                                if [[ $fileentry == \#* ]]; then
+                                if [[ $fileentry =~ ^(\#|\*\.) ]]; then
                                         continue
                                 fi
-                                parsed=$(echo $fileentry | sed -e "s/^\*\.//")
+                                parsed=$(echo $fileentry)
-                                if grep -q "$parsed" "$outputfile"; then
+                                if grep -qx "$parsed" "$outputfile"; then
                                         continue
                                 fi
                                 for i in ${cacheip}; do
                                         echo "${i} ${parsed}" >> "$outputfile"
                                 done
-                        done <<< $(cat ${basedir}/$filename);
+                        done <<< $(cat ${basedir}/$filename | sort);
                 done <<< $(jq -r ".cache_domains[$entry].domain_files[$fileid]" $path)
         done <<< $(jq -r ".cache_domains[$entry].domain_files | to_entries[] | .key" $path)
 done <<< $(jq -r '.cache_domains | to_entries[] | .key' $path)
+
+cat << EOF
+Configuration generation completed.
+
+Please copy the following files:
+- ./${outputdir}/lancache.conf to /etc/dnsmasq/dnsmasq.d/
+- ./${outputdir}/hosts to /etc/dnsmasq/
+EOF

scripts/create-unbound.sh

@@ -52,14 +52,21 @@ while read entry; do
 					continue
 				fi
 				parsed=$(echo $fileentry | sed -e "s/^\*\.//")
-				if grep -q "$parsed" $outputfile; then
+				if grep -qx "$parsed" $outputfile; then
 					continue
 				fi
 				echo "  local-zone: \"${parsed}\" redirect" >> $outputfile
 				for i in ${cacheip}; do
 					echo "  local-data: \"${parsed} 30 IN A ${i}\"" >> $outputfile
 				done
-			done <<< $(cat ${basedir}/$filename);
+			done <<< $(cat ${basedir}/$filename | sort);
 		done <<< $(jq -r ".cache_domains[$entry].domain_files[$fileid]" $path)
 	done <<< $(jq -r ".cache_domains[$entry].domain_files | to_entries[] | .key" $path)
 done <<< $(jq -r '.cache_domains | to_entries[] | .key' $path)
+
+cat << EOF
+Configuration generation completed.
+
+Please copy the following files:
+- ./${outputdir}/*.conf to /etc/unbound/unbound.conf.d/
+EOF