Update scripts and add combined_output flag

scripts/README.md

@@ -5,9 +5,11 @@
 The respective shell scripts contained within this directory can be utilised to generate application specific compliant
 configuration which can be utilised with:
 
-* Dnsmasq
-* Unbound
 * AdGuard Home
+* BIND9
+* Dnsmasq/Pi-hole
+* Squid
+* Unbound
 
 ## Usage
 
@@ -16,6 +18,7 @@ configuration which can be utilised with:
    The following example assumes a single shared Cacheserver IP:
 ```json
 {
+    "combined_output": false,
     "ips": {
         "generic":      ["10.10.10.200"]
     },
@@ -33,8 +36,9 @@ configuration which can be utilised with:
 }
 ```
 3. Run generation script relative to your DNS implementation: `bash create-dnsmasq.sh`.
-4. Copy files from `output/{dnsmasq,unbound}/*` to the respective locations for Dnsmasq/Unbound.
+4. If `combined_output` is set to `true` this will result in a single output file: `lancache.conf` with all your enabled services (applies to Adguard Home, Dnsmasq or Unbound).
-5. Restart Dnsmasq or Unbound.
+5. Copy files from `output/{adguardhome,dnsmasq,rpz,squid,unbound}/*` to the respective locations for Dnsmasq/Unbound.
+6. Restart the appropriate service.
 
 ### Notes for Dnsmasq users
 
@@ -44,5 +48,5 @@ Multi-IP Lancache setups are only supported with Dnsmasq or Pi-hole versions >=
 
 ### Notes for AdGuard Home users
 
-1. In the `config.json`, you may want to add an entry for your non-cached DNS upstreams. You can input this in `ip.adguardhome_upstream` as an array.
+1. Utilising `"combined_output": true` is more convenient.
-2. Once you have ran the script, you can point the upstream list to the text file generated. For example: `upstream_dns_file: "/root/cache-domains/scripts/output/adguardhome/cache-domains.txt"`
+2. Once you have run the script and uploaded the file to the appropriate location, you should navigate to Adguard Home -> Filters -> DNS blocklists -> Add blocklist -> Add a custom list.

scripts/config.example.json

@@ -1,6 +1,6 @@
 {
+    "combined_output": false,
     "ips": {
-		"adguardhome_upstream": ["94.140.14.140", "tls://dns.google", "https://dns.google/dns-query"],
         "steam":        ["10.10.3.10", "10.10.3.11"],
         "origin":       "10.10.3.12",
         "blizzard":     "10.10.3.13",

scripts/create-adguardhome.sh

@@ -6,37 +6,31 @@ path="${basedir}/cache_domains.json"
 export IFS=' '
 
 test=$(which jq);
-out=$?
+if [ $? -gt 0 ] ; then
-if [ $out -gt 0 ] ; then
     echo "This script requires jq to be installed."
     echo "Your package manager should be able to find it"
     exit 1
 fi
 
 cachenamedefault="disabled"
+combinedoutput=$(jq -r ".combined_output" config.json)
 
-while read -r line; do
+while read line; do
     ip=$(jq ".ips[\"${line}\"]" config.json)
     declare "cacheip${line}"="${ip}"
 done <<< $(jq -r '.ips | to_entries[] | .key' config.json)
 
-agh_upstreams=$(jq -r ".ips[\"adguardhome_upstream\"] | .[]" config.json)
+while read line; do
-
-while read -r line; do
     name=$(jq -r ".cache_domains[\"${line}\"]" config.json)
-        declare "cachename${line}"="${name}"
+    declare "cachename$line"="$name"
 done <<< $(jq -r '.cache_domains | to_entries[] | .key' config.json)
 
 rm -rf ${outputdir}
 mkdir -p ${outputdir}
-
+while read entry; do
-# add upstreams
-echo "${agh_upstreams}" >> "${outputdir}/cache-domains.txt"
-
-while read -r entry; do
     unset cacheip
     unset cachename
-        key=$(jq -r ".cache_domains[$entry].name" $path)
+    key=$(jq -r ".cache_domains[$entry].name" ${path})
     cachename="cachename${key}"
     if [ -z "${!cachename}" ]; then
         cachename="cachenamedefault"
@@ -46,32 +40,41 @@ while read -r entry; do
     fi
     cacheipname="cacheip${!cachename}"
     cacheip=$(jq -r 'if type == "array" then .[] else . end' <<< ${!cacheipname} | xargs)
-        while read -r fileid; do
+    while read fileid; do
-                while read -r filename; do
+        while read filename; do
-                        destfilename="cache-domains.txt"  #$(echo $filename | sed -e 's/txt/conf/')
+            destfilename=$(echo ${filename} | sed -e 's/txt/conf/')
             outputfile=${outputdir}/${destfilename}
             touch ${outputfile}
-                        while read -r fileentry; do
+            while read fileentry; do
-                                # Ignore comments, newlines and wildcards
+                # Ignore comments and newlines
                 if [[ ${fileentry} == \#* ]] || [[ -z ${fileentry} ]]; then
                     continue
                 fi
+                domainprefix="|"
+                if [[ $fileentry =~ ^\*\. ]]; then
+                    domainprefix="||"
+                fi
                 parsed=$(echo ${fileentry} | sed -e "s/^\*\.//")
-                                for i in ${cacheip}; do
+                if grep -q "${domainprefix}${parsed}^\$dnsrewrite" ${outputfile}; then
-                                        if grep -qx "\[/${parsed}/\]${i}" "${outputfile}"; then
                     continue
                 fi
-                                        echo "[/${parsed}/]${i}" >> "${outputfile}"
+                for i in ${cacheip}; do
+                    echo "${domainprefix}${parsed}^\$dnsrewrite=${i}" >> ${outputfile}
                 done
-                        done <<< $(cat ${basedir}/${filename} | sort);
+            done <<< $(cat ${basedir}/$filename | sort);
-                done <<< $(jq -r ".cache_domains[${entry}].domain_files[$fileid]" ${path})
+        done <<< $(jq -r ".cache_domains[${entry}].domain_files[${fileid}]" ${path})
     done <<< $(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path})
 done <<< $(jq -r '.cache_domains | to_entries[] | .key' ${path})
 
+if [[ ${combinedoutput} == "true" ]]; then
+    for file in ${outputdir}/*; do f=${file//${outputdir}\/} && f=${f//.conf} && echo "# ${f^}" >> ${outputdir}/lancache.conf && cat ${file} >> ${outputdir}/lancache.conf && rm ${file}; done
+fi
+
 cat << EOF
 Configuration generation completed.
 
-Please point the setting upstream_dns_file in AdGuardHome.yaml to the generated file.
+Please copy the following files:
-For example:
+- ./${outputdir}/*.conf to /opt/adguardhome/work/userfilters/
-upstream_dns_file: "/root/cache-domains/scripts/output/adguardhome/cache-domains.txt"
+- Navigate to Adguard Home -> Filters -> DNS blocklists -> Add blocklist -> Add a custom list
+- Add list for each service or utilise the combined output for a single list
 EOF

scripts/create-dnsmasq.sh

@@ -6,14 +6,14 @@ path="${basedir}/cache_domains.json"
 export IFS=' '
 
 test=$(which jq);
-out=$?
+if [ $? -gt 0 ] ; then
-if [ $out -gt 0 ] ; then
     echo "This script requires jq to be installed."
     echo "Your package manager should be able to find it"
     exit 1
 fi
 
 cachenamedefault="disabled"
+combinedoutput=$(jq -r ".combined_output" config.json)
 
 while read -r line; do
         ip=$(jq ".ips[\"${line}\"]" config.json)
@@ -30,7 +30,7 @@ mkdir -p ${outputdir}
 while read -r entry; do
     unset cacheip
     unset cachename
-        key=$(jq -r ".cache_domains[$entry].name" $path)
+    key=$(jq -r ".cache_domains[${entry}].name" ${path})
     cachename="cachename${key}"
     if [ -z "${!cachename}" ]; then
         cachename="cachenamedefault"
@@ -42,7 +42,7 @@ while read -r entry; do
     cacheip=$(jq -r 'if type == "array" then .[] else . end' <<< ${!cacheipname} | xargs)
     while read -r fileid; do
         while read -r filename; do
-                        destfilename=$(echo $filename | sed -e 's/txt/conf/')
+            destfilename=$(echo ${filename} | sed -e 's/txt/conf/')
             outputfile=${outputdir}/${destfilename}
             touch ${outputfile}
             while read -r fileentry; do
@@ -64,6 +64,10 @@ while read -r entry; do
     done <<< $(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path})
 done <<< $(jq -r '.cache_domains | to_entries[] | .key' ${path})
 
+if [[ ${combinedoutput} == "true" ]]; then
+    for file in ${outputdir}/*; do f=${file//${outputdir}\/} && f=${f//.conf} && echo "# ${f^}" >> ${outputdir}/lancache.conf && cat ${file} >> ${outputdir}/lancache.conf && rm ${file}; done
+fi
+
 cat << EOF
 Configuration generation completed.
 

scripts/create-rpz.sh

@@ -7,8 +7,7 @@ basedomain=${1:-lancache.net}
 export IFS=' '
 
 test=$(which jq);
-out=$?
+if [ $? -gt 0 ] ; then
-if [ $out -gt 0 ] ; then
     echo "This script requires jq to be installed."
     echo "Your package manager should be able to find it"
     exit 1
@@ -18,36 +17,36 @@ cachenamedefault="disabled"
 
 while read line; do
     ip=$(jq ".ips[\"${line}\"]" config.json)
-	declare "cacheip$line"="$ip"
+    declare "cacheip${line}"="${ip}"
 done <<< $(jq -r '.ips | to_entries[] | .key' config.json)
 
 while read line; do
     name=$(jq -r ".cache_domains[\"${line}\"]" config.json)
-	declare "cachename$line"="$name"
+    declare "cachename${line}"="${name}"
 done <<< $(jq -r '.cache_domains | to_entries[] | .key' config.json)
 
 rm -rf ${outputdir}
 mkdir -p ${outputdir}
-outputfile=${outputdir}/db.rpz.$basedomain
+outputfile=${outputdir}/db.rpz.${basedomain}
-cat > $outputfile << EOF
+cat > ${outputfile} << EOF
 \$TTL 60 ; default TTL
-\$ORIGIN rpz.$basedomain.
+\$ORIGIN rpz.${basedomain}.
-@       SOA     ns1.$basedomain. admin.$basedomain. (
+@       SOA     ns1.${basedomain}. admin.${basedomain}. (
 		$(date +%Y%m%d01) ; serial
                 604800     ; refresh (1 week)
                 600      ; retry (10 mins)
                 600      ; expire (10 mins)
                 600      ; minimum (10 mins)
                 )
-        NS      ns1.$basedomain.
+        NS      ns1.${basedomain}.
-        NS      ns2.$basedomain.
+        NS      ns2.${basedomain}.
 
 EOF
 
 while read entry; do
     unset cacheip
     unset cachename
-	key=$(jq -r ".cache_domains[$entry].name" $path)
+    key=$(jq -r ".cache_domains[${entry}].name" ${path})
     cachename="cachename${key}"
     if [ -z "${!cachename}" ]; then
         cachename="cachenamedefault"
@@ -59,16 +58,16 @@ while read entry; do
     cacheip=$(jq -r 'if type == "array" then .[] else . end' <<< ${!cacheipname} | xargs)
     while read fileid; do
         while read filename; do
-			echo "" >> $outputfile
+            echo "" >> ${outputfile}
-			echo "; $(echo $filename | sed -e 's/.txt$//')" >> $outputfile
+            echo "; $(echo ${filename} | sed -e 's/.txt$//')" >> ${outputfile}
-			destfilename=$(echo $filename | sed -e 's/txt/conf/')
+            destfilename=$(echo ${filename} | sed -e 's/txt/conf/')
             while read fileentry; do
                 # Ignore comments and newlines
-				if [[ $fileentry == \#* ]] || [[ -z $fileentry ]]; then
+                if [[ ${fileentry} == \#* ]] || [[ -z ${fileentry} ]]; then
                     continue
                 fi
-				parsed=$(echo $fileentry)
+                parsed=$(echo ${fileentry})
-				if grep -qx "^\"${parsed}\". " $outputfile; then
+                if grep -qx "^\"${parsed}\". " ${outputfile}; then
                     continue
                 fi
                 t=""
@@ -88,27 +87,27 @@ while read entry; do
                         "${parsed}" \
                         "${t}" \
                         "${i}" \
-						>> $outputfile
+                        >> ${outputfile}
                 done
-			done <<< $(cat ${basedir}/$filename | sort);
+            done <<< $(cat ${basedir}/${filename} | sort);
-		done <<< $(jq -r ".cache_domains[$entry].domain_files[$fileid]" $path)
+        done <<< $(jq -r ".cache_domains[${entry}].domain_files[${fileid}]" ${path})
-	done <<< $(jq -r ".cache_domains[$entry].domain_files | to_entries[] | .key" $path)
+    done <<< $(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path})
-done <<< $(jq -r '.cache_domains | to_entries[] | .key' $path)
+done <<< $(jq -r '.cache_domains | to_entries[] | .key' ${path})
 
 cat << EOF
 Configuration generation completed.
 
 Please include the rpz zone in your bind configuration"
-- cp $outputfile /etc/bind
+- cp ${outputfile} /etc/bind
 - configure the zone and use it
 
 options {
     [...]
-    response-policy {zone "rpz.$basedomain";};
+    response-policy {zone "rpz.${basedomain}";};
     [...]
 }
 zone "rpz.$basedomain" {
     type master;
-    file "/etc/bind/db.rpz.$basedomain";
+    file "/etc/bind/db.rpz.${basedomain}";
 };
 EOF

scripts/create-squid.sh

@@ -7,8 +7,7 @@ REGEX="^\\*\\.(.*)$"
 export IFS=' '
 
 test=$(which jq);
-out=$?
+if [ $? -gt 0 ] ; then
-if [ $out -gt 0 ] ; then
     echo "This script requires jq to be installed."
     echo "Your package manager should be able to find it"
     exit 1
@@ -25,7 +24,7 @@ rm -rf ${outputdir}
 mkdir -p ${outputdir}
 while read -r entry; do
     unset cachename
-        key=$(jq -r ".cache_domains[$entry].name" $path)
+    key=$(jq -r ".cache_domains[$entry].name" ${path})
     cachename="cachename${key}"
     if [ -z "${!cachename}" ]; then
         cachename="cachenamedefault"
@@ -43,7 +42,7 @@ while read -r entry; do
                 if [[ ${fileentry} == \#* ]] || [[ -z ${fileentry} ]]; then
                     continue
                 fi
-				# Handle wildcards to squid wildcards
+                # Ha wildcards to squid wildcards
                 parsed=$(echo ${fileentry} | sed -e "s/^\*\./\./")
                 # If we have cdn.thing and *.cdn.thing in cache_domains
                 # Squid requires ONLY cdn.thing
@@ -57,10 +56,9 @@ while read -r entry; do
                         continue
                     fi
                 fi
-
                 echo "${parsed}" >> "${outputfile}"
             done <<< $(cat ${basedir}/${filename} | sort);
-                done <<< $(jq -r ".cache_domains[${entry}].domain_files[$fileid]" ${path})
+        done <<< $(jq -r ".cache_domains[${entry}].domain_files[${fileid}]" ${path})
     done <<< $(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path})
 done <<< $(jq -r '.cache_domains | to_entries[] | .key' ${path})
 

scripts/create-unbound.sh

@@ -6,23 +6,23 @@ path="${basedir}/cache_domains.json"
 export IFS=' '
 
 test=$(which jq);
-out=$?
+if [ $? -gt 0 ] ; then
-if [ $out -gt 0 ] ; then
     echo "This script requires jq to be installed."
     echo "Your package manager should be able to find it"
     exit 1
 fi
 
 cachenamedefault="disabled"
+combinedoutput=$(jq -r ".combined_output" config.json)
 
 while read line; do
     ip=$(jq ".ips[\"${line}\"]" config.json)
-	declare "cacheip$line"="$ip"
+    declare "cacheip${line}"="${ip}"
 done <<< $(jq -r '.ips | to_entries[] | .key' config.json)
 
 while read line; do
     name=$(jq -r ".cache_domains[\"${line}\"]" config.json)
-	declare "cachename$line"="$name"
+    declare "cachename${line}"="${name}"
 done <<< $(jq -r '.cache_domains | to_entries[] | .key' config.json)
 
 rm -rf ${outputdir}
@@ -30,7 +30,7 @@ mkdir -p ${outputdir}
 while read entry; do
     unset cacheip
     unset cachename
-	key=$(jq -r ".cache_domains[$entry].name" $path)
+    key=$(jq -r ".cache_domains[${entry}].name" ${path})
     cachename="cachename${key}"
     if [ -z "${!cachename}" ]; then
         cachename="cachenamedefault"
@@ -42,29 +42,33 @@ while read entry; do
     cacheip=$(jq -r 'if type == "array" then .[] else . end' <<< ${!cacheipname} | xargs)
     while read fileid; do
         while read filename; do
-			destfilename=$(echo $filename | sed -e 's/txt/conf/')
+            destfilename=$(echo ${filename} | sed -e 's/txt/conf/')
             outputfile=${outputdir}/${destfilename}
-			touch $outputfile
+            touch ${outputfile}
             while read fileentry; do
                 # Ignore comments and newlines
-				if [[ $fileentry == \#* ]] || [[ -z $fileentry ]]; then
+                if [[ ${fileentry} == \#* ]] || [[ -z ${fileentry} ]]; then
                     continue
                 fi
-				parsed=$(echo $fileentry | sed -e "s/^\*\.//")
+                parsed=$(echo ${fileentry} | sed -e "s/^\*\.//")
-				if grep -qx "  local-zone: \"${parsed}\" redirect" $outputfile; then
+                if grep -qx "  local-zone: \"${parsed}\" redirect" ${outputfile}; then
                     continue
                 fi
-        if [[ $(head -n 1 $outputfile) != "server:" ]]; then
+                if [[ $(head -n 1 ${outputfile}) != "server:" ]]; then
-            echo "server:" >> $outputfile
+                    echo "server:" >> ${outputfile}
                 fi
-				echo "  local-zone: \"${parsed}\" redirect" >> $outputfile
+                echo "  local-zone: \"${parsed}\" redirect" >> ${outputfile}
                 for i in ${cacheip}; do
-					echo "  local-data: \"${parsed} 30 IN A ${i}\"" >> $outputfile
+                    echo "  local-data: \"${parsed} 30 IN A ${i}\"" >> ${outputfile}
                 done
-			done <<< $(cat ${basedir}/$filename | sort);
+            done <<< $(cat ${basedir}/${filename} | sort);
-		done <<< $(jq -r ".cache_domains[$entry].domain_files[$fileid]" $path)
+        done <<< $(jq -r ".cache_domains[${entry}].domain_files[${fileid}]" ${path})
-	done <<< $(jq -r ".cache_domains[$entry].domain_files | to_entries[] | .key" $path)
+    done <<< $(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path})
-done <<< $(jq -r '.cache_domains | to_entries[] | .key' $path)
+done <<< $(jq -r '.cache_domains | to_entries[] | .key' ${path})
+
+if [[ ${combinedoutput} == "true" ]]; then
+    for file in ${outputdir}/*; do f=${file//${outputdir}\/} && f=${f//.conf} && echo "# ${f^}" >> ${outputdir}/lancache.conf && cat ${file} >> ${outputdir}/lancache.conf && rm ${file}; done
+fi
 
 cat << EOF
 Configuration generation completed.