diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..2e62126 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,8 @@ +root = true + +[*] +indent_style = space +indent_size = 4 +trim_trailing_whitespace = true +end_of_line = lf +insert_final_newline = true diff --git a/faq.md b/faq.md index 2afec64..d4ab901 100644 --- a/faq.md +++ b/faq.md @@ -20,15 +20,15 @@ Many of the maintainers of this repo also contribute to [the lancachenet project There are several reasons why a particular service / CDN / website might not be on this list. Here are some of the more common ones: - 1. It's not technically possible to cache it. Many popular websites, including video streaming sites and even some games CDN's use SSL Encryption (i.e https URLs) to serve their content. Because the client opens a secure connection directly to the host, there is no way for the network operator to see what they are downloading, nor cache it. Whilst there are several approaches to work around this, such as MITM techniques, they usually rely on control over the client device to affect SSL Certificates - control somebody running a BYOC LAN typically does not have over the devices customers bring. - - - [These issues](https://github.com/uklans/cache-domains/issues?q=is%3Aissue+is%3Aopen+label%3Ahttps-cantfix) contain game CDNs that we would like to include, but cannot for this reason. - - 2. It's out of scope for a LAN. We try to keep this list targeted towards people running LANs. Whilst some none game-related CDNs are included for things like windows updates that use internet bandwidth at LANs, we do not go searching for unrelated sites / hostnames. - - 3. It's not a good cache target / it would not get a good hit ratio. Game downloads are a great cache target because they are large, remain the same for every user and are likely to be downloaded multiple times at a LAN. Other hostnames that only serve dynamic or media files, or content that is not likely to be downloaded multiple times is not a good cache target and can waste valuable storage space on your cache server. This can lead to potentially more valuable content being evicted from the cache due to low space. - - 4. We simply don't yet have a tested list of hostnames for it yet. This is the category you can help with - if you have something that doesn't fall into one of the above reasons not to include it, we would love to review your PR. See [the readme](https://github.com/uklans/cache-domains) for instructions on how to add a new CDN. +1. It's not technically possible to cache it. Many popular websites, including video streaming sites and even some games CDN's use SSL Encryption (i.e https URLs) to serve their content. Because the client opens a secure connection directly to the host, there is no way for the network operator to see what they are downloading, nor cache it. Whilst there are several approaches to work around this, such as MITM techniques, they usually rely on control over the client device to affect SSL Certificates - control somebody running a BYOC LAN typically does not have over the devices customers bring. + + - [These issues](https://github.com/uklans/cache-domains/issues?q=is%3Aissue+is%3Aopen+label%3Ahttps-cantfix) contain game CDNs that we would like to include, but cannot for this reason. + +2. It's out of scope for a LAN. We try to keep this list targeted towards people running LANs. Whilst some none game-related CDNs are included for things like windows updates that use internet bandwidth at LANs, we do not go searching for unrelated sites / hostnames. + +3. It's not a good cache target / it would not get a good hit ratio. Game downloads are a great cache target because they are large, remain the same for every user and are likely to be downloaded multiple times at a LAN. Other hostnames that only serve dynamic or media files, or content that is not likely to be downloaded multiple times is not a good cache target and can waste valuable storage space on your cache server. This can lead to potentially more valuable content being evicted from the cache due to low space. + +4. We simply don't yet have a tested list of hostnames for it yet. This is the category you can help with - if you have something that doesn't fall into one of the above reasons not to include it, we would love to review your PR. See [the readme](https://github.com/uklans/cache-domains) for instructions on how to add a new CDN. ## SNI Proxy / HTTPS @@ -38,4 +38,4 @@ It does not allow https / SSL content to be inspected or cached. Hostnames that ## How can I test an addition to this list? -If you are using the lancachenet project, you can fork this repo, add your test hostnames and then use [these instructions](https://github.com/lancachenet/lancache-dns#custom-forksbranches) to use it with your cache instance rather than the main repo. \ No newline at end of file +If you are using the lancachenet project, you can fork this repo, add your test hostnames and then use [these instructions](https://github.com/lancachenet/lancache-dns#custom-forksbranches) to use it with your cache instance rather than the main repo. diff --git a/scripts/README.md b/scripts/README.md index 26463c1..c9c7bdf 100755 --- a/scripts/README.md +++ b/scripts/README.md @@ -5,36 +5,41 @@ The respective shell scripts contained within this directory can be utilised to generate application specific compliant configuration which can be utilised with: -* Dnsmasq -* Unbound * AdGuard Home +* BIND9 +* Dnsmasq/Pi-hole +* Squid +* Unbound ## Usage 1. Copy `config.example.json` to `config.json`. 2. Modify `config.json` to include your Cacheserver's IP(s) and the CDNs you plan to cache. - The following example assumes a single shared Cacheserver IP: + +The following example assumes a single shared Cacheserver IP: ```json { - "ips": { - "generic": ["10.10.10.200"] - }, - "cache_domains": { - "blizzard": "generic", - "epicgames": "generic", - "nintendo": "generic", - "origin": "generic", - "riot": "generic", - "sony": "generic", - "steam": "generic", - "uplay": "generic", - "wsus": "generic" - } + "combined_output": false, + "ips": { + "monolithic": ["10.10.10.200"] + }, + "cache_domains": { + "blizzard": "monolithic", + "epicgames": "monolithic", + "nintendo": "monolithic", + "origin": "monolithic", + "riot": "monolithic", + "sony": "monolithic", + "steam": "monolithic", + "uplay": "monolithic", + "wsus": "monolithic" + } } ``` 3. Run generation script relative to your DNS implementation: `bash create-dnsmasq.sh`. -4. Copy files from `output/{dnsmasq,unbound}/*` to the respective locations for Dnsmasq/Unbound. -5. Restart Dnsmasq or Unbound. +4. If `combined_output` is set to `true` this will result in a single output file: `lancache.conf` with all your enabled services (applies to Adguard Home, Dnsmasq or Unbound). +5. Copy files from `output/{adguardhome,dnsmasq,rpz,squid,unbound}/*` to the respective locations for Dnsmasq/Unbound. +6. Restart the appropriate service. ### Notes for Dnsmasq users @@ -44,5 +49,5 @@ Multi-IP Lancache setups are only supported with Dnsmasq or Pi-hole versions >= ### Notes for AdGuard Home users -1. In the `config.json`, you may want to add an entry for your non-cached DNS upstreams. You can input this in `ip.adguardhome_upstream` as an array. -2. Once you have ran the script, you can point the upstream list to the text file generated. For example: `upstream_dns_file: "/root/cache-domains/scripts/output/adguardhome/cache-domains.txt"` \ No newline at end of file +1. Utilising `"combined_output": true` is more convenient. +2. Once you have run the script and uploaded the file to the appropriate location, you should navigate to Adguard Home -> Filters -> DNS blocklists -> Add blocklist -> Add a custom list. diff --git a/scripts/config.example.json b/scripts/config.example.json index e1afdb5..e9d5b4f 100644 --- a/scripts/config.example.json +++ b/scripts/config.example.json @@ -1,20 +1,20 @@ { - "ips": { - "adguardhome_upstream": ["94.140.14.140", "tls://dns.google", "https://dns.google/dns-query"], - "steam": ["10.10.3.10", "10.10.3.11"], - "origin": "10.10.3.12", - "blizzard": "10.10.3.13", - "windows": "10.10.3.14", - "riot": "10.10.3.15", - "generic": "10.10.3.16" - }, - "cache_domains": { - "default": "generic", - "blizzard": "blizzard", - "origin": "origin", - "riot": "riot", - "steam": "steam", - "wsus": "windows", - "xboxlive": "windows" - } + "combined_output": false, + "ips": { + "steam": ["10.10.3.10", "10.10.3.11"], + "origin": "10.10.3.12", + "blizzard": "10.10.3.13", + "windows": "10.10.3.14", + "riot": "10.10.3.15", + "generic": "10.10.3.16" + }, + "cache_domains": { + "default": "generic", + "blizzard": "blizzard", + "origin": "origin", + "riot": "riot", + "steam": "steam", + "wsus": "windows", + "xboxlive": "windows" + } } diff --git a/scripts/create-adguardhome.sh b/scripts/create-adguardhome.sh old mode 100644 new mode 100755 index 352a8d8..abcd1c9 --- a/scripts/create-adguardhome.sh +++ b/scripts/create-adguardhome.sh @@ -6,72 +6,75 @@ path="${basedir}/cache_domains.json" export IFS=' ' test=$(which jq); -out=$? -if [ $out -gt 0 ] ; then - echo "This script requires jq to be installed." - echo "Your package manager should be able to find it" - exit 1 +if [ $? -gt 0 ] ; then + echo "This script requires jq to be installed." + echo "Your package manager should be able to find it" + exit 1 fi cachenamedefault="disabled" +combinedoutput=$(jq -r ".combined_output" config.json) -while read -r line; do - ip=$(jq ".ips[\"${line}\"]" config.json) - declare "cacheip${line}"="${ip}" +while read line; do + ip=$(jq ".ips[\"${line}\"]" config.json) + declare "cacheip${line}"="${ip}" done <<< $(jq -r '.ips | to_entries[] | .key' config.json) -agh_upstreams=$(jq -r ".ips[\"adguardhome_upstream\"] | .[]" config.json) - -while read -r line; do - name=$(jq -r ".cache_domains[\"${line}\"]" config.json) - declare "cachename${line}"="${name}" +while read line; do + name=$(jq -r ".cache_domains[\"${line}\"]" config.json) + declare "cachename$line"="$name" done <<< $(jq -r '.cache_domains | to_entries[] | .key' config.json) rm -rf ${outputdir} mkdir -p ${outputdir} - -# add upstreams -echo "${agh_upstreams}" >> "${outputdir}/cache-domains.txt" - -while read -r entry; do - unset cacheip - unset cachename - key=$(jq -r ".cache_domains[$entry].name" $path) - cachename="cachename${key}" - if [ -z "${!cachename}" ]; then - cachename="cachenamedefault" - fi - if [[ ${!cachename} == "disabled" ]]; then - continue; - fi - cacheipname="cacheip${!cachename}" - cacheip=$(jq -r 'if type == "array" then .[] else . end' <<< ${!cacheipname} | xargs) - while read -r fileid; do - while read -r filename; do - destfilename="cache-domains.txt" #$(echo $filename | sed -e 's/txt/conf/') - outputfile=${outputdir}/${destfilename} - touch ${outputfile} - while read -r fileentry; do - # Ignore comments, newlines and wildcards - if [[ ${fileentry} == \#* ]] || [[ -z ${fileentry} ]]; then - continue - fi - parsed=$(echo ${fileentry} | sed -e "s/^\*\.//") - for i in ${cacheip}; do - if grep -qx "\[/${parsed}/\]${i}" "${outputfile}"; then - continue - fi - echo "[/${parsed}/]${i}" >> "${outputfile}" - done - done <<< $(cat ${basedir}/${filename} | sort); - done <<< $(jq -r ".cache_domains[${entry}].domain_files[$fileid]" ${path}) - done <<< $(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path}) +while read entry; do + unset cacheip + unset cachename + key=$(jq -r ".cache_domains[$entry].name" ${path}) + cachename="cachename${key}" + if [ -z "${!cachename}" ]; then + cachename="cachenamedefault" + fi + if [[ ${!cachename} == "disabled" ]]; then + continue; + fi + cacheipname="cacheip${!cachename}" + cacheip=$(jq -r 'if type == "array" then .[] else . end' <<< ${!cacheipname} | xargs) + while read fileid; do + while read filename; do + destfilename=$(echo ${filename} | sed -e 's/txt/conf/') + outputfile=${outputdir}/${destfilename} + touch ${outputfile} + while read fileentry; do + # Ignore comments and newlines + if [[ ${fileentry} == \#* ]] || [[ -z ${fileentry} ]]; then + continue + fi + domainprefix="|" + if [[ $fileentry =~ ^\*\. ]]; then + domainprefix="||" + fi + parsed=$(echo ${fileentry} | sed -e "s/^\*\.//") + if grep -q "${domainprefix}${parsed}^\$dnsrewrite" ${outputfile}; then + continue + fi + for i in ${cacheip}; do + echo "${domainprefix}${parsed}^\$dnsrewrite=${i}" >> ${outputfile} + done + done <<< $(cat ${basedir}/$filename | sort); + done <<< $(jq -r ".cache_domains[${entry}].domain_files[${fileid}]" ${path}) + done <<< $(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path}) done <<< $(jq -r '.cache_domains | to_entries[] | .key' ${path}) +if [[ ${combinedoutput} == "true" ]]; then + for file in ${outputdir}/*; do f=${file//${outputdir}\/} && f=${f//.conf} && echo "# ${f^}" >> ${outputdir}/lancache.conf && cat ${file} >> ${outputdir}/lancache.conf && rm ${file}; done +fi + cat << EOF Configuration generation completed. -Please point the setting upstream_dns_file in AdGuardHome.yaml to the generated file. -For example: -upstream_dns_file: "/root/cache-domains/scripts/output/adguardhome/cache-domains.txt" -EOF \ No newline at end of file +Please copy the following files: +- ./${outputdir}/*.conf to /opt/adguardhome/work/userfilters/ +- Navigate to Adguard Home -> Filters -> DNS blocklists -> Add blocklist -> Add a custom list +- Add list for each service or utilise the combined output for a single list +EOF diff --git a/scripts/create-dnsmasq.sh b/scripts/create-dnsmasq.sh index 46483ac..49a6e25 100755 --- a/scripts/create-dnsmasq.sh +++ b/scripts/create-dnsmasq.sh @@ -6,14 +6,14 @@ path="${basedir}/cache_domains.json" export IFS=' ' test=$(which jq); -out=$? -if [ $out -gt 0 ] ; then - echo "This script requires jq to be installed." - echo "Your package manager should be able to find it" - exit 1 +if [ $? -gt 0 ] ; then + echo "This script requires jq to be installed." + echo "Your package manager should be able to find it" + exit 1 fi cachenamedefault="disabled" +combinedoutput=$(jq -r ".combined_output" config.json) while read -r line; do ip=$(jq ".ips[\"${line}\"]" config.json) @@ -28,42 +28,46 @@ done <<< $(jq -r '.cache_domains | to_entries[] | .key' config.json) rm -rf ${outputdir} mkdir -p ${outputdir} while read -r entry; do - unset cacheip - unset cachename - key=$(jq -r ".cache_domains[$entry].name" $path) - cachename="cachename${key}" - if [ -z "${!cachename}" ]; then - cachename="cachenamedefault" - fi - if [[ ${!cachename} == "disabled" ]]; then - continue; - fi - cacheipname="cacheip${!cachename}" - cacheip=$(jq -r 'if type == "array" then .[] else . end' <<< ${!cacheipname} | xargs) - while read -r fileid; do - while read -r filename; do - destfilename=$(echo $filename | sed -e 's/txt/conf/') - outputfile=${outputdir}/${destfilename} - touch ${outputfile} - while read -r fileentry; do - # Ignore comments, newlines and wildcards - if [[ ${fileentry} == \#* ]] || [[ -z ${fileentry} ]]; then - continue - fi - parsed=$(echo ${fileentry} | sed -e "s/^\*\.//") - for i in ${cacheip}; do - if ! grep -qx "address=/${parsed}/${i}" "${outputfile}"; then - echo "address=/${parsed}/${i}" >> "${outputfile}" - fi - if ! grep -qx "local=/${parsed}/" "${outputfile}"; then - echo "local=/${parsed}/" >> "${outputfile}" - fi - done - done <<< $(cat ${basedir}/${filename} | sort); - done <<< $(jq -r ".cache_domains[${entry}].domain_files[$fileid]" ${path}) - done <<< $(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path}) + unset cacheip + unset cachename + key=$(jq -r ".cache_domains[${entry}].name" ${path}) + cachename="cachename${key}" + if [ -z "${!cachename}" ]; then + cachename="cachenamedefault" + fi + if [[ ${!cachename} == "disabled" ]]; then + continue; + fi + cacheipname="cacheip${!cachename}" + cacheip=$(jq -r 'if type == "array" then .[] else . end' <<< ${!cacheipname} | xargs) + while read -r fileid; do + while read -r filename; do + destfilename=$(echo ${filename} | sed -e 's/txt/conf/') + outputfile=${outputdir}/${destfilename} + touch ${outputfile} + while read -r fileentry; do + # Ignore comments, newlines and wildcards + if [[ ${fileentry} == \#* ]] || [[ -z ${fileentry} ]]; then + continue + fi + parsed=$(echo ${fileentry} | sed -e "s/^\*\.//") + for i in ${cacheip}; do + if ! grep -qx "address=/${parsed}/${i}" "${outputfile}"; then + echo "address=/${parsed}/${i}" >> "${outputfile}" + fi + if ! grep -qx "local=/${parsed}/" "${outputfile}"; then + echo "local=/${parsed}/" >> "${outputfile}" + fi + done + done <<< $(cat ${basedir}/${filename} | sort); + done <<< $(jq -r ".cache_domains[${entry}].domain_files[$fileid]" ${path}) + done <<< $(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path}) done <<< $(jq -r '.cache_domains | to_entries[] | .key' ${path}) +if [[ ${combinedoutput} == "true" ]]; then + for file in ${outputdir}/*; do f=${file//${outputdir}\/} && f=${f//.conf} && echo "# ${f^}" >> ${outputdir}/lancache.conf && cat ${file} >> ${outputdir}/lancache.conf && rm ${file}; done +fi + cat << EOF Configuration generation completed. diff --git a/scripts/create-rpz.sh b/scripts/create-rpz.sh index 95cc457..a2bd94f 100755 --- a/scripts/create-rpz.sh +++ b/scripts/create-rpz.sh @@ -7,108 +7,107 @@ basedomain=${1:-lancache.net} export IFS=' ' test=$(which jq); -out=$? -if [ $out -gt 0 ] ; then - echo "This script requires jq to be installed." - echo "Your package manager should be able to find it" - exit 1 +if [ $? -gt 0 ] ; then + echo "This script requires jq to be installed." + echo "Your package manager should be able to find it" + exit 1 fi cachenamedefault="disabled" while read line; do - ip=$(jq ".ips[\"${line}\"]" config.json) - declare "cacheip$line"="$ip" + ip=$(jq ".ips[\"${line}\"]" config.json) + declare "cacheip${line}"="${ip}" done <<< $(jq -r '.ips | to_entries[] | .key' config.json) while read line; do - name=$(jq -r ".cache_domains[\"${line}\"]" config.json) - declare "cachename$line"="$name" + name=$(jq -r ".cache_domains[\"${line}\"]" config.json) + declare "cachename${line}"="${name}" done <<< $(jq -r '.cache_domains | to_entries[] | .key' config.json) rm -rf ${outputdir} mkdir -p ${outputdir} -outputfile=${outputdir}/db.rpz.$basedomain -cat > $outputfile << EOF +outputfile=${outputdir}/db.rpz.${basedomain} +cat > ${outputfile} << EOF \$TTL 60 ; default TTL -\$ORIGIN rpz.$basedomain. -@ SOA ns1.$basedomain. admin.$basedomain. ( +\$ORIGIN rpz.${basedomain}. +@ SOA ns1.${basedomain}. admin.${basedomain}. ( $(date +%Y%m%d01) ; serial 604800 ; refresh (1 week) 600 ; retry (10 mins) 600 ; expire (10 mins) 600 ; minimum (10 mins) ) - NS ns1.$basedomain. - NS ns2.$basedomain. + NS ns1.${basedomain}. + NS ns2.${basedomain}. EOF while read entry; do - unset cacheip - unset cachename - key=$(jq -r ".cache_domains[$entry].name" $path) - cachename="cachename${key}" - if [ -z "${!cachename}" ]; then - cachename="cachenamedefault" - fi - if [[ ${!cachename} == "disabled" ]]; then - continue; - fi - cacheipname="cacheip${!cachename}" - cacheip=$(jq -r 'if type == "array" then .[] else . end' <<< ${!cacheipname} | xargs) - while read fileid; do - while read filename; do - echo "" >> $outputfile - echo "; $(echo $filename | sed -e 's/.txt$//')" >> $outputfile - destfilename=$(echo $filename | sed -e 's/txt/conf/') - while read fileentry; do - # Ignore comments and newlines - if [[ $fileentry == \#* ]] || [[ -z $fileentry ]]; then - continue - fi - parsed=$(echo $fileentry) - if grep -qx "^\"${parsed}\". " $outputfile; then - continue - fi - t="" - for i in ${cacheip}; do - # only one cname per domain is allowed - if [[ ${t} = "CNAME" ]]; then - continue - fi - # for cnames you must use a fqdn with trailing dot - t="CNAME" - if [[ ${i} =~ ^[0-9\.]+$ ]] ; then - t="A" - elif [[ ! ${i} =~ \.$ ]] ; then - i="${i}." - fi - printf "%-50s IN %s %s\n" \ - "${parsed}" \ - "${t}" \ - "${i}" \ - >> $outputfile - done - done <<< $(cat ${basedir}/$filename | sort); - done <<< $(jq -r ".cache_domains[$entry].domain_files[$fileid]" $path) - done <<< $(jq -r ".cache_domains[$entry].domain_files | to_entries[] | .key" $path) -done <<< $(jq -r '.cache_domains | to_entries[] | .key' $path) + unset cacheip + unset cachename + key=$(jq -r ".cache_domains[${entry}].name" ${path}) + cachename="cachename${key}" + if [ -z "${!cachename}" ]; then + cachename="cachenamedefault" + fi + if [[ ${!cachename} == "disabled" ]]; then + continue; + fi + cacheipname="cacheip${!cachename}" + cacheip=$(jq -r 'if type == "array" then .[] else . end' <<< ${!cacheipname} | xargs) + while read fileid; do + while read filename; do + echo "" >> ${outputfile} + echo "; $(echo ${filename} | sed -e 's/.txt$//')" >> ${outputfile} + destfilename=$(echo ${filename} | sed -e 's/txt/conf/') + while read fileentry; do + # Ignore comments and newlines + if [[ ${fileentry} == \#* ]] || [[ -z ${fileentry} ]]; then + continue + fi + parsed=$(echo ${fileentry}) + if grep -qx "^\"${parsed}\". " ${outputfile}; then + continue + fi + t="" + for i in ${cacheip}; do + # only one cname per domain is allowed + if [[ ${t} = "CNAME" ]]; then + continue + fi + # for cnames you must use a fqdn with trailing dot + t="CNAME" + if [[ ${i} =~ ^[0-9\.]+$ ]] ; then + t="A" + elif [[ ! ${i} =~ \.$ ]] ; then + i="${i}." + fi + printf "%-50s IN %s %s\n" \ + "${parsed}" \ + "${t}" \ + "${i}" \ + >> ${outputfile} + done + done <<< $(cat ${basedir}/${filename} | sort); + done <<< $(jq -r ".cache_domains[${entry}].domain_files[${fileid}]" ${path}) + done <<< $(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path}) +done <<< $(jq -r '.cache_domains | to_entries[] | .key' ${path}) cat << EOF Configuration generation completed. Please include the rpz zone in your bind configuration" -- cp $outputfile /etc/bind +- cp ${outputfile} /etc/bind - configure the zone and use it options { [...] - response-policy {zone "rpz.$basedomain";}; + response-policy {zone "rpz.${basedomain}";}; [...] } zone "rpz.$basedomain" { type master; - file "/etc/bind/db.rpz.$basedomain"; + file "/etc/bind/db.rpz.${basedomain}"; }; EOF diff --git a/scripts/create-squid.sh b/scripts/create-squid.sh index f98f042..9607ccb 100755 --- a/scripts/create-squid.sh +++ b/scripts/create-squid.sh @@ -7,61 +7,59 @@ REGEX="^\\*\\.(.*)$" export IFS=' ' test=$(which jq); -out=$? -if [ $out -gt 0 ] ; then - echo "This script requires jq to be installed." - echo "Your package manager should be able to find it" - exit 1 +if [ $? -gt 0 ] ; then + echo "This script requires jq to be installed." + echo "Your package manager should be able to find it" + exit 1 fi cachenamedefault="disabled" while read -r line; do - name=$(jq -r ".cache_domains[\"${line}\"]" config.json) - declare "cachename${line}"="${name}" + name=$(jq -r ".cache_domains[\"${line}\"]" config.json) + declare "cachename${line}"="${name}" done <<< $(jq -r '.cache_domains | to_entries[] | .key' config.json) rm -rf ${outputdir} mkdir -p ${outputdir} while read -r entry; do - unset cachename - key=$(jq -r ".cache_domains[$entry].name" $path) - cachename="cachename${key}" - if [ -z "${!cachename}" ]; then - cachename="cachenamedefault" - fi - if [[ ${!cachename} == "disabled" ]]; then - continue; - fi - while read -r fileid; do - while read -r filename; do - destfilename=$(echo ${!cachename}.txt) - outputfile=${outputdir}/${destfilename} - touch ${outputfile} - while read -r fileentry; do - # Ignore comments - if [[ ${fileentry} == \#* ]] || [[ -z ${fileentry} ]]; then - continue - fi - # Handle wildcards to squid wildcards - parsed=$(echo ${fileentry} | sed -e "s/^\*\./\./") - # If we have cdn.thing and *.cdn.thing in cache_domains - # Squid requires ONLY cdn.thing - # - # If the fileentry starts with *.cdn.thing - if [[ ${fileentry} =~ $REGEX ]]; then - # Does the cache_domains file also contain cdn.thing - grep "${BASH_REMATCH[1]}" ${basedir}/${filename} | grep -v "${fileentry}" > /dev/null - if [[ $? -eq 0 ]]; then - # Skip *.cdn.thing as cdn.thing will be collected earlier/later - continue - fi - fi - - echo "${parsed}" >> "${outputfile}" - done <<< $(cat ${basedir}/${filename} | sort); - done <<< $(jq -r ".cache_domains[${entry}].domain_files[$fileid]" ${path}) - done <<< $(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path}) + unset cachename + key=$(jq -r ".cache_domains[$entry].name" ${path}) + cachename="cachename${key}" + if [ -z "${!cachename}" ]; then + cachename="cachenamedefault" + fi + if [[ ${!cachename} == "disabled" ]]; then + continue; + fi + while read -r fileid; do + while read -r filename; do + destfilename=$(echo ${!cachename}.txt) + outputfile=${outputdir}/${destfilename} + touch ${outputfile} + while read -r fileentry; do + # Ignore comments + if [[ ${fileentry} == \#* ]] || [[ -z ${fileentry} ]]; then + continue + fi + # Ha wildcards to squid wildcards + parsed=$(echo ${fileentry} | sed -e "s/^\*\./\./") + # If we have cdn.thing and *.cdn.thing in cache_domains + # Squid requires ONLY cdn.thing + # + # If the fileentry starts with *.cdn.thing + if [[ ${fileentry} =~ $REGEX ]]; then + # Does the cache_domains file also contain cdn.thing + grep "${BASH_REMATCH[1]}" ${basedir}/${filename} | grep -v "${fileentry}" > /dev/null + if [[ $? -eq 0 ]]; then + # Skip *.cdn.thing as cdn.thing will be collected earlier/later + continue + fi + fi + echo "${parsed}" >> "${outputfile}" + done <<< $(cat ${basedir}/${filename} | sort); + done <<< $(jq -r ".cache_domains[${entry}].domain_files[${fileid}]" ${path}) + done <<< $(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path}) done <<< $(jq -r '.cache_domains | to_entries[] | .key' ${path}) cat << EOF diff --git a/scripts/create-unbound.sh b/scripts/create-unbound.sh index e147adc..52de21b 100755 --- a/scripts/create-unbound.sh +++ b/scripts/create-unbound.sh @@ -6,65 +6,69 @@ path="${basedir}/cache_domains.json" export IFS=' ' test=$(which jq); -out=$? -if [ $out -gt 0 ] ; then - echo "This script requires jq to be installed." - echo "Your package manager should be able to find it" - exit 1 +if [ $? -gt 0 ] ; then + echo "This script requires jq to be installed." + echo "Your package manager should be able to find it" + exit 1 fi cachenamedefault="disabled" +combinedoutput=$(jq -r ".combined_output" config.json) -while read line; do - ip=$(jq ".ips[\"${line}\"]" config.json) - declare "cacheip$line"="$ip" +while read line; do + ip=$(jq ".ips[\"${line}\"]" config.json) + declare "cacheip${line}"="${ip}" done <<< $(jq -r '.ips | to_entries[] | .key' config.json) -while read line; do - name=$(jq -r ".cache_domains[\"${line}\"]" config.json) - declare "cachename$line"="$name" +while read line; do + name=$(jq -r ".cache_domains[\"${line}\"]" config.json) + declare "cachename${line}"="${name}" done <<< $(jq -r '.cache_domains | to_entries[] | .key' config.json) rm -rf ${outputdir} mkdir -p ${outputdir} -while read entry; do - unset cacheip - unset cachename - key=$(jq -r ".cache_domains[$entry].name" $path) - cachename="cachename${key}" - if [ -z "${!cachename}" ]; then - cachename="cachenamedefault" - fi - if [[ ${!cachename} == "disabled" ]]; then - continue; - fi - cacheipname="cacheip${!cachename}" - cacheip=$(jq -r 'if type == "array" then .[] else . end' <<< ${!cacheipname} | xargs) - while read fileid; do - while read filename; do - destfilename=$(echo $filename | sed -e 's/txt/conf/') - outputfile=${outputdir}/${destfilename} - touch $outputfile - while read fileentry; do - # Ignore comments and newlines - if [[ $fileentry == \#* ]] || [[ -z $fileentry ]]; then - continue - fi - parsed=$(echo $fileentry | sed -e "s/^\*\.//") - if grep -qx " local-zone: \"${parsed}\" redirect" $outputfile; then - continue - fi - if [[ $(head -n 1 $outputfile) != "server:" ]]; then - echo "server:" >> $outputfile - fi - echo " local-zone: \"${parsed}\" redirect" >> $outputfile - for i in ${cacheip}; do - echo " local-data: \"${parsed} 30 IN A ${i}\"" >> $outputfile - done - done <<< $(cat ${basedir}/$filename | sort); - done <<< $(jq -r ".cache_domains[$entry].domain_files[$fileid]" $path) - done <<< $(jq -r ".cache_domains[$entry].domain_files | to_entries[] | .key" $path) -done <<< $(jq -r '.cache_domains | to_entries[] | .key' $path) +while read entry; do + unset cacheip + unset cachename + key=$(jq -r ".cache_domains[${entry}].name" ${path}) + cachename="cachename${key}" + if [ -z "${!cachename}" ]; then + cachename="cachenamedefault" + fi + if [[ ${!cachename} == "disabled" ]]; then + continue; + fi + cacheipname="cacheip${!cachename}" + cacheip=$(jq -r 'if type == "array" then .[] else . end' <<< ${!cacheipname} | xargs) + while read fileid; do + while read filename; do + destfilename=$(echo ${filename} | sed -e 's/txt/conf/') + outputfile=${outputdir}/${destfilename} + touch ${outputfile} + while read fileentry; do + # Ignore comments and newlines + if [[ ${fileentry} == \#* ]] || [[ -z ${fileentry} ]]; then + continue + fi + parsed=$(echo ${fileentry} | sed -e "s/^\*\.//") + if grep -qx " local-zone: \"${parsed}\" redirect" ${outputfile}; then + continue + fi + if [[ $(head -n 1 ${outputfile}) != "server:" ]]; then + echo "server:" >> ${outputfile} + fi + echo " local-zone: \"${parsed}\" redirect" >> ${outputfile} + for i in ${cacheip}; do + echo " local-data: \"${parsed} 30 IN A ${i}\"" >> ${outputfile} + done + done <<< $(cat ${basedir}/${filename} | sort); + done <<< $(jq -r ".cache_domains[${entry}].domain_files[${fileid}]" ${path}) + done <<< $(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path}) +done <<< $(jq -r '.cache_domains | to_entries[] | .key' ${path}) + +if [[ ${combinedoutput} == "true" ]]; then + for file in ${outputdir}/*; do f=${file//${outputdir}\/} && f=${f//.conf} && echo "# ${f^}" >> ${outputdir}/lancache.conf && cat ${file} >> ${outputdir}/lancache.conf && rm ${file}; done +fi cat << EOF Configuration generation completed.