Update scripts and add combined_output flag (#251)

* Update scripts and add combined_output flag

* Add editorconfig to enforcing formatting requirements

* Adjust generic references to monolithic
This commit is contained in:
Amir Zarrinkafsh 2024-10-25 19:13:50 +11:00 committed by GitHub
parent 7fbb21e32c
commit 67594ce10c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 324 additions and 303 deletions

8
.editorconfig Normal file
View File

@ -0,0 +1,8 @@
root = true
[*]
indent_style = space
indent_size = 4
trim_trailing_whitespace = true
end_of_line = lf
insert_final_newline = true

View File

@ -5,36 +5,41 @@
The respective shell scripts contained within this directory can be utilised to generate application specific compliant
configuration which can be utilised with:
* Dnsmasq
* Unbound
* AdGuard Home
* BIND9
* Dnsmasq/Pi-hole
* Squid
* Unbound
## Usage
1. Copy `config.example.json` to `config.json`.
2. Modify `config.json` to include your Cacheserver's IP(s) and the CDNs you plan to cache.
The following example assumes a single shared Cacheserver IP:
```json
{
"combined_output": false,
"ips": {
"generic": ["10.10.10.200"]
"monolithic": ["10.10.10.200"]
},
"cache_domains": {
"blizzard": "generic",
"epicgames": "generic",
"nintendo": "generic",
"origin": "generic",
"riot": "generic",
"sony": "generic",
"steam": "generic",
"uplay": "generic",
"wsus": "generic"
"blizzard": "monolithic",
"epicgames": "monolithic",
"nintendo": "monolithic",
"origin": "monolithic",
"riot": "monolithic",
"sony": "monolithic",
"steam": "monolithic",
"uplay": "monolithic",
"wsus": "monolithic"
}
}
```
3. Run generation script relative to your DNS implementation: `bash create-dnsmasq.sh`.
4. Copy files from `output/{dnsmasq,unbound}/*` to the respective locations for Dnsmasq/Unbound.
5. Restart Dnsmasq or Unbound.
4. If `combined_output` is set to `true` this will result in a single output file: `lancache.conf` with all your enabled services (applies to Adguard Home, Dnsmasq or Unbound).
5. Copy files from `output/{adguardhome,dnsmasq,rpz,squid,unbound}/*` to the respective locations for Dnsmasq/Unbound.
6. Restart the appropriate service.
### Notes for Dnsmasq users
@ -44,5 +49,5 @@ Multi-IP Lancache setups are only supported with Dnsmasq or Pi-hole versions >=
### Notes for AdGuard Home users
1. In the `config.json`, you may want to add an entry for your non-cached DNS upstreams. You can input this in `ip.adguardhome_upstream` as an array.
2. Once you have ran the script, you can point the upstream list to the text file generated. For example: `upstream_dns_file: "/root/cache-domains/scripts/output/adguardhome/cache-domains.txt"`
1. Utilising `"combined_output": true` is more convenient.
2. Once you have run the script and uploaded the file to the appropriate location, you should navigate to Adguard Home -> Filters -> DNS blocklists -> Add blocklist -> Add a custom list.

View File

@ -1,6 +1,6 @@
{
"combined_output": false,
"ips": {
"adguardhome_upstream": ["94.140.14.140", "tls://dns.google", "https://dns.google/dns-query"],
"steam": ["10.10.3.10", "10.10.3.11"],
"origin": "10.10.3.12",
"blizzard": "10.10.3.13",

55
scripts/create-adguardhome.sh Normal file → Executable file
View File

@ -6,37 +6,31 @@ path="${basedir}/cache_domains.json"
export IFS=' '
test=$(which jq);
out=$?
if [ $out -gt 0 ] ; then
if [ $? -gt 0 ] ; then
echo "This script requires jq to be installed."
echo "Your package manager should be able to find it"
exit 1
fi
cachenamedefault="disabled"
combinedoutput=$(jq -r ".combined_output" config.json)
while read -r line; do
while read line; do
ip=$(jq ".ips[\"${line}\"]" config.json)
declare "cacheip${line}"="${ip}"
done <<< $(jq -r '.ips | to_entries[] | .key' config.json)
agh_upstreams=$(jq -r ".ips[\"adguardhome_upstream\"] | .[]" config.json)
while read -r line; do
while read line; do
name=$(jq -r ".cache_domains[\"${line}\"]" config.json)
declare "cachename${line}"="${name}"
declare "cachename$line"="$name"
done <<< $(jq -r '.cache_domains | to_entries[] | .key' config.json)
rm -rf ${outputdir}
mkdir -p ${outputdir}
# add upstreams
echo "${agh_upstreams}" >> "${outputdir}/cache-domains.txt"
while read -r entry; do
while read entry; do
unset cacheip
unset cachename
key=$(jq -r ".cache_domains[$entry].name" $path)
key=$(jq -r ".cache_domains[$entry].name" ${path})
cachename="cachename${key}"
if [ -z "${!cachename}" ]; then
cachename="cachenamedefault"
@ -46,32 +40,41 @@ while read -r entry; do
fi
cacheipname="cacheip${!cachename}"
cacheip=$(jq -r 'if type == "array" then .[] else . end' <<< ${!cacheipname} | xargs)
while read -r fileid; do
while read -r filename; do
destfilename="cache-domains.txt" #$(echo $filename | sed -e 's/txt/conf/')
while read fileid; do
while read filename; do
destfilename=$(echo ${filename} | sed -e 's/txt/conf/')
outputfile=${outputdir}/${destfilename}
touch ${outputfile}
while read -r fileentry; do
# Ignore comments, newlines and wildcards
while read fileentry; do
# Ignore comments and newlines
if [[ ${fileentry} == \#* ]] || [[ -z ${fileentry} ]]; then
continue
fi
domainprefix="|"
if [[ $fileentry =~ ^\*\. ]]; then
domainprefix="||"
fi
parsed=$(echo ${fileentry} | sed -e "s/^\*\.//")
for i in ${cacheip}; do
if grep -qx "\[/${parsed}/\]${i}" "${outputfile}"; then
if grep -q "${domainprefix}${parsed}^\$dnsrewrite" ${outputfile}; then
continue
fi
echo "[/${parsed}/]${i}" >> "${outputfile}"
for i in ${cacheip}; do
echo "${domainprefix}${parsed}^\$dnsrewrite=${i}" >> ${outputfile}
done
done <<< $(cat ${basedir}/${filename} | sort);
done <<< $(jq -r ".cache_domains[${entry}].domain_files[$fileid]" ${path})
done <<< $(cat ${basedir}/$filename | sort);
done <<< $(jq -r ".cache_domains[${entry}].domain_files[${fileid}]" ${path})
done <<< $(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path})
done <<< $(jq -r '.cache_domains | to_entries[] | .key' ${path})
if [[ ${combinedoutput} == "true" ]]; then
for file in ${outputdir}/*; do f=${file//${outputdir}\/} && f=${f//.conf} && echo "# ${f^}" >> ${outputdir}/lancache.conf && cat ${file} >> ${outputdir}/lancache.conf && rm ${file}; done
fi
cat << EOF
Configuration generation completed.
Please point the setting upstream_dns_file in AdGuardHome.yaml to the generated file.
For example:
upstream_dns_file: "/root/cache-domains/scripts/output/adguardhome/cache-domains.txt"
Please copy the following files:
- ./${outputdir}/*.conf to /opt/adguardhome/work/userfilters/
- Navigate to Adguard Home -> Filters -> DNS blocklists -> Add blocklist -> Add a custom list
- Add list for each service or utilise the combined output for a single list
EOF

View File

@ -6,14 +6,14 @@ path="${basedir}/cache_domains.json"
export IFS=' '
test=$(which jq);
out=$?
if [ $out -gt 0 ] ; then
if [ $? -gt 0 ] ; then
echo "This script requires jq to be installed."
echo "Your package manager should be able to find it"
exit 1
fi
cachenamedefault="disabled"
combinedoutput=$(jq -r ".combined_output" config.json)
while read -r line; do
ip=$(jq ".ips[\"${line}\"]" config.json)
@ -30,7 +30,7 @@ mkdir -p ${outputdir}
while read -r entry; do
unset cacheip
unset cachename
key=$(jq -r ".cache_domains[$entry].name" $path)
key=$(jq -r ".cache_domains[${entry}].name" ${path})
cachename="cachename${key}"
if [ -z "${!cachename}" ]; then
cachename="cachenamedefault"
@ -42,7 +42,7 @@ while read -r entry; do
cacheip=$(jq -r 'if type == "array" then .[] else . end' <<< ${!cacheipname} | xargs)
while read -r fileid; do
while read -r filename; do
destfilename=$(echo $filename | sed -e 's/txt/conf/')
destfilename=$(echo ${filename} | sed -e 's/txt/conf/')
outputfile=${outputdir}/${destfilename}
touch ${outputfile}
while read -r fileentry; do
@ -64,6 +64,10 @@ while read -r entry; do
done <<< $(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path})
done <<< $(jq -r '.cache_domains | to_entries[] | .key' ${path})
if [[ ${combinedoutput} == "true" ]]; then
for file in ${outputdir}/*; do f=${file//${outputdir}\/} && f=${f//.conf} && echo "# ${f^}" >> ${outputdir}/lancache.conf && cat ${file} >> ${outputdir}/lancache.conf && rm ${file}; done
fi
cat << EOF
Configuration generation completed.

View File

@ -7,8 +7,7 @@ basedomain=${1:-lancache.net}
export IFS=' '
test=$(which jq);
out=$?
if [ $out -gt 0 ] ; then
if [ $? -gt 0 ] ; then
echo "This script requires jq to be installed."
echo "Your package manager should be able to find it"
exit 1
@ -18,36 +17,36 @@ cachenamedefault="disabled"
while read line; do
ip=$(jq ".ips[\"${line}\"]" config.json)
declare "cacheip$line"="$ip"
declare "cacheip${line}"="${ip}"
done <<< $(jq -r '.ips | to_entries[] | .key' config.json)
while read line; do
name=$(jq -r ".cache_domains[\"${line}\"]" config.json)
declare "cachename$line"="$name"
declare "cachename${line}"="${name}"
done <<< $(jq -r '.cache_domains | to_entries[] | .key' config.json)
rm -rf ${outputdir}
mkdir -p ${outputdir}
outputfile=${outputdir}/db.rpz.$basedomain
cat > $outputfile << EOF
outputfile=${outputdir}/db.rpz.${basedomain}
cat > ${outputfile} << EOF
\$TTL 60 ; default TTL
\$ORIGIN rpz.$basedomain.
@ SOA ns1.$basedomain. admin.$basedomain. (
\$ORIGIN rpz.${basedomain}.
@ SOA ns1.${basedomain}. admin.${basedomain}. (
$(date +%Y%m%d01) ; serial
604800 ; refresh (1 week)
600 ; retry (10 mins)
600 ; expire (10 mins)
600 ; minimum (10 mins)
)
NS ns1.$basedomain.
NS ns2.$basedomain.
NS ns1.${basedomain}.
NS ns2.${basedomain}.
EOF
while read entry; do
unset cacheip
unset cachename
key=$(jq -r ".cache_domains[$entry].name" $path)
key=$(jq -r ".cache_domains[${entry}].name" ${path})
cachename="cachename${key}"
if [ -z "${!cachename}" ]; then
cachename="cachenamedefault"
@ -59,16 +58,16 @@ while read entry; do
cacheip=$(jq -r 'if type == "array" then .[] else . end' <<< ${!cacheipname} | xargs)
while read fileid; do
while read filename; do
echo "" >> $outputfile
echo "; $(echo $filename | sed -e 's/.txt$//')" >> $outputfile
destfilename=$(echo $filename | sed -e 's/txt/conf/')
echo "" >> ${outputfile}
echo "; $(echo ${filename} | sed -e 's/.txt$//')" >> ${outputfile}
destfilename=$(echo ${filename} | sed -e 's/txt/conf/')
while read fileentry; do
# Ignore comments and newlines
if [[ $fileentry == \#* ]] || [[ -z $fileentry ]]; then
if [[ ${fileentry} == \#* ]] || [[ -z ${fileentry} ]]; then
continue
fi
parsed=$(echo $fileentry)
if grep -qx "^\"${parsed}\". " $outputfile; then
parsed=$(echo ${fileentry})
if grep -qx "^\"${parsed}\". " ${outputfile}; then
continue
fi
t=""
@ -88,27 +87,27 @@ while read entry; do
"${parsed}" \
"${t}" \
"${i}" \
>> $outputfile
>> ${outputfile}
done
done <<< $(cat ${basedir}/$filename | sort);
done <<< $(jq -r ".cache_domains[$entry].domain_files[$fileid]" $path)
done <<< $(jq -r ".cache_domains[$entry].domain_files | to_entries[] | .key" $path)
done <<< $(jq -r '.cache_domains | to_entries[] | .key' $path)
done <<< $(cat ${basedir}/${filename} | sort);
done <<< $(jq -r ".cache_domains[${entry}].domain_files[${fileid}]" ${path})
done <<< $(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path})
done <<< $(jq -r '.cache_domains | to_entries[] | .key' ${path})
cat << EOF
Configuration generation completed.
Please include the rpz zone in your bind configuration"
- cp $outputfile /etc/bind
- cp ${outputfile} /etc/bind
- configure the zone and use it
options {
[...]
response-policy {zone "rpz.$basedomain";};
response-policy {zone "rpz.${basedomain}";};
[...]
}
zone "rpz.$basedomain" {
type master;
file "/etc/bind/db.rpz.$basedomain";
file "/etc/bind/db.rpz.${basedomain}";
};
EOF

View File

@ -7,8 +7,7 @@ REGEX="^\\*\\.(.*)$"
export IFS=' '
test=$(which jq);
out=$?
if [ $out -gt 0 ] ; then
if [ $? -gt 0 ] ; then
echo "This script requires jq to be installed."
echo "Your package manager should be able to find it"
exit 1
@ -25,7 +24,7 @@ rm -rf ${outputdir}
mkdir -p ${outputdir}
while read -r entry; do
unset cachename
key=$(jq -r ".cache_domains[$entry].name" $path)
key=$(jq -r ".cache_domains[$entry].name" ${path})
cachename="cachename${key}"
if [ -z "${!cachename}" ]; then
cachename="cachenamedefault"
@ -43,7 +42,7 @@ while read -r entry; do
if [[ ${fileentry} == \#* ]] || [[ -z ${fileentry} ]]; then
continue
fi
# Handle wildcards to squid wildcards
# Ha wildcards to squid wildcards
parsed=$(echo ${fileentry} | sed -e "s/^\*\./\./")
# If we have cdn.thing and *.cdn.thing in cache_domains
# Squid requires ONLY cdn.thing
@ -57,10 +56,9 @@ while read -r entry; do
continue
fi
fi
echo "${parsed}" >> "${outputfile}"
done <<< $(cat ${basedir}/${filename} | sort);
done <<< $(jq -r ".cache_domains[${entry}].domain_files[$fileid]" ${path})
done <<< $(jq -r ".cache_domains[${entry}].domain_files[${fileid}]" ${path})
done <<< $(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path})
done <<< $(jq -r '.cache_domains | to_entries[] | .key' ${path})

View File

@ -6,23 +6,23 @@ path="${basedir}/cache_domains.json"
export IFS=' '
test=$(which jq);
out=$?
if [ $out -gt 0 ] ; then
if [ $? -gt 0 ] ; then
echo "This script requires jq to be installed."
echo "Your package manager should be able to find it"
exit 1
fi
cachenamedefault="disabled"
combinedoutput=$(jq -r ".combined_output" config.json)
while read line; do
ip=$(jq ".ips[\"${line}\"]" config.json)
declare "cacheip$line"="$ip"
declare "cacheip${line}"="${ip}"
done <<< $(jq -r '.ips | to_entries[] | .key' config.json)
while read line; do
name=$(jq -r ".cache_domains[\"${line}\"]" config.json)
declare "cachename$line"="$name"
declare "cachename${line}"="${name}"
done <<< $(jq -r '.cache_domains | to_entries[] | .key' config.json)
rm -rf ${outputdir}
@ -30,7 +30,7 @@ mkdir -p ${outputdir}
while read entry; do
unset cacheip
unset cachename
key=$(jq -r ".cache_domains[$entry].name" $path)
key=$(jq -r ".cache_domains[${entry}].name" ${path})
cachename="cachename${key}"
if [ -z "${!cachename}" ]; then
cachename="cachenamedefault"
@ -42,29 +42,33 @@ while read entry; do
cacheip=$(jq -r 'if type == "array" then .[] else . end' <<< ${!cacheipname} | xargs)
while read fileid; do
while read filename; do
destfilename=$(echo $filename | sed -e 's/txt/conf/')
destfilename=$(echo ${filename} | sed -e 's/txt/conf/')
outputfile=${outputdir}/${destfilename}
touch $outputfile
touch ${outputfile}
while read fileentry; do
# Ignore comments and newlines
if [[ $fileentry == \#* ]] || [[ -z $fileentry ]]; then
if [[ ${fileentry} == \#* ]] || [[ -z ${fileentry} ]]; then
continue
fi
parsed=$(echo $fileentry | sed -e "s/^\*\.//")
if grep -qx " local-zone: \"${parsed}\" redirect" $outputfile; then
parsed=$(echo ${fileentry} | sed -e "s/^\*\.//")
if grep -qx " local-zone: \"${parsed}\" redirect" ${outputfile}; then
continue
fi
if [[ $(head -n 1 $outputfile) != "server:" ]]; then
echo "server:" >> $outputfile
if [[ $(head -n 1 ${outputfile}) != "server:" ]]; then
echo "server:" >> ${outputfile}
fi
echo " local-zone: \"${parsed}\" redirect" >> $outputfile
echo " local-zone: \"${parsed}\" redirect" >> ${outputfile}
for i in ${cacheip}; do
echo " local-data: \"${parsed} 30 IN A ${i}\"" >> $outputfile
echo " local-data: \"${parsed} 30 IN A ${i}\"" >> ${outputfile}
done
done <<< $(cat ${basedir}/$filename | sort);
done <<< $(jq -r ".cache_domains[$entry].domain_files[$fileid]" $path)
done <<< $(jq -r ".cache_domains[$entry].domain_files | to_entries[] | .key" $path)
done <<< $(jq -r '.cache_domains | to_entries[] | .key' $path)
done <<< $(cat ${basedir}/${filename} | sort);
done <<< $(jq -r ".cache_domains[${entry}].domain_files[${fileid}]" ${path})
done <<< $(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path})
done <<< $(jq -r '.cache_domains | to_entries[] | .key' ${path})
if [[ ${combinedoutput} == "true" ]]; then
for file in ${outputdir}/*; do f=${file//${outputdir}\/} && f=${f//.conf} && echo "# ${f^}" >> ${outputdir}/lancache.conf && cat ${file} >> ${outputdir}/lancache.conf && rm ${file}; done
fi
cat << EOF
Configuration generation completed.