diff --git a/.editorconfig b/.editorconfig index 2e62126..e028911 100644 --- a/.editorconfig +++ b/.editorconfig @@ -2,7 +2,13 @@ root = true [*] indent_style = space -indent_size = 4 +indent_size = 2 trim_trailing_whitespace = true end_of_line = lf insert_final_newline = true + +[*.md] +indent_size = 0 + +[*.sh] +indent_style = tab \ No newline at end of file diff --git a/.github/workflows/mega-linter.yml b/.github/workflows/mega-linter.yml new file mode 100644 index 0000000..ebecca6 --- /dev/null +++ b/.github/workflows/mega-linter.yml @@ -0,0 +1,55 @@ +# MegaLinter GitHub Action configuration file +# More info at https://megalinter.io +--- +name: MegaLinter + +on: + pull_request: + branches: + - master + +env: + GITHUB_STATUS_REPORTER: true + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + megalinter: + name: MegaLinter + runs-on: ubuntu-latest + + permissions: + contents: write + issues: write + pull-requests: write + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + with: + token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }} + fetch-depth: 0 + + - name: MegaLinter + uses: oxsecurity/megalinter@v8 + id: ml + env: + VALIDATE_ALL_CODEBASE: >- + ${{ + github.event_name == 'push' && + github.ref == 'refs/heads/master' + }} + + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Archive production artifacts + uses: actions/upload-artifact@v4 + if: success() || failure() + with: + name: MegaLinter reports + include-hidden-files: "true" + path: | + megalinter-reports + mega-linter.log diff --git a/.gitignore b/.gitignore index dfa9afb..20fd64e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ scripts/output scripts/config.json +megalinter-reports/ diff --git a/.lefthook.yml b/.lefthook.yml new file mode 100644 index 0000000..7d657c4 --- /dev/null +++ b/.lefthook.yml @@ -0,0 +1,19 @@ +pre-commit: + parallel: true + jobs: + - name: sort cache domains + run: jq ".cache_domains |= sort_by(.name)" {all_files} > {all_files}.tmp && mv {all_files}.tmp {all_files} + glob: "cache_domains.json" + + - name: sort cdns + run: | + for f in {all_files}; do + sort -uo ${f} ${f} + done + env: + LC_ALL: C + glob: "*.txt" + + - name: megalinter + run: docker run --rm -e USER=$(id -u) -e GROUP=$(id -g) -v "$PWD:/tmp/lint" oxsecurity/megalinter:v8 + stage_fixed: true diff --git a/.mega-linter.yml b/.mega-linter.yml new file mode 100644 index 0000000..bfca897 --- /dev/null +++ b/.mega-linter.yml @@ -0,0 +1,39 @@ +# Configuration file for MegaLinter +# +# See all available variables at https://megalinter.io/latest/config-file/ and in +# linters documentation + +APPLY_FIXES: all +FLAVOR_SUGGESTIONS: false +PRINT_ALPACA: false +SHOW_ELAPSED_TIME: true + +ENABLE: + - ACTION + - BASH + - EDITORCONFIG + - JSON + - MARKDOWN + - SPELL + - YAML + +DISABLE_LINTERS: + - JSON_V8R + - MARKDOWN_MARKDOWNLINT + - SPELL_CSPELL + - YAML_V8R + +PRE_COMMANDS: + - command: apk add --no-cache yq + cwd: root + - command: yq -i ".cache_domains |= sort_by(.name)" cache_domains.json + cwd: workspace + - command: | + for f in *.txt; do + sort -uo ${f} ${f} + done + cwd: workspace + +POST_COMMANDS: + - command: find . -user root -group root -exec chown ${USER}:${GROUP} {} \; + cwd: workspace diff --git a/blizzard.txt b/blizzard.txt index 4093330..dfda7c0 100644 --- a/blizzard.txt +++ b/blizzard.txt @@ -1,11 +1,11 @@ -dist.blizzard.com -dist.blizzard.com.edgesuite.net +*.cdn.blizzard.com blizzard.vo.llnwd.net blzddist1-a.akamaihd.net blzddist2-a.akamaihd.net blzddist3-a.akamaihd.net +cdn.blizzard.com +dist.blizzard.com +dist.blizzard.com.edgesuite.net +edge.blizzard.top.comcast.net level3.blizzard.com nydus.battle.net -edge.blizzard.top.comcast.net -cdn.blizzard.com -*.cdn.blizzard.com diff --git a/cache_domains.json b/cache_domains.json index 5628fb6..a8539f3 100644 --- a/cache_domains.json +++ b/cache_domains.json @@ -1,136 +1,136 @@ { - "cache_domains": [ - { - "name": "arenanet", - "description": "CDN for guild wars, HoT", - "domain_files": ["arenanet.txt"] - }, - { - "name": "blizzard", - "description": "CDN for blizzard/battle.net", - "domain_files": ["blizzard.txt"] - }, - { - "name": "cod", - "description": "CDN for Call of Duty", - "domain_files": ["cod.txt"] - }, - { - "name": "bsg", - "description": "CDN for Battle State Games, Tarkov", - "domain_files": ["bsg.txt"] - }, - { - "name": "cityofheroes", - "description": "CDN for City of Heroes (Homecoming)", - "domain_files": ["cityofheroes.txt"] - }, - { - "name": "daybreak", - "description": "Daybreak games CDN", - "domain_files": ["daybreak.txt"] - }, - { - "name": "epicgames", - "description": "CDN for Epic Games", - "domain_files": ["epicgames.txt"] - }, - { - "name": "frontier", - "description": "CDN for frontier games", - "domain_files": ["frontier.txt"] - }, - { - "name": "nexusmods", - "description": "Nexus mods / skyrim content", - "domain_files": ["nexusmods.txt"] - }, - { - "name": "neverwinter", - "description": "Cryptic CDN for Neverwinter", - "domain_files": ["neverwinter.txt"] - }, - { - "name": "nintendo", - "description": "CDN for Nintendo consoles and download servers", - "domain_files": ["nintendo.txt"] - }, - { - "name": "origin", - "description": "CDN for origin", - "notes": "Should be enabled for HTTP traffic only or with a HTTPS proxy else origin client download fails", - "mixed_content": true, - "domain_files": ["origin.txt"] - }, - { - "name": "pathofexile", - "description": "CDN for Path Of Exile", - "domain_files": ["pathofexile.txt"] - }, - { - "name": "renegadex", - "description": "CDN for Renegade X", - "domain_files": ["renegadex.txt"] - }, - { - "name": "riot", - "description": "CDN for riot games", - "domain_files": ["riot.txt"] - }, - { - "name": "rockstar", - "description": "CDN for rockstar games", - "domain_files": ["rockstar.txt"] - }, - { - "name": "sony", - "description": "CDN for sony / playstation", - "domain_files": ["sony.txt"] - }, - { - "name": "square", - "description": "CDN for Final Fantasy XIV", - "domain_files": ["square.txt"] - }, - { - "name": "steam", - "description": "CDN for steam platform", - "domain_files": ["steam.txt"] - }, - { - "name": "uplay", - "description": "CDN for uplay downloader", - "domain_files": ["uplay.txt"] - }, - { - "name": "teso", - "description": "CDN for The Elder Scrolls Online", - "domain_files": ["teso.txt"] - }, - { - "name": "warframe", - "description": "CDN for Warframe", - "domain_files": ["warframe.txt"] - }, - { - "name": "wargaming", - "description": "CDN for wargaming.net", - "domain_files": ["wargaming.net.txt"] - }, - { - "name": "wsus", - "description": "CDN for windows updates", - "domain_files": ["windowsupdates.txt"] - }, - { - "name": "xboxlive", - "description": "CDN for xboxlive", - "domain_files": ["xboxlive.txt"] - }, - { - "name": "test", - "description": "Test CDN, recommended to enable for additional diagnostics", - "domain_files": ["test.txt"] - } - ] + "cache_domains": [ + { + "name": "arenanet", + "description": "CDN for Guild Wars, HoT", + "domain_files": ["arenanet.txt"] + }, + { + "name": "blizzard", + "description": "CDN for Blizzard/Battle.net", + "domain_files": ["blizzard.txt"] + }, + { + "name": "bsg", + "description": "CDN for Battle State Games, Tarkov", + "domain_files": ["bsg.txt"] + }, + { + "name": "cityofheroes", + "description": "CDN for City of Heroes (Homecoming)", + "domain_files": ["cityofheroes.txt"] + }, + { + "name": "cod", + "description": "CDN for Call of Duty", + "domain_files": ["cod.txt"] + }, + { + "name": "daybreak", + "description": "CDN for Daybreak Games", + "domain_files": ["daybreak.txt"] + }, + { + "name": "epicgames", + "description": "CDN for Epic Games", + "domain_files": ["epicgames.txt"] + }, + { + "name": "frontier", + "description": "CDN for Frontier Games", + "domain_files": ["frontier.txt"] + }, + { + "name": "neverwinter", + "description": "Cryptic CDN for Neverwinter", + "domain_files": ["neverwinter.txt"] + }, + { + "name": "nexusmods", + "description": "CDN for Nexus Mods/Skyrim content", + "domain_files": ["nexusmods.txt"] + }, + { + "name": "nintendo", + "description": "CDN for Nintendo consoles", + "domain_files": ["nintendo.txt"] + }, + { + "name": "origin", + "description": "CDN for Origin", + "notes": "Should be enabled for HTTP traffic only or with a HTTPS proxy else origin client download fails", + "mixed_content": true, + "domain_files": ["origin.txt"] + }, + { + "name": "pathofexile", + "description": "CDN for Path Of Exile", + "domain_files": ["pathofexile.txt"] + }, + { + "name": "renegadex", + "description": "CDN for Renegade X", + "domain_files": ["renegadex.txt"] + }, + { + "name": "riot", + "description": "CDN for Riot Games", + "domain_files": ["riot.txt"] + }, + { + "name": "rockstar", + "description": "CDN for Rockstar Games", + "domain_files": ["rockstar.txt"] + }, + { + "name": "sony", + "description": "CDN for Sony/PSN", + "domain_files": ["sony.txt"] + }, + { + "name": "square", + "description": "CDN for Final Fantasy XIV", + "domain_files": ["square.txt"] + }, + { + "name": "steam", + "description": "CDN for Steam", + "domain_files": ["steam.txt"] + }, + { + "name": "teso", + "description": "CDN for The Elder Scrolls Online", + "domain_files": ["teso.txt"] + }, + { + "name": "test", + "description": "CDN for Testing, recommended to enable for additional diagnostics", + "domain_files": ["test.txt"] + }, + { + "name": "uplay", + "description": "CDN for Ubisoft", + "domain_files": ["uplay.txt"] + }, + { + "name": "warframe", + "description": "CDN for Warframe", + "domain_files": ["warframe.txt"] + }, + { + "name": "wargaming", + "description": "CDN for Wargaming.net", + "domain_files": ["wargaming.net.txt"] + }, + { + "name": "wsus", + "description": "CDN for Windows Updates", + "domain_files": ["windowsupdates.txt"] + }, + { + "name": "xboxlive", + "description": "CDN for Xbox Live", + "domain_files": ["xboxlive.txt"] + } + ] } diff --git a/cityofheroes.txt b/cityofheroes.txt index 19aa469..fe3d985 100644 --- a/cityofheroes.txt +++ b/cityofheroes.txt @@ -1,4 +1,4 @@ +cdn-eu1.homecomingservers.com cdn-na1.homecomingservers.com cdn-na2.homecomingservers.com cdn-na3.homecomingservers.com -cdn-eu1.homecomingservers.com diff --git a/daybreak.txt b/daybreak.txt index 56c62d0..deaeb0b 100644 --- a/daybreak.txt +++ b/daybreak.txt @@ -1,6 +1,2 @@ -# Daybreak games -# # Do NOT cache manifest.patch.daybreakgames.com -# -#PS2 pls.patch.daybreakgames.com diff --git a/epicgames.txt b/epicgames.txt index 1003c77..eaaa6e8 100644 --- a/epicgames.txt +++ b/epicgames.txt @@ -1,14 +1,14 @@ -cdn1.epicgames.com -cdn2.epicgames.com cdn.unrealengine.com +cdn1.epicgames.com cdn1.unrealengine.com +cdn2.epicgames.com cdn2.unrealengine.com cdn3.unrealengine.com +cloudflare.epicgamescdn.com download.epicgames.com download2.epicgames.com download3.epicgames.com download4.epicgames.com +egdownload.fastly-edge.com epicgames-download1.akamaized.net fastly-download.epicgames.com -cloudflare.epicgamescdn.com -egdownload.fastly-edge.com diff --git a/nintendo.txt b/nintendo.txt index cdc1e01..19f27f3 100644 --- a/nintendo.txt +++ b/nintendo.txt @@ -1,11 +1,11 @@ +*.hac.lp1.d4c.nintendo.net +*.hac.lp1.eshop.nintendo.net +*.wup.eshop.nintendo.net +*.wup.shop.nintendo.net ccs.cdn.wup.shop.nintendo.net.edgesuite.net +ecs-lp1.hac.shop.nintendo.net geisha-wup.cdn.nintendo.net geisha-wup.cdn.nintendo.net.edgekey.net idbe-wup.cdn.nintendo.net idbe-wup.cdn.nintendo.net.edgekey.net -ecs-lp1.hac.shop.nintendo.net receive-lp1.dg.srv.nintendo.net -*.wup.shop.nintendo.net -*.wup.eshop.nintendo.net -*.hac.lp1.d4c.nintendo.net -*.hac.lp1.eshop.nintendo.net diff --git a/origin.txt b/origin.txt index 0a24713..9fe89c2 100644 --- a/origin.txt +++ b/origin.txt @@ -1,3 +1,3 @@ -origin-a.akamaihd.net -lvlt.cdn.ea.com cdn-patch.swtor.com +lvlt.cdn.ea.com +origin-a.akamaihd.net diff --git a/renegadex.txt b/renegadex.txt index 02a4fc3..579aa17 100644 --- a/renegadex.txt +++ b/renegadex.txt @@ -1,2 +1,2 @@ -patches.totemarts.services patches.totemarts.games +patches.totemarts.services diff --git a/riot.txt b/riot.txt index b75b6f5..d6de3b3 100644 --- a/riot.txt +++ b/riot.txt @@ -1,5 +1,5 @@ +*.dyn.riotcdn.net l3cdn.riotgames.com -worldwide.l3cdn.riotgames.com riotgamespatcher-a.akamaihd.net riotgamespatcher-a.akamaihd.net.edgesuite.net -*.dyn.riotcdn.net +worldwide.l3cdn.riotgames.com diff --git a/scripts/config.example.json b/scripts/config.example.json index e9d5b4f..6784614 100644 --- a/scripts/config.example.json +++ b/scripts/config.example.json @@ -1,20 +1,20 @@ { - "combined_output": false, - "ips": { - "steam": ["10.10.3.10", "10.10.3.11"], - "origin": "10.10.3.12", - "blizzard": "10.10.3.13", - "windows": "10.10.3.14", - "riot": "10.10.3.15", - "generic": "10.10.3.16" - }, - "cache_domains": { - "default": "generic", - "blizzard": "blizzard", - "origin": "origin", - "riot": "riot", - "steam": "steam", - "wsus": "windows", - "xboxlive": "windows" - } + "combined_output": false, + "ips": { + "steam": ["10.10.3.10", "10.10.3.11"], + "origin": "10.10.3.12", + "blizzard": "10.10.3.13", + "windows": "10.10.3.14", + "riot": "10.10.3.15", + "generic": "10.10.3.16" + }, + "cache_domains": { + "default": "generic", + "blizzard": "blizzard", + "origin": "origin", + "riot": "riot", + "steam": "steam", + "wsus": "windows", + "xboxlive": "windows" + } } diff --git a/scripts/create-adguardhome.sh b/scripts/create-adguardhome.sh index 25df23a..c7048f2 100755 --- a/scripts/create-adguardhome.sh +++ b/scripts/create-adguardhome.sh @@ -3,75 +3,76 @@ basedir=".." outputdir="output/adguardhome" path="${basedir}/cache_domains.json" -export IFS=' ' +export IFS=" " -test=$(which jq); -if [ $? -gt 0 ] ; then - echo "This script requires jq to be installed." - echo "Your package manager should be able to find it" - exit 1 +if ! command -v jq >/dev/null; then + cat <<-EOF + This script requires jq to be installed. + Your package manager should be able to find it + EOF + exit 1 fi cachenamedefault="disabled" combinedoutput=$(jq -r ".combined_output" config.json) -while read line; do - ip=$(jq ".ips[\"${line}\"]" config.json) - declare "cacheip${line}"="${ip}" -done <<< $(jq -r '.ips | to_entries[] | .key' config.json) +while read -r line; do + ip=$(jq ".ips[\"${line}\"]" config.json) + declare "cacheip${line}"="${ip}" +done <<<"$(jq -r ".ips | to_entries[] | .key" config.json)" -while read line; do - name=$(jq -r ".cache_domains[\"${line}\"]" config.json) - declare "cachename$line"="$name" -done <<< $(jq -r '.cache_domains | to_entries[] | .key' config.json) +while read -r line; do + name=$(jq -r ".cache_domains[\"${line}\"]" config.json) + declare "cachename${line}"="${name}" +done <<<"$(jq -r ".cache_domains | to_entries[] | .key" config.json)" rm -rf ${outputdir} mkdir -p ${outputdir} -while read entry; do - unset cacheip - unset cachename - key=$(jq -r ".cache_domains[$entry].name" ${path}) - cachename="cachename${key}" - if [ -z "${!cachename}" ]; then - cachename="cachenamedefault" - fi - if [[ ${!cachename} == "disabled" ]]; then - continue; - fi - cacheipname="cacheip${!cachename}" - cacheip=$(jq -r 'if type == "array" then .[] else . end' <<< ${!cacheipname} | xargs) - while read fileid; do - while read filename; do - destfilename=$(echo ${filename} | sed -e 's/txt/conf/') - outputfile=${outputdir}/${destfilename} - touch ${outputfile} - while read fileentry; do - # Ignore comments and newlines - if [[ ${fileentry} == \#* ]] || [[ -z ${fileentry} ]]; then - continue - fi - domainprefix="|" - if [[ $fileentry =~ ^\*\. ]]; then - domainprefix="||" - fi - parsed=$(echo ${fileentry} | sed -e "s/^\*\.//") - if grep -q "${domainprefix}${parsed}^\$dnsrewrite" ${outputfile}; then - continue - fi - for i in ${cacheip}; do - echo "${domainprefix}${parsed}^\$dnsrewrite=${i}" >> ${outputfile} - echo "${domainprefix}${parsed}^\$dnstype=AAAA" >> ${outputfile} - done - done <<< $(cat ${basedir}/$filename | sort); - done <<< $(jq -r ".cache_domains[${entry}].domain_files[${fileid}]" ${path}) - done <<< $(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path}) -done <<< $(jq -r '.cache_domains | to_entries[] | .key' ${path}) +while read -r entry; do + unset cacheip + unset cachename + key=$(jq -r ".cache_domains[$entry].name" ${path}) + cachename="cachename${key}" + if [ -z "${!cachename}" ]; then + cachename=${cachenamedefault} + fi + if [[ ${cachename} == "disabled" ]]; then + continue + fi + cacheipname="cacheip${!cachename}" + cacheip=$(jq -r "if type == \"array\" then .[] else . end" <<<"${!cacheipname}" | xargs) + while read -r fileid; do + while read -r filename; do + destfilename=${filename//txt/conf} + outputfile=${outputdir}/${destfilename} + touch "${outputfile}" + while read -r fileentry; do + # Ignore comments and newlines + if [[ ${fileentry} == \#* ]] || [[ -z ${fileentry} ]]; then + continue + fi + domainprefix="|" + if [[ $fileentry =~ ^\*\. ]]; then + domainprefix="||" + fi + parsed="${fileentry#\*\.}" + if grep -q "${domainprefix}${parsed}^\$dnsrewrite" "${outputfile}"; then + continue + fi + for i in ${cacheip}; do + echo "${domainprefix}${parsed}^\$dnsrewrite=${i}" >>"${outputfile}" + echo "${domainprefix}${parsed}^\$dnstype=AAAA" >>"${outputfile}" + done + done <<<"$(cat ${basedir}/"${filename}" | sort)" + done <<<"$(jq -r ".cache_domains[${entry}].domain_files[${fileid}]" ${path})" + done <<<"$(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path})" +done <<<"$(jq -r ".cache_domains | to_entries[] | .key" ${path})" if [[ ${combinedoutput} == "true" ]]; then - for file in ${outputdir}/*; do f=${file//${outputdir}\/} && f=${f//.conf} && echo "# ${f^}" >> ${outputdir}/lancache.conf && cat ${file} >> ${outputdir}/lancache.conf && rm ${file}; done + for file in "${outputdir}"/*; do f=${file//${outputdir}\//} && f=${f//.conf/} && echo "# ${f^}" >>${outputdir}/lancache.conf && cat "${file}" >>${outputdir}/lancache.conf && rm "${file}"; done fi -cat << EOF +cat </dev/null; then + cat <<-EOF + This script requires jq to be installed. + Your package manager should be able to find it + EOF + exit 1 fi cachenamedefault="disabled" combinedoutput=$(jq -r ".combined_output" config.json) while read -r line; do - ip=$(jq ".ips[\"${line}\"]" config.json) - declare "cacheip${line}"="${ip}" -done <<< $(jq -r '.ips | to_entries[] | .key' config.json) + ip=$(jq ".ips[\"${line}\"]" config.json) + declare "cacheip${line}"="${ip}" +done <<<"$(jq -r ".ips | to_entries[] | .key" config.json)" while read -r line; do - name=$(jq -r ".cache_domains[\"${line}\"]" config.json) - declare "cachename${line}"="${name}" -done <<< $(jq -r '.cache_domains | to_entries[] | .key' config.json) + name=$(jq -r ".cache_domains[\"${line}\"]" config.json) + declare "cachename${line}"="${name}" +done <<<"$(jq -r ".cache_domains | to_entries[] | .key" config.json)" rm -rf ${outputdir} mkdir -p ${outputdir} while read -r entry; do - unset cacheip - unset cachename - key=$(jq -r ".cache_domains[${entry}].name" ${path}) - cachename="cachename${key}" - if [ -z "${!cachename}" ]; then - cachename="cachenamedefault" - fi - if [[ ${!cachename} == "disabled" ]]; then - continue; - fi - cacheipname="cacheip${!cachename}" - cacheip=$(jq -r 'if type == "array" then .[] else . end' <<< ${!cacheipname} | xargs) - while read -r fileid; do - while read -r filename; do - destfilename=$(echo ${filename} | sed -e 's/txt/conf/') - outputfile=${outputdir}/${destfilename} - touch ${outputfile} - while read -r fileentry; do - # Ignore comments, newlines and wildcards - if [[ ${fileentry} == \#* ]] || [[ -z ${fileentry} ]]; then - continue - fi - parsed=$(echo ${fileentry} | sed -e "s/^\*\.//") - for i in ${cacheip}; do - if ! grep -qx "address=/${parsed}/${i}" "${outputfile}"; then - echo "address=/${parsed}/${i}" >> "${outputfile}" - fi - if ! grep -qx "local=/${parsed}/" "${outputfile}"; then - echo "local=/${parsed}/" >> "${outputfile}" - fi - done - done <<< $(cat ${basedir}/${filename} | sort); - done <<< $(jq -r ".cache_domains[${entry}].domain_files[$fileid]" ${path}) - done <<< $(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path}) -done <<< $(jq -r '.cache_domains | to_entries[] | .key' ${path}) + unset cacheip + unset cachename + key=$(jq -r ".cache_domains[${entry}].name" ${path}) + cachename="cachename${key}" + if [ -z "${!cachename}" ]; then + cachename=${cachenamedefault} + fi + if [[ ${cachename} == "disabled" ]]; then + continue + fi + cacheipname="cacheip${!cachename}" + cacheip=$(jq -r "if type == \"array\" then .[] else . end" <<<"${!cacheipname}" | xargs) + while read -r fileid; do + while read -r filename; do + destfilename=${filename//txt/conf} + outputfile=${outputdir}/${destfilename} + touch "${outputfile}" + while read -r fileentry; do + # Ignore comments, newlines and wildcards + if [[ ${fileentry} == \#* ]] || [[ -z ${fileentry} ]]; then + continue + fi + parsed="${fileentry#\*\.}" + for i in ${cacheip}; do + if ! grep -qx "address=/${parsed}/${i}" "${outputfile}"; then + echo "address=/${parsed}/${i}" >>"${outputfile}" + fi + if ! grep -qx "local=/${parsed}/" "${outputfile}"; then + echo "local=/${parsed}/" >>"${outputfile}" + fi + done + done <<<"$(cat ${basedir}/"${filename}" | sort)" + done <<<"$(jq -r ".cache_domains[${entry}].domain_files[${fileid}]" ${path})" + done <<<"$(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path})" +done <<<"$(jq -r ".cache_domains | to_entries[] | .key" ${path})" if [[ ${combinedoutput} == "true" ]]; then - for file in ${outputdir}/*; do f=${file//${outputdir}\/} && f=${f//.conf} && echo "# ${f^}" >> ${outputdir}/lancache.conf && cat ${file} >> ${outputdir}/lancache.conf && rm ${file}; done + for file in "${outputdir}"/*; do f=${file//${outputdir}\//} && f=${f//.conf/} && echo "# ${f^}" >>${outputdir}/lancache.conf && cat "${file}" >>${outputdir}/lancache.conf && rm "${file}"; done fi -cat << EOF +cat </dev/null; then + cat <<-EOF + This script requires jq to be installed. + Your package manager should be able to find it + EOF + exit 1 fi cachenamedefault="disabled" -while read line; do - ip=$(jq ".ips[\"${line}\"]" config.json) - declare "cacheip${line}"="${ip}" -done <<< $(jq -r '.ips | to_entries[] | .key' config.json) +while read -r line; do + ip=$(jq ".ips[\"${line}\"]" config.json) + declare "cacheip${line}"="${ip}" +done <<<"$(jq -r ".ips | to_entries[] | .key" config.json)" -while read line; do - name=$(jq -r ".cache_domains[\"${line}\"]" config.json) - declare "cachename${line}"="${name}" -done <<< $(jq -r '.cache_domains | to_entries[] | .key' config.json) +while read -r line; do + name=$(jq -r ".cache_domains[\"${line}\"]" config.json) + declare "cachename${line}"="${name}" +done <<<"$(jq -r ".cache_domains | to_entries[] | .key" config.json)" rm -rf ${outputdir} mkdir -p ${outputdir} outputfile=${outputdir}/db.rpz.${basedomain} -cat > ${outputfile} << EOF +cat >"${outputfile}" <> ${outputfile} - echo "; $(echo ${filename} | sed -e 's/.txt$//')" >> ${outputfile} - destfilename=$(echo ${filename} | sed -e 's/txt/conf/') - while read fileentry; do - # Ignore comments and newlines - if [[ ${fileentry} == \#* ]] || [[ -z ${fileentry} ]]; then - continue - fi - parsed=$(echo ${fileentry}) - if grep -qx "^\"${parsed}\". " ${outputfile}; then - continue - fi - t="" - for i in ${cacheip}; do - # only one cname per domain is allowed - if [[ ${t} = "CNAME" ]]; then - continue - fi - # for cnames you must use a fqdn with trailing dot - t="CNAME" - if [[ ${i} =~ ^[0-9\.]+$ ]] ; then - t="A" - elif [[ ! ${i} =~ \.$ ]] ; then - i="${i}." - fi - printf "%-50s IN %s %s\n" \ - "${parsed}" \ - "${t}" \ - "${i}" \ - >> ${outputfile} - done - done <<< $(cat ${basedir}/${filename} | sort); - done <<< $(jq -r ".cache_domains[${entry}].domain_files[${fileid}]" ${path}) - done <<< $(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path}) -done <<< $(jq -r '.cache_domains | to_entries[] | .key' ${path}) +while read -r entry; do + unset cacheip + unset cachename + key=$(jq -r ".cache_domains[${entry}].name" ${path}) + cachename="cachename${key}" + if [ -z "${!cachename}" ]; then + cachename=${cachenamedefault} + fi + if [[ ${cachename} == "disabled" ]]; then + continue + fi + cacheipname="cacheip${!cachename}" + cacheip=$(jq -r "if type == \"array\" then .[] else . end" <<<"${!cacheipname}" | xargs) + while read -r fileid; do + while read -r filename; do + echo "" >>"${outputfile}" + echo "; ${filename//.txt//}" >>"${outputfile}" + while read -r fileentry; do + # Ignore comments and newlines + if [[ ${fileentry} == \#* ]] || [[ -z ${fileentry} ]]; then + continue + fi + parsed=${fileentry} + if grep -qx "^\"${parsed}\". " "${outputfile}"; then + continue + fi + t="" + for i in ${cacheip}; do + # only one cname per domain is allowed + if [[ ${t} = "CNAME" ]]; then + continue + fi + # for cnames you must use a fqdn with trailing dot + t="CNAME" + if [[ ${i} =~ ^[0-9\.]+$ ]]; then + t="A" + elif [[ ! ${i} =~ \.$ ]]; then + i="${i}." + fi + printf "%-50s IN %s %s\n" \ + "${parsed}" \ + "${t}" \ + "${i}" \ + >>"${outputfile}" + done + done <<<"$(cat ${basedir}/"${filename}" | sort)" + done <<<"$(jq -r ".cache_domains[${entry}].domain_files[${fileid}]" ${path})" + done <<<"$(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path})" +done <<<"$(jq -r ".cache_domains | to_entries[] | .key" ${path})" -cat << EOF +cat </dev/null; then + cat <<-EOF + This script requires jq to be installed. + Your package manager should be able to find it + EOF + exit 1 fi cachenamedefault="disabled" while read -r line; do - name=$(jq -r ".cache_domains[\"${line}\"]" config.json) - declare "cachename${line}"="${name}" -done <<< $(jq -r '.cache_domains | to_entries[] | .key' config.json) + name=$(jq -r ".cache_domains[\"${line}\"]" config.json) + declare "cachename${line}"="${name}" +done <<<"$(jq -r ".cache_domains | to_entries[] | .key" config.json)" rm -rf ${outputdir} mkdir -p ${outputdir} while read -r entry; do - unset cachename - key=$(jq -r ".cache_domains[$entry].name" ${path}) - cachename="cachename${key}" - if [ -z "${!cachename}" ]; then - cachename="cachenamedefault" - fi - if [[ ${!cachename} == "disabled" ]]; then - continue; - fi - while read -r fileid; do - while read -r filename; do - destfilename=$(echo ${!cachename}.txt) - outputfile=${outputdir}/${destfilename} - touch ${outputfile} - while read -r fileentry; do - # Ignore comments - if [[ ${fileentry} == \#* ]] || [[ -z ${fileentry} ]]; then - continue - fi - # Ha wildcards to squid wildcards - parsed=$(echo ${fileentry} | sed -e "s/^\*\./\./") - # If we have cdn.thing and *.cdn.thing in cache_domains - # Squid requires ONLY cdn.thing - # - # If the fileentry starts with *.cdn.thing - if [[ ${fileentry} =~ $REGEX ]]; then - # Does the cache_domains file also contain cdn.thing - grep "${BASH_REMATCH[1]}" ${basedir}/${filename} | grep -v "${fileentry}" > /dev/null - if [[ $? -eq 0 ]]; then - # Skip *.cdn.thing as cdn.thing will be collected earlier/later - continue - fi - fi - echo "${parsed}" >> "${outputfile}" - done <<< $(cat ${basedir}/${filename} | sort); - done <<< $(jq -r ".cache_domains[${entry}].domain_files[${fileid}]" ${path}) - done <<< $(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path}) -done <<< $(jq -r '.cache_domains | to_entries[] | .key' ${path}) + unset cachename + key=$(jq -r ".cache_domains[$entry].name" ${path}) + cachename="cachename${key}" + if [ -z "${!cachename}" ]; then + cachename=${cachenamedefault} + fi + if [[ ${cachename} == "disabled" ]]; then + continue + fi + while read -r fileid; do + while read -r filename; do + destfilename=${!cachename}.txt + outputfile=${outputdir}/${destfilename} + touch "${outputfile}" + while read -r fileentry; do + # Ignore comments + if [[ ${fileentry} == \#* ]] || [[ -z ${fileentry} ]]; then + continue + fi + # Ha wildcards to squid wildcards + parsed="${fileentry#\*}" + # If we have cdn.thing and *.cdn.thing in cache_domains + # Squid requires ONLY cdn.thing + # + # If the fileentry starts with *.cdn.thing + if [[ ${fileentry} =~ $REGEX ]]; then + # Does the cache_domains file also contain cdn.thing + if grep "${BASH_REMATCH[1]}" ${basedir}/"${filename}" | grep -v "${fileentry}" >/dev/null; then + # Skip *.cdn.thing as cdn.thing will be collected earlier/later + continue + fi + fi + echo "${parsed}" >>"${outputfile}" + done <<<"$(cat ${basedir}/"${filename}" | sort)" + done <<<"$(jq -r ".cache_domains[${entry}].domain_files[${fileid}]" ${path})" + done <<<"$(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path})" +done <<<"$(jq -r ".cache_domains | to_entries[] | .key" ${path})" -cat << EOF +cat </dev/null; then + cat <<-EOF + This script requires jq to be installed. + Your package manager should be able to find it + EOF + exit 1 fi cachenamedefault="disabled" combinedoutput=$(jq -r ".combined_output" config.json) -while read line; do - ip=$(jq ".ips[\"${line}\"]" config.json) - declare "cacheip${line}"="${ip}" -done <<< $(jq -r '.ips | to_entries[] | .key' config.json) +while read -r line; do + ip=$(jq ".ips[\"${line}\"]" config.json) + declare "cacheip${line}"="${ip}" +done <<<"$(jq -r ".ips | to_entries[] | .key" config.json)" -while read line; do - name=$(jq -r ".cache_domains[\"${line}\"]" config.json) - declare "cachename${line}"="${name}" -done <<< $(jq -r '.cache_domains | to_entries[] | .key' config.json) +while read -r line; do + name=$(jq -r ".cache_domains[\"${line}\"]" config.json) + declare "cachename${line}"="${name}" +done <<<"$(jq -r ".cache_domains | to_entries[] | .key" config.json)" rm -rf ${outputdir} mkdir -p ${outputdir} -while read entry; do - unset cacheip - unset cachename - key=$(jq -r ".cache_domains[${entry}].name" ${path}) - cachename="cachename${key}" - if [ -z "${!cachename}" ]; then - cachename="cachenamedefault" - fi - if [[ ${!cachename} == "disabled" ]]; then - continue; - fi - cacheipname="cacheip${!cachename}" - cacheip=$(jq -r 'if type == "array" then .[] else . end' <<< ${!cacheipname} | xargs) - while read fileid; do - while read filename; do - destfilename=$(echo ${filename} | sed -e 's/txt/conf/') - outputfile=${outputdir}/${destfilename} - touch ${outputfile} - while read fileentry; do - # Ignore comments and newlines - if [[ ${fileentry} == \#* ]] || [[ -z ${fileentry} ]]; then - continue - fi - parsed=$(echo ${fileentry} | sed -e "s/^\*\.//") - if grep -qx " local-zone: \"${parsed}\" redirect" ${outputfile}; then - continue - fi - if [[ $(head -n 1 ${outputfile}) != "server:" ]]; then - echo "server:" >> ${outputfile} - fi - echo " local-zone: \"${parsed}\" redirect" >> ${outputfile} - for i in ${cacheip}; do - echo " local-data: \"${parsed} 30 IN A ${i}\"" >> ${outputfile} - done - done <<< $(cat ${basedir}/${filename} | sort); - done <<< $(jq -r ".cache_domains[${entry}].domain_files[${fileid}]" ${path}) - done <<< $(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path}) -done <<< $(jq -r '.cache_domains | to_entries[] | .key' ${path}) +while read -r entry; do + unset cacheip + unset cachename + key=$(jq -r ".cache_domains[${entry}].name" ${path}) + cachename="cachename${key}" + if [ -z "${!cachename}" ]; then + cachename=${cachenamedefault} + fi + if [[ ${cachename} == "disabled" ]]; then + continue + fi + cacheipname="cacheip${!cachename}" + cacheip=$(jq -r "if type == \"array\" then .[] else . end" <<<"${!cacheipname}" | xargs) + while read -r fileid; do + while read -r filename; do + destfilename=${filename//txt/conf} + outputfile=${outputdir}/${destfilename} + touch "${outputfile}" + while read -r fileentry; do + # Ignore comments and newlines + if [[ ${fileentry} == \#* ]] || [[ -z ${fileentry} ]]; then + continue + fi + parsed="${fileentry#\*\.}" + if grep -qx " local-zone: \"${parsed}\" redirect" "${outputfile}"; then + continue + fi + if [[ $(head -n 1 "${outputfile}") != "server:" ]]; then + echo "server:" >>"${outputfile}" + fi + echo " local-zone: \"${parsed}\" redirect" >>"${outputfile}" + for i in ${cacheip}; do + echo " local-data: \"${parsed} 30 IN A ${i}\"" >>"${outputfile}" + done + done <<<"$(cat ${basedir}/"${filename}" | sort)" + done <<<"$(jq -r ".cache_domains[${entry}].domain_files[${fileid}]" ${path})" + done <<<"$(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path})" +done <<<"$(jq -r ".cache_domains | to_entries[] | .key" ${path})" if [[ ${combinedoutput} == "true" ]]; then - for file in ${outputdir}/*; do f=${file//${outputdir}\/} && f=${f//.conf} && echo "# ${f^}" >> ${outputdir}/lancache.conf && cat ${file} >> ${outputdir}/lancache.conf && rm ${file}; done + for file in "${outputdir}"/*; do f=${file//${outputdir}\//} && f=${f//.conf/} && echo "# ${f^}" >>${outputdir}/lancache.conf && cat "${file}" >>${outputdir}/lancache.conf && rm "${file}"; done fi -cat << EOF +cat <