mirror/antizapret-pac-generator-light
1
0
Fork 0
mirror of https://bitbucket.org/anticensority/antizapret-pac-generator-light.git synced 2025-02-22 17:33:13 +03:00
Code Issues Packages Projects Releases Wiki Activity

New function: test reachability with curl

Browse Source
This commit is contained in:
ValdikSS 2025-02-08 18:03:55 +03:00
parent c150c9535e
commit 4e00d40fca
3 changed files with 40 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
config/config.sh
View File

@ -14,3 +14,7 @@ PACFILE_NOSSL="result/proxy-host-nossl.pac"
# Perform DNS resolving to detect and filter non-existent domains # Perform DNS resolving to detect and filter non-existent domains
RESOLVE_NXDOMAIN="no" RESOLVE_NXDOMAIN="no"
# Perform HTTP (port 80)/HTTPS (port 443) reachability test with curl,
# to detect and filter broken websites
TEST_WEB_AVAILABILITY="no"

7
parse.sh
View File

@ -31,6 +31,13 @@ then
awk -f scripts/getzones.awk temp/hostlist_original_with_include.txt | grep -v -F -x -f temp/exclude-hosts.txt | cat - temp/include-hosts.txt | sort -u > result/hostlist_zones.txt awk -f scripts/getzones.awk temp/hostlist_original_with_include.txt | grep -v -F -x -f temp/exclude-hosts.txt | cat - temp/include-hosts.txt | sort -u > result/hostlist_zones.txt
fi fi
if [[ "$TEST_WEB_AVAILABILITY" == "yes" ]];
then
timeout 2h parallel --bar -j96 -n1 -a result/hostlist_zones.txt scripts/test-with-curl.sh > temp/web-unreachable-exclude-hosts.txt || true
cat temp/web-unreachable-exclude-hosts.txt >> temp/exclude-hosts.txt
awk -f scripts/getzones.awk temp/hostlist_original_with_include.txt | grep -v -F -x -f temp/exclude-hosts.txt | cat - temp/include-hosts.txt | sort -u > result/hostlist_zones.txt
fi
# Generate a list of IP addresses # Generate a list of IP addresses
awk -F';' '$1 ~ /\// {print $1}' temp/list.csv | grep -P '([0-9]{1,3}\.){3}[0-9]{1,3}\/[0-9]{1,2}' -o | sort -Vu > result/iplist_special_range.txt awk -F';' '$1 ~ /\// {print $1}' temp/list.csv | grep -P '([0-9]{1,3}\.){3}[0-9]{1,3}\/[0-9]{1,2}' -o | sort -Vu > result/iplist_special_range.txt

29
scripts/test-with-curl.sh Executable file
View File

@ -0,0 +1,29 @@
#!/bin/bash
function curl_open() {
local WEBSITE="$1"
curl --max-time 10 --compressed --fail --fail-early \
-H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:134.0) Gecko/20100101 Firefox/134.0 antizapret.prostovpn.org anti-censorship tester' \
"http://$WEBSITE" &> /dev/null
RESULT_HTTP="$?"
curl --max-time 10 --compressed --fail --fail-early \
-H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:134.0) Gecko/20100101 Firefox/134.0 antizapret.prostovpn.org anti-censorship tester' \
-k "https://$WEBSITE" &> /dev/null
RESULT_HTTPS="$?"
if [ "$RESULT_HTTP" -ne 0 ] && [ "$RESULT_HTTPS" -ne 0 ]; then
# If both HTTP and HTTPS failed, the website doesn't work
return 1
fi
return 0
}
WEBSITE="$1"
curl_open "$WEBSITE"
ERRCODE="$?"
if [ "$ERRCODE" -ne 0 ]; then
#echo "$WEBSITE does not respond, marking as failed, not adding to the PAC list"
echo "$WEBSITE"
fi
exit $ERRCODE