wiki-cafe/wikiverse-analysis
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add argument processing, update data

Browse Source
This commit is contained in:
3wc
2023-03-24 00:43:22 -04:00
parent fef4cd7b0f
commit 51d443b160
2 changed files with 788 additions and 779 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1552
wikiverse-https-scan.txt.csv → wikiverse-https-scan.csv
View File

File diff suppressed because it is too large Load Diff

15
wikiverse-scan.sh Normal file → Executable file
View File

@ -1,3 +1,12 @@
#!/bin/bash
html_file="${1:-search.fed.wiki.org.html}"
if [ -z $html_file ]; then
echo "HTML file '$html_file' does not exist"
exit 1;
fi
process_result() {
wiki=$1
farm=$(echo "$wiki" | xargs tldextract | cut -d' ' -f2,3 | sed 's/ /./g')
@ -6,10 +15,10 @@ process_result() {
echo "$wiki,$farm,$status_http,$status_https";
}
echo "Wiki,Farm,HTTP?,HTTPS?" > ./wikiverse-https-scan.txt.csv
echo "Wiki,Farm,HTTP?,HTTPS?" > ./wikiverse-https-scan.csv
for wiki in $(pup '#results a attr{href}' < search.fed.wiki.org.html); do
for wiki in $(pup '#results a attr{href}' < "$html_file"); do
process_result "$wiki" &
done >> wikiverse-https-scan.txt.csv
done >> wikiverse-https-scan.csv
wait $(jobs -p)