#!/bin/bash
UA='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36'
# Output directory as the first argument, so a second capture does not overwrite
# the first: wave 1 was `/tmp/psharvest`, wave 2 `/tmp/psharvest-w2`. The
# directory is emptied first, so it has to be a scratch path - passing `pages`
# would otherwise delete the published captures.
#
# After a run, the files become the published capture like this:
#   gzip -9 -n $D/*.html
#   mv $D/*.html.gz pages/w2/  &&  mv $D/_fetchlog.tsv fetchlog-w2.tsv
D="${1:-/tmp/psharvest}"
case "$D" in
  /tmp/*) ;;
  *) echo "refusing: output directory must be under /tmp, got $D" >&2; exit 1 ;;
esac
rm -rf "$D"; mkdir -p "$D"
: > $D/_fetchlog.tsv
while read -r name url; do
  [ -z "$name" ] && continue
  ts=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
  code=$(curl -sSL -m 90 --compressed -A "$UA" -H 'Accept-Language: en-US,en;q=0.9' \
     -o "$D/$name.html" -w '%{http_code}\t%{url_effective}\t%{size_download}' "$url" 2>/dev/null)
  printf '%s\t%s\t%s\t%s\n' "$name" "$url" "$ts" "$code" >> $D/_fetchlog.tsv
  echo "$name $ts $code"
done <<'EOF'
smmlaunch https://smmlaunch.com/services
adderpanel https://adderpanel.com/services
crescitaly https://crescitaly.com/services
smmcost https://smmcost.com/services
smmkings https://smmkings.com/services
growfastsmm https://growfastsmm.com/services
1xpanel https://1xpanel.com/services
morethanpanel https://morethanpanel.com/services
mysmm https://mysmm.co/services
smmfolgen https://smmfolgen.com/services
easytopromo https://easytopromo.com/services
smmpanelone https://smmpanel.one/services
globalsmm https://global-smm.com/services
autosmo https://autosmo.com/services
peakerr https://peakerr.com/services
EOF
echo "=== sizes ==="; ls -la $D