#!/bin/bash
cd /root/recon
OUT=loot/names
mkdir -p $OUT
DOMAINS="corporatediligence.com darkforesttech.com projectaries.net amyskitchen.net villanova.edu unm.edu acsdsc.org usfamilyhealth.org mdjia.org southmetro.org owens.edu wilkescc.edu chenega.com navajoutility.com"
PAGES="/ /about /about-us /team /our-team /people /staff /leadership /contact /contact-us /attorneys /faculty /directory"
: > $OUT/all_raw.txt
for D in $DOMAINS; do
  P=$OUT/$D.txt; : > $P
  for PG in $PAGES; do
    timeout 12 curl -skL -m 10 -A 'Mozilla/5.0' "https://$D$PG" 2>/dev/null \
      | sed 's/<[^>]*>/ /g' \
      | grep -oE '\b[A-Z][a-z]{2,12}[ ]+[A-Z][a-z]{2,16}\b' >> $P
  done
  grep -viE 'United|States|Privacy|Policy|Contact|About|Home|Page|Read|More|Team|Staff|Login|Sign|Search|Cookie|Terms|Rights|Reserved|Our|Your|Learn|News|Event|Careers|Support|Services|Solutions|Company|Office|Board|Director|Manager|Email|Phone|Address|Copyright|All|The|And|For|With|This|That|From|Have|Will|You|Not|Are|Was|Were|Has|Can|May' $P | sort -u > $P.clean
  echo "$D raw=$(wc -l < $P) clean=$(wc -l < $P.clean)"
  cat $P.clean >> $OUT/all_raw.txt
done
sort -u $OUT/all_raw.txt > $OUT/names_all.txt
wc -l $OUT/names_all.txt
head -20 $OUT/names_all.txt
