#!/bin/bash
cd /root/recon
echo '== product histogram (multi-sig junk excluded):'
awk -F'\t' '{n=gsub(/,/,",",$5)} n<4 {split($5,a,","); for(k in a) print a[k]}' hitsB.tsv | sort | uniq -c | sort -rn
echo '== sector join:'
> sector_full.tsv
for list in gov_domains.txt medical_domains.txt legal_domains.txt; do
  sec=${list%%_*}
  while read d; do
    [ -z "$d" ] && continue
    ips=$(grep -P "\t$d$" dom_ip.tsv | cut -f1)
    for ip in $ips; do
      m=$(grep -F "$ip	" hitsB.tsv | head -1)
      [ -n "$m" ] && echo -e "$sec\t$d\t$m" >> sector_full.tsv
    done
  done < $list
done
wc -l sector_full.tsv
cut -f1,2 sector_full.tsv | head -40
echo MFDONE
