from collections import Counter

rows = []
for line in open("/root/recon/cp_portal.tsv", encoding="utf-8", errors="replace"):
    f = line.rstrip("\n").split("\t")
    while len(f) < 5:
        f.append("")
    rows.append(f)

doms = Counter()
hints = Counter()
titles = Counter()
with_dom = 0
for ip, host, title, hint, dom in rows:
    if dom:
        with_dom += 1
        for d in dom.split(","):
            if d:
                doms[d] += 1
    for h in hint.split(","):
        if h:
            hints[h] += 1
    if title:
        titles[title] += 1

print("rows", len(rows), "with_domain", with_dom)
print("== hints")
for k, v in hints.most_common(15):
    print("  %-14s %d" % (k, v))
print("== titles")
for k, v in titles.most_common(12):
    print("  %-55s %d" % (k[:55], v))
print("== domains")
for k, v in doms.most_common(40):
    print("  %-45s %d" % (k, v))
