import asyncio
import itertools

import aiohttp

LIST = "/root/recon/scan_p443.list"
STEP = 150
CONC = 1200
OUT = "/root/recon/sample_products.tsv"

MARKS = [
    ("smartermail", ("smartermail",)),
    ("whd", ("web help desk",)),
    ("papercut", ("papercut",)),
    ("zimbra", ("zimbra web client", "zimbra collaboration")),
    ("netscaler", ("netscaler", "citrix gateway")),
    ("confluence", ("confluence",)),
    ("jira", ("jira",)),
    ("gitlab", ("gitlab",)),
    ("solarwinds", ("solarwinds",)),
    ("beyondtrust", ("beyondtrust", "bomgar")),
    ("panos", ("global-protect", "pan-os", "panos")),
    ("fortinet", ("fortinet", "fortigate")),
    ("zammad", ("zammad",)),
]

stats = {m: 0 for m, _ in MARKS}
done = 0
hits = []
sem = None


async def one(sess, ip):
    global done
    try:
        async with sem:
            async with sess.get("https://%s/" % ip, ssl=False, allow_redirects=True) as r:
                b = (await r.read(200000)).decode("utf-8", "ignore").lower()
                hdr = " ".join("%s:%s" % (k.lower(), v) for k, v in r.headers.items()).lower()
                for name, pats in MARKS:
                    if any(p in b or p in hdr for p in pats):
                        stats[name] += 1
                        hits.append("%s\t%s\t%s" % (ip, name, r.status))
    except Exception:
        pass
    finally:
        done += 1


async def main():
    global sem
    sem = asyncio.Semaphore(CONC)
    conn = aiohttp.TCPConnector(limit=0, ssl=False, force_close=True)
    to = aiohttp.ClientTimeout(total=10, connect=4, sock_read=6)
    ips = []
    with open(LIST) as f:
        for i, line in enumerate(f):
            if line.startswith("#"):
                continue
            if i % STEP:
                continue
            p = line.split()
            if len(p) >= 4:
                ips.append(p[3])
            if len(ips) >= 200000:
                break
    async with aiohttp.ClientSession(timeout=to, connector=conn,
                                     headers={"User-Agent": "Mozilla/5.0"}) as sess:
        await asyncio.gather(*[one(sess, ip) for ip in ips])
    open(OUT, "w").write("\n".join(hits))
    print("sampled", len(ips), "done", done)
    for k, v in sorted(stats.items(), key=lambda x: -x[1]):
        print("%-12s %d" % (k, v))


asyncio.run(main())
