import io, re, ssl, sys, zipfile, urllib.request
from concurrent.futures import ThreadPoolExecutor

CTX = ssl._create_unverified_context()
PAT = re.compile(r'(AKIA[0-9A-Z]{16}|ASIA[0-9A-Z]{16}|aws_secret_access_key|BEGIN [A-Z ]*PRIVATE KEY|jdbc:[a-z0-9]+://[^\s"\'<>]+|(?:postgres(?:ql)?|mysql|mongodb(?:\+srv)?|redis|amqp)://[^\s"\'<>]+|ldaps?://[^\s"\'<>]+|(?i:password|passwd|passphrase|secret|token|api[_-]?key|client_secret|access_key)[^A-Za-z0-9]{1,4}[=:][ \t]*[^\s"\'<>,;]{5,})')
TEXT = re.compile(r'\.(txt|md|json|ya?ml|xml|properties|conf|cfg|ini|sh|tf|tfvars|env|py|js|ts|gradle|pom|sql|j2|template|sample|example)$', re.I)
LIMIT = 300000


def get(u):
    req = urllib.request.Request(u, headers={'User-Agent': 'Mozilla/5.0'})
    return urllib.request.urlopen(req, timeout=25, context=CTX).read()


def scan(row):
    repo, name, size, ctype, url = row
    try:
        blob = get(url)
    except Exception:
        return []
    out = []
    try:
        z = zipfile.ZipFile(io.BytesIO(blob))
    except Exception:
        return []
    for zi in z.infolist():
        if zi.file_size > 400000 or not TEXT.search(zi.filename):
            continue
        try:
            data = z.read(zi)
        except Exception:
            continue
        txt = data.decode('utf-8', 'replace')
        for m in PAT.finditer(txt):
            s = txt[max(0, m.start() - 60):m.start() + 120].replace('\n', ' ')
            out.append('%s :: %s :: %s' % (name, zi.filename, ' '.join(s.split())[:220]))
    return out


def main():
    rows = [l.rstrip('\n').split('\t') for l in open('/tmp/.nx_list.tsv')]
    sel = [r for r in rows if r[0] == 'raw-hosted' and r[4].endswith('.zip') and int(r[2] or 0) < LIMIT]
    print('selected', len(sel), flush=True)
    f = open('/tmp/.nx_hits.txt', 'w')
    done = 0
    with ThreadPoolExecutor(max_workers=8) as ex:
        for res in ex.map(scan, sel):
            done += 1
            for line in res:
                f.write(line + '\n')
            if done % 200 == 0:
                f.flush()
                print('done', done, flush=True)
    f.close()
    print('finished', done, flush=True)

main()
