import io, re, ssl, sys, tarfile, zipfile, urllib.request
from concurrent.futures import ThreadPoolExecutor

CTX = ssl._create_unverified_context()
MODE = sys.argv[1]
PAT = re.compile(r'(AKIA[0-9A-Z]{16}|ASIA[0-9A-Z]{16}|gh[pous]_[A-Za-z0-9]{20,}|xox[baprs]-[A-Za-z0-9-]{10,}|sk_live_[A-Za-z0-9]{20,}|AIza[0-9A-Za-z_\-]{30,}|AccountKey=[A-Za-z0-9+/=]{20,}|SharedAccessKey=[A-Za-z0-9+/=]{20,}|BEGIN [A-Z ]*PRIVATE KEY|jdbc:[a-z0-9]+://[^\s"\'<>]+|(?:postgres(?:ql)?|mysql|mongodb(?:\+srv)?|redis|amqp)://[^ /@]+:[^ /@]+@[^\s"\'<>]+|(?i:password|passwd|secret|api[_-]?key|client_secret|access[_-]?key)[^A-Za-z0-9]{1,4}[=:][ \t]*["\']?[A-Za-z0-9!@#$%^&*_\-\.\+/]{6,})')
TEXT = re.compile(r'\.(txt|md|json|ya?ml|xml|properties|conf|cfg|ini|sh|tf|tfvars|env|py|js|ts|java|gradle|pom|sql|j2|template|sample|example|npmrc|pypirc)$', re.I)


def get(u):
    req = urllib.request.Request(u, headers={'User-Agent': 'Mozilla/5.0'})
    return urllib.request.urlopen(req, timeout=40, context=CTX).read()


def members(blob, name):
    if name.endswith(('.tgz', '.tar.gz', '.whl')):
        try:
            t = tarfile.open(fileobj=io.BytesIO(blob), mode='r:gz')
            for m in t.getmembers():
                if m.isfile() and m.size < 300000 and TEXT.search(m.name):
                    yield m.name, t.extractfile(m).read()
        except Exception:
            return
    else:
        try:
            z = zipfile.ZipFile(io.BytesIO(blob))
        except Exception:
            return
        for zi in z.infolist():
            if not zi.is_dir() and zi.file_size < 300000 and TEXT.search(zi.filename):
                try:
                    yield zi.filename, z.read(zi)
                except Exception:
                    continue


def scan(row):
    repo, name, size, ctype, url = row
    try:
        blob = get(url)
    except Exception:
        return []
    out = []
    for fn, data in members(blob, name):
        txt = data.decode('utf-8', 'replace')
        for m in PAT.finditer(txt):
            s = txt[max(0, m.start() - 50):m.start() + 130].replace('\n', ' ')
            out.append('%s :: %s :: %s' % (name, fn, ' '.join(s.split())[:200]))
    return out


def main():
    rows = [l.rstrip('\n').split('\t') for l in open('/tmp/.nx_list.tsv')]
    if MODE == 'pkgs':
        sel = [r for r in rows if r[0] in ('npm-hosted', 'pypi-hosted', 'helm-hosted') and 0 < int(r[2] or 0) < 8000000]
    else:
        sel = [r for r in rows if r[0] == 'raw-hosted' and r[4].endswith(('.zip', '.jar', '.tar', '.gz')) and 300000 <= int(r[2] or 0) < 6000000]
    print('selected', len(sel), flush=True)
    f = open('/tmp/.nx_hits2.txt', 'w')
    done = 0
    with ThreadPoolExecutor(max_workers=8) as ex:
        for res in ex.map(scan, sel):
            done += 1
            for line in res:
                f.write(line + '\n')
            if done % 200 == 0:
                f.flush()
                print('done', done, flush=True)
    f.close()
    print('finished', done, flush=True)

main()
