<commit_before>from __future__ import print_function
import sys
import requests
from sqli import check
from bs4 import BeautifulSoup
from textwrap import dedent
from argparse import ArgumentParser
from datetime import datetime

_parser = ArgumentParser(description="Find vulnerable Python from SO")
_parser.add_argument("tags", nargs="*",
                     default=["sql"],
                     help="Search posts tagged with tags")

_SITE = "stackoverflow"
_FILTER = "!5-dm_.B4KdW3(tKMnD-gYaOdS-mkdxhSIbFHRm"


def fetch_post_soup(item):
    body = item["body"]

    if "answers" in item:
        body += "\n".join(a["body"] for a in item["answers"])

    return BeautifulSoup(body, "lxml")


def fetch(tags):
    url = 'https://api.stackexchange.com/2.2/questions'
    params = dict(
        page=1,
        pagesize=100,
        order="desc",
        sort="creation",
        tagged=';'.join(["python"] + list(tags)),
        site=_SITE,
        filter=_FILTER,
    )
    resp = requests.get(url, params=params)
    items = resp.json()["items"]
    for it in items:
        link = it["link"]
        created = datetime.utcfromtimestamp(it["creation_date"])
        soup = fetch_post_soup(it)
        for code in soup.select("code"):
            source = dedent(code.text)
            try:
                poisoned = check(source)
                if poisoned:
                    print("{:%Y-%m-%d} {}:".format(created, link))
                    for p in poisoned:
                        print("#{}: {}".format(p.get_lineno(), p.get_source()))
                    break

            except Exception as e:
                # Just ignore
                pass
<commit_msg>Parse code blocks separately<commit_after>from __future__ import print_function
import sys
import requests
from sqli import check
from bs4 import BeautifulSoup
from textwrap import dedent
from argparse import ArgumentParser
from datetime import datetime
from itertools import chain

_parser = ArgumentParser(description="Find vulnerable Python from SO")
_parser.add_argument("tags", nargs="*",
                     default=["sql"],
                     help="Search posts tagged with tags")

_SITE = "stackoverflow"
_FILTER = "!5-dm_.B4KdW3(tKMnD-gYaOdS-mkdxhSIbFHRm"


def fetch_post_soup(item):
    bs = [item["body"]]

    if "answers" in item:
        bs += [a["body"] for a in item["answers"]]

    return (BeautifulSoup(body, "lxml") for body in bs)


def fetch(tags):
    url = 'https://api.stackexchange.com/2.2/questions'
    params = dict(
        page=1,
        pagesize=100,
        order="desc",
        sort="creation",
        tagged=';'.join(["python"] + list(tags)),
        site=_SITE,
        filter=_FILTER,
    )
    resp = requests.get(url, params=params)
    items = resp.json()["items"]
    for it in items:
        link = it["link"]
        created = datetime.utcfromtimestamp(it["creation_date"])
        soups = fetch_post_soup(it)
        codes = chain.from_iterable(soup.select("code") for soup in soups)
        poisoned = []
        for code in codes:
            source = dedent(code.text)
            try:
                poisoned.extend(check(source))

            except Exception as e:
                # Just ignore
                pass

        if poisoned:
            print("{:%Y-%m-%d} {}:".format(created, link))
            for p in poisoned:
                print("#{}: {}".format(p.get_lineno(), p.get_source()))

{'fraction_non_alphanumeric': 0.12434478741991846, 'fraction_numerical': 0.005824111822947, 'mean_word_length': 2.283938814531549, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5247984', 'n_tokens_mistral': 1075, 'n_tokens_neox': 1023, 'n_words': 258}