<commit_before>import re
import json
import requests
import vim

API_KEY = "<KEY>


def strip_html(html):
    pattern = re.compile(r'<.*?>')
    return pattern.sub('', html)


def query_google(query, domain):
    search = "https://ajax.googleapis.com/ajax/services/search/web?v=1.0&q=%s"
    url = search % (query + ":" + domain)
    response = requests.get(url)
    data = json.loads(response.text)["responseData"]["results"]
    urls = []
    for result in data:
        url = result["url"]
        if is_valid_url(url):
            urls.append(url)
    return urls


def get_question_id(url):
    try:
        qid = int(url.split("/")[-2])
    except:
        qid = None
    return qid


def is_valid_url(url):
    return "stackoverflow" in url and "/tagged/" not in url


def get_question_data(qid):
    response = requests.get("https://api.stackexchange.com/2.2/questions/%s/answers?order=&sort=votes&site=stackoverflow&key=%s&filter=!)Q2B_4mND07Uc*hKpm6.P0Q5" % (qid, API_KEY))
    return json.loads(response.text)["items"]


def parse_question_data(data, _filter):
    post = {
        "question": "",
        "answers": []
    }
    if len(data) == 0 or "title" not in data[0]:
        return post
    post["question"] = data[0]["title"]
    for answer in data:
        answer_data = parse_answer(answer)
        if _filter == "accepted":
            if answer_data[2]:
                post["answers"].append(parse_answer(answer))
        elif _filter == "top":
            post["answers"].append(parse_answer(answer))
            break
        else:
            post["answers"].append(parse_answer(answer))
    return post


def parse_answer(answer):
    author = answer["owner"]["display_name"]
    content = answer["body"]
    is_accepted = answer["is_accepted"]
    upvotes = answer["score"]
    url = answer["share_link"]
    return [content, url, is_accepted, upvotes, author]


def fetch_mass_data(query, _filter):
    urls = query_google(query, "www.stackoverflow.com")
    posts = []
    for url in urls:
        qid = get_question_id(url)
        if not qid:
            continue
        data = get_question_data(qid)
        posts.append(parse_question_data(data, _filter))
    return posts


# StackAnswers output ---------------------------------------------------------


def _goto_window_for_buffer(b):
    w = int(vim.eval('bufwinnr(%d)' % int(b)))
    vim.command('%dwincmd w' % w)


def _goto_window_for_buffer_name(bn):
    b = vim.eval('bufnr("%s")' % bn)
    return _goto_window_for_buffer(b)


def _output_preview_text(lines):
    _goto_window_for_buffer_name('__Answers__')
    vim.command('setlocal modifiable')
    lines = [line.encode('utf-8').replace("\n", "\r") for line in lines]
    vim.current.buffer[:] = lines
    vim.command('silent %s/\r\+/\n/ge')
    vim.command('silent %s/\%x00/\r/ge')
    vim.command('setlocal nomodifiable')


def _generate_stack_answers_format(posts):
    response = []
    for post in posts:
        answers = len(post["answers"])
        # If there is no question, pass
        if post["question"] == "":
            continue

        question = "Q: " + post["question"]
        response.append(question)
        response.append("%d Answer(s)" % answers)
        for answer in post["answers"]:
            response.append("=" * 80)
            response.append(answer[1] + " Upvotes: " + str(answer[3]))
            response.append(strip_html(answer[0]) + "\r")
    return response


def stackAnswers(query, _filter):
    query = vim.eval("a:2")
    _filter = vim.eval("g:stack_filter")
    data = fetch_mass_data(query, _filter)
    _output_preview_text(_generate_stack_answers_format(data))<commit_msg>Output message if not connected to the Internet<commit_after>import re
import json
import requests
import vim
import sys

API_KEY = "<KEY>


def strip_html(html):
    pattern = re.compile(r'<.*?>')
    return pattern.sub('', html)


def query_google(query, domain):
    search = "https://ajax.googleapis.com/ajax/services/search/web?v=1.0&q=%s"
    url = search % (query + ":" + domain)
    try:
        response = requests.get(url)
    except:
        return None
    data = json.loads(response.text)["responseData"]["results"]
    urls = []
    for result in data:
        url = result["url"]
        if is_valid_url(url):
            urls.append(url)
    return urls


def get_question_id(url):
    try:
        qid = int(url.split("/")[-2])
    except:
        qid = None
    return qid


def is_valid_url(url):
    return "stackoverflow" in url and "/tagged/" not in url


def get_question_data(qid):
    response = requests.get("https://api.stackexchange.com/2.2/questions/%s/answers?order=&sort=votes&site=stackoverflow&key=%s&filter=!)Q2B_4mND07Uc*hKpm6.P0Q5" % (qid, API_KEY))
    return json.loads(response.text)["items"]


def parse_question_data(data, _filter):
    post = {
        "question": "",
        "answers": []
    }
    if len(data) == 0 or "title" not in data[0]:
        return post
    post["question"] = data[0]["title"]
    for answer in data:
        answer_data = parse_answer(answer)
        if _filter == "accepted":
            if answer_data[2]:
                post["answers"].append(parse_answer(answer))
        elif _filter == "top":
            post["answers"].append(parse_answer(answer))
            break
        else:
            post["answers"].append(parse_answer(answer))
    return post


def parse_answer(answer):
    author = answer["owner"]["display_name"]
    content = answer["body"]
    is_accepted = answer["is_accepted"]
    upvotes = answer["score"]
    url = answer["share_link"]
    return [content, url, is_accepted, upvotes, author]


def fetch_mass_data(query, _filter):
    urls = query_google(query, "www.stackoverflow.com")
    if urls is None:
        return None
    posts = []
    for url in urls:
        qid = get_question_id(url)
        if not qid:
            continue
        data = get_question_data(qid)
        posts.append(parse_question_data(data, _filter))
    return posts


# StackAnswers output ---------------------------------------------------------


def _goto_window_for_buffer(b):
    w = int(vim.eval('bufwinnr(%d)' % int(b)))
    vim.command('%dwincmd w' % w)


def _goto_window_for_buffer_name(bn):
    b = vim.eval('bufnr("%s")' % bn)
    return _goto_window_for_buffer(b)


def _output_preview_text(lines):
    _goto_window_for_buffer_name('__Answers__')
    vim.command('setlocal modifiable')
    lines = [line.encode('utf-8').replace("\n", "\r") for line in lines]
    vim.current.buffer[:] = lines
    vim.command('silent %s/\r\+/\n/ge')
    vim.command('silent %s/\%x00/\r/ge')
    vim.command('setlocal nomodifiable')


def _generate_stack_answers_format(posts):
    response = []
    for post in posts:
        answers = len(post["answers"])
        # If there is no question, pass
        if post["question"] == "":
            continue

        question = "Q: " + post["question"]
        response.append(question)
        response.append("%d Answer(s)" % answers)
        for answer in post["answers"]:
            response.append("=" * 80)
            response.append(answer[1] + " Upvotes: " + str(answer[3]))
            response.append(strip_html(answer[0]) + "\r")
    return response


def stackAnswers(query, _filter):
    query = vim.eval("a:2")
    _filter = vim.eval("g:stack_filter")
    data = fetch_mass_data(query, _filter)
    if data is None:
        _output_preview_text(["Not connected to the Internet"])
    else:
        _output_preview_text(_generate_stack_answers_format(data))
{'fraction_non_alphanumeric': 0.15702588483888008, 'fraction_numerical': 0.006603275224511358, 'mean_word_length': 2.749009900990099, 'pattern_counts': {'":': 12, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '480956', 'n_tokens_mistral': 2556, 'n_tokens_neox': 2479, 'n_words': 502}