<commit_before>import argparse
from base64 import b64encode
from base64 import b64decode
import codecs
from collections import defaultdict
from collections import namedtuple
from bs4 import BeautifulSoup
from github import Github
from github import GithubException
from jinja2 import Template
from jinja2 import UndefinedError
from pkg_resources import parse_version
import re
import requests
import subprocess
from tqdm import tqdm
import yaml


pypi_pkg_uri = 'https://pypi.python.org/pypi/{}/json'.format

fs_tuple = namedtuple('fs_status', 'success '
                                   'needs_update '
                                   'data')

status_data_tuple = namedtuple('status_data', 'text '
                                              'yaml_strs '
                                              'pypi_version '
                                              'reqs '
                                              'blob_sha')


def pypi_org_sha(package_name, version, bundle_type):
    """
    Scrape pypi.org for SHA256 of the source bundle
    :param str package_name: Name of package (PROPER case)
    :param str version: version for which to get sha
    :param str bundle_type: ".tar.gz", ".zip" - format of bundle
    :returns: `str` -- SHA256 for a source bundle
    """
    r = requests.get('https://pypi.org/project/{}/{}/#files'.format(
        package_name,
        version))

    bs = BeautifulSoup(r.text, 'html5lib')
    try:
        sha_val = bs.find('a',
                          {'href':
                           re.compile('https://files.pythonhosted.org.*{}-{}{}'.
                                      format(package_name,
                                             version,
                                             bundle_type))
                           }).next.next.next['data-clipboard-text']
    except AttributeError:
        # Bad parsing of page, couldn't get SHA256
        return None

    return sha_val


def pypi_version_str(package_name):
    """
    Retrive the latest version of a package in pypi
    :param str package_name:
    :return: `str` -- Version string
    """
    r = requests.get(pypi_pkg_uri(package_name))
    if not r.ok:
        return False
    return r.json()['info']['version'].strip()


def parsed_meta_yaml(text):
    """
    :param str text: The raw text in conda-forge feedstock meta.yaml file
    :return: `dict|None` -- parsed YAML dict if successful, None if not
    """
    try:
        yaml_dict = yaml.load(Template(text).render())
    except UndefinedError:
        # assume we hit a RECIPE_DIR reference in the vars and can't parse it.
        # just erase for now
        try:
            yaml_dict = yaml.load(
                Template(
                    re.sub('{{ (environ\[")?RECIPE_DIR("])? }}/', '',
                           text)
                ).render())
        except:
            return None
    except:
        return None

    return yaml_dict


def basic_patch(text, yaml_strs, pypi_version, blob_sha):
    """
    Given a meta.yaml file, version strings, and appropriate hashes,
    find and replace old versions and hashes, and create a patch.
    :param str text: The raw text
    :param dict yaml_strs:
    :param str pypi_version:
    :param str blob_sha:
    :return: `tpl(bool,str|dict)` -- True if success and commit dict for github, false and error otherwise
    """
    pypi_sha = pypi_org_sha(
        '-'.join(yaml_strs['source_fn'].split('-')[:-1]),
        pypi_version,
        yaml_strs['source_fn'].split(yaml_strs['version'])[-1]
    )

    if pypi_sha is None:
        return False, 'Couldn\'t get SHA from PyPI'

    if text.find(yaml_strs['version']) < 0 or text.find(yaml_strs['sha256']) < 0:
        # if we can't change both the version and the hash
        # do nothing
        return False, 'Couldn\'t find current version or SHA in meta.yaml'

    new_text = text.replace(yaml_strs['version'], pypi_version).\
        replace(yaml_strs['source']['sha256'], pypi_sha)

    commit_dict = {
        'message': 'Tick version to {}'.format(pypi_version),
        'content': b64encode(new_text.encode('utf-8')).decode('utf-8'),
        'sha': blob_sha
    }

    return True, commit_dict


def user_feedstocks(user):
    """
    :param github.AuthenticatedUser.AutheticatedUser user:
    :return: `list` -- list of conda-forge feedstocks the user maintains
    """
    feedstocks = []
    for team in tqdm(user.get_teams()):

        # Each conda-forge team manages one feedstock
        # If a team has more than one repo, skip it.
        if team.repos_count != 1:<commit_msg>Fix dict key: [‘source’][‘sha256’] -> [‘sha256’]<commit_after>import argparse
from base64 import b64encode
from base64 import b64decode
import codecs
from collections import defaultdict
from collections import namedtuple
from bs4 import BeautifulSoup
from github import Github
from github import GithubException
from jinja2 import Template
from jinja2 import UndefinedError
from pkg_resources import parse_version
import re
import requests
import subprocess
from tqdm import tqdm
import yaml


pypi_pkg_uri = 'https://pypi.python.org/pypi/{}/json'.format

fs_tuple = namedtuple('fs_status', 'success '
                                   'needs_update '
                                   'data')

status_data_tuple = namedtuple('status_data', 'text '
                                              'yaml_strs '
                                              'pypi_version '
                                              'reqs '
                                              'blob_sha')


def pypi_org_sha(package_name, version, bundle_type):
    """
    Scrape pypi.org for SHA256 of the source bundle
    :param str package_name: Name of package (PROPER case)
    :param str version: version for which to get sha
    :param str bundle_type: ".tar.gz", ".zip" - format of bundle
    :returns: `str` -- SHA256 for a source bundle
    """
    r = requests.get('https://pypi.org/project/{}/{}/#files'.format(
        package_name,
        version))

    bs = BeautifulSoup(r.text, 'html5lib')
    try:
        sha_val = bs.find('a',
                          {'href':
                           re.compile('https://files.pythonhosted.org.*{}-{}{}'.
                                      format(package_name,
                                             version,
                                             bundle_type))
                           }).next.next.next['data-clipboard-text']
    except AttributeError:
        # Bad parsing of page, couldn't get SHA256
        return None

    return sha_val


def pypi_version_str(package_name):
    """
    Retrive the latest version of a package in pypi
    :param str package_name:
    :return: `str` -- Version string
    """
    r = requests.get(pypi_pkg_uri(package_name))
    if not r.ok:
        return False
    return r.json()['info']['version'].strip()


def parsed_meta_yaml(text):
    """
    :param str text: The raw text in conda-forge feedstock meta.yaml file
    :return: `dict|None` -- parsed YAML dict if successful, None if not
    """
    try:
        yaml_dict = yaml.load(Template(text).render())
    except UndefinedError:
        # assume we hit a RECIPE_DIR reference in the vars and can't parse it.
        # just erase for now
        try:
            yaml_dict = yaml.load(
                Template(
                    re.sub('{{ (environ\[")?RECIPE_DIR("])? }}/', '',
                           text)
                ).render())
        except:
            return None
    except:
        return None

    return yaml_dict


def basic_patch(text, yaml_strs, pypi_version, blob_sha):
    """
    Given a meta.yaml file, version strings, and appropriate hashes,
    find and replace old versions and hashes, and create a patch.
    :param str text: The raw text
    :param dict yaml_strs:
    :param str pypi_version:
    :param str blob_sha:
    :return: `tpl(bool,str|dict)` -- True if success and commit dict for github, false and error otherwise
    """
    pypi_sha = pypi_org_sha(
        '-'.join(yaml_strs['source_fn'].split('-')[:-1]),
        pypi_version,
        yaml_strs['source_fn'].split(yaml_strs['version'])[-1]
    )

    if pypi_sha is None:
        return False, 'Couldn\'t get SHA from PyPI'

    if text.find(yaml_strs['version']) < 0 or text.find(yaml_strs['sha256']) < 0:
        # if we can't change both the version and the hash
        # do nothing
        return False, 'Couldn\'t find current version or SHA in meta.yaml'

    new_text = text.replace(yaml_strs['version'], pypi_version).\
        replace(yaml_strs['sha256'], pypi_sha)

    commit_dict = {
        'message': 'Tick version to {}'.format(pypi_version),
        'content': b64encode(new_text.encode('utf-8')).decode('utf-8'),
        'sha': blob_sha
    }

    return True, commit_dict


def user_feedstocks(user):
    """
    :param github.AuthenticatedUser.AutheticatedUser user:
    :return: `list` -- list of conda-forge feedstocks the user maintains
    """
    feedstocks = []
    for team in tqdm(user.get_teams()):

        # Each conda-forge team manages one feedstock
        # If a team has more than one repo, skip it.
        if team.repos_count != 1:
{'fraction_non_alphanumeric': 0.10423205974672584, 'fraction_numerical': 0.008442472129018292, 'mean_word_length': 2.0225711481844946, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 4, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1425582', 'n_tokens_mistral': 2782, 'n_tokens_neox': 2595, 'n_words': 773}