<commit_before># -*- coding: utf-8 -*-
"""
    grepo_base.github
    ~~~~~~~~~~~~~~~~~

    Github.com backend for `grepo`.
"""
import itertools
from httplib import HTTPConnection
from datetime import datetime

from django.conf import settings
from django.utils import simplejson as json

from grepo_base.models import Repository, Language


#: Github api search path
SEARCH_PATH = "/api/v2/json/repos/search/language:{lang}?start_page={page}"
#: Github host, used for httplib for connect to api
GITHUB = "github.com"


def fetch(language, page):
    connection = HTTPConnection(GITHUB)
    connection.request("GET", SEARCH_PATH.format(lang=language, page=page))
    response = connection.getresponse()
    foo = response.read()
    return foo


def list():
    """Yields all repositories one by one."""
    for language in Language.objects.all():
        for page in itertools.count(1):
            data = fetch(language.name, page)
            repositories = json.loads(data)["repositories"]

            if not repositories:
                break

            for repository in repositories:
                if not repository["language"]:
                    continue

                # Note: `source` and `language` field should be handled
                # by the caller.
                yield {
                    "url": repository["url"],
                    "name": repository["name"],
                    "language": repository["language"],
                    "summary": repository["description"],
                    "score": calculate_repository_score(repository),
                    "updated_at": datetime.utcnow(),
                    "created_at": datetime.utcnow()
                }


def update(repository):
    return repository  # A simple pass-through for now.


def calculate_repository_score(data):
    """Calculates and returns Grepo-score for a given repository.

    .. todo:: query for pull requests and add them to the exponent
              argument.
    """
    parse = lambda d: datetime.strptime(d, "%Y/%m/%dT %H:%M:%S %z")

    data.update(
        created_at=parse(data["created_at"]),
        pushed_at=parse(data["pushed_at"])
    )

    return (data["created_at"] - data["pushed_at"]) * math.exp(
        1 / (data["open_issues"] + data["watchers"] / data["forks"])
    )<commit_msg>Fixed a typo in `github.calculate_repository_score()`<commit_after># -*- coding: utf-8 -*-
"""
    grepo_base.github
    ~~~~~~~~~~~~~~~~~

    Github.com backend for `grepo`.
"""
import itertools
from httplib import HTTPConnection
from datetime import datetime

from django.conf import settings
from django.utils import simplejson as json

from grepo_base.models import Repository, Language


#: Github api search path
SEARCH_PATH = "/api/v2/json/repos/search/language:{lang}?start_page={page}"
#: Github host, used for httplib for connect to api
GITHUB = "github.com"


def fetch(language, page):
    connection = HTTPConnection(GITHUB)
    connection.request("GET", SEARCH_PATH.format(lang=language, page=page))
    response = connection.getresponse()
    foo = response.read()
    return foo


def list():
    """Yields all repositories one by one."""
    for language in Language.objects.all():
        for page in itertools.count(1):
            data = fetch(language.name, page)
            repositories = json.loads(data)["repositories"]

            if not repositories:
                break

            for repository in repositories:
                if not repository["language"]:
                    continue

                # Note: `source` and `language` field should be handled
                # by the caller.
                yield {
                    "url": repository["url"],
                    "name": repository["name"],
                    "language": repository["language"],
                    "summary": repository["description"],
                    "score": calculate_repository_score(repository),
                    "updated_at": datetime.utcnow(),
                    "created_at": datetime.utcnow()
                }


def update(repository):
    return repository  # A simple pass-through for now.


def calculate_repository_score(data):
    """Calculates and returns Grepo-score for a given repository.

    .. todo:: query for pull requests and add them to the exponent
              argument.
    """
    parse = lambda d: datetime.strptime(d, "%Y/%m/%dT %H:%M:%S %z")

    data.update(
        created_at=parse(data["created_at"]),
        pushed_at=parse(data["pushed_at"])
    )

    return (data["created_at"] - data["pushed_at"]).days * math.exp(
        1 / (data["open_issues"] + data["watchers"] / data["forks"])
    )
{'fraction_non_alphanumeric': 0.1237465329635161, 'fraction_numerical': 0.0017068487305312568, 'mean_word_length': 2.370237239396118, 'pattern_counts': {'":': 14, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '800493', 'n_tokens_mistral': 1314, 'n_tokens_neox': 1243, 'n_words': 348}