<commit_before>from github import Github, GithubException
from github_database import engine, Commit, File
from sqlalchemy.orm import sessionmaker
from ConfigParser import ConfigParser

PAGE_MAX_SIZE = 100
START_PAGE = 1


def _get_missing_commits_from_db(db_session, repo):
    db_commits = db_session \
        .query(Commit.sha) \
        .filter(Commit.repository_name == repo.name) \
        .all()
    page_num = len(db_commits)/PAGE_MAX_SIZE + START_PAGE
    commits = []

    try:
        commits = repo.get_commits().reversed.get_page(page_num)
    except GithubException as e:
        if e.status == 409 and e.data['message'] != unicode('Git Repository is empty.'):
            raise e

    return [commit for commit in commits \
            if commit.sha not in [db_commit.sha for db_commit in db_commits]]


def _get_new_commits(db_session, commit_api):
    commits = _get_missing_commits_from_db(db_session, commit_api)
    while commits:
        for commit in commits:
            yield commit
        commits = _get_missing_commits_from_db(db_session, commit_api)


def run():
    config = ConfigParser()
    config.read('config.ini')
    github_api = Github(login_or_token=config.get('Main', 'GithubAccessToken'), per_page=PAGE_MAX_SIZE)
    db_session = sessionmaker(bind=engine)()

    org_name = unicode('google')
    github_google = github_api.get_organization(org_name)

    for repo in github_google.get_repos():
        for commit in _get_new_commits(db_session, repo):
            db_commit = Commit(
                sha=commit.sha,
                date=commit.last_modified,
                message=commit.commit.message,<commit_msg>Removed an needed backslash between brackets<commit_after>from github import Github, GithubException
from github_database import engine, Commit, File
from sqlalchemy.orm import sessionmaker
from ConfigParser import ConfigParser

PAGE_MAX_SIZE = 100
START_PAGE = 1


def _get_missing_commits_from_db(db_session, repo):
    db_commits = db_session \
        .query(Commit.sha) \
        .filter(Commit.repository_name == repo.name) \
        .all()
    page_num = len(db_commits)/PAGE_MAX_SIZE + START_PAGE
    commits = []

    try:
        commits = repo.get_commits().reversed.get_page(page_num)
    except GithubException as e:
        if e.status == 409 and e.data['message'] != unicode('Git Repository is empty.'):
            raise e

    return [commit for commit in commits
            if commit.sha not in [db_commit.sha for db_commit in db_commits]]


def _get_new_commits(db_session, commit_api):
    commits = _get_missing_commits_from_db(db_session, commit_api)
    while commits:
        for commit in commits:
            yield commit
        commits = _get_missing_commits_from_db(db_session, commit_api)


def run():
    config = ConfigParser()
    config.read('config.ini')
    github_api = Github(login_or_token=config.get('Main', 'GithubAccessToken'), per_page=PAGE_MAX_SIZE)
    db_session = sessionmaker(bind=engine)()

    org_name = unicode('google')
    github_google = github_api.get_organization(org_name)

    for repo in github_google.get_repos():
        for commit in _get_new_commits(db_session, repo):
            db_commit = Commit(
                sha=commit.sha,
                date=commit.last_modified,
                message=commit.commit.message,
{'fraction_non_alphanumeric': 0.08759342301943199, 'fraction_numerical': 0.004185351270553065, 'mean_word_length': 3.192982456140351, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4756110', 'n_tokens_mistral': 1085, 'n_tokens_neox': 1018, 'n_words': 234}