<commit_before>import re
import os
import sys
import requests

# Matches a valid DOI
doi_matcher = re.compile('\b(10[.][0-9]{4,}(?:[.][0-9]+)*/(?:(?!["&\'<>])\S)+)\b')

# Matches a [citation_key DOI] pair
key_file_matcher = re.compile('(\S+)\s+(\S+)')

# Matches the beginning of a BibTeX entry
bibtex_key_matcher = re.compile('@\w+{(.+?),')


def convert_file(filename, driver, out_to_file, verbose):
    """Convert a file in the format [citation_key doi_key]* into a valid bibtex file"""

    if verbose:
        print("Retrieving entries from {}".format(driver.url))

    doi_entries = parse_file(filename, verbose)

    bib_entries = get_all_bibliography_data(doi_entries, driver, verbose)

    bib_entries = replace_citation_keys(bib_entries)

    writefn = write_to_file if out_to_file else write_to_stdout

    writefn(bib_entries, filename, verbose)

def write_to_file(entries, filename, verbose):
    (root, ext) = os.path.splitext(filename)
    bibtex_file = root + ".bib"

    with open(bibtex_file, 'w') as bib:
        for citation in entries:
            print(citation, file=bib)

    if verbose:
        print("Wrote generated BibTeX file as {}".format(bibtex_file))

def write_to_stdout(entries, filename, verbose):
    for citation in entries:
        print(citation)

def parse_file(filename, verbose):
    """Parses a file in the format [citation_key doi_key]* into a dictionary {citation_key: doi_key}*"""

    doi_entries = {}

    with open(filename) as f:
        lines = f.readlines()

    for i, line in enumerate(lines, start=1):

        parsed_line = parse_line(i, line)

        if parsed_line is None:
            # Line was blank, or a comment
            continue

        citation_key, doi = parsed_line

        # Don't allow duplicate citation keys
        if citation_key in doi_entries:
            error("Line {}: duplicate citation key: {}".format(i, citation_key))

        doi_entries[citation_key] = doi

    if verbose:
        print("Found {} citations in {}".format(len(doi_entries), filename))

    if not doi_entries:
        error("Nothing to do")

    return doi_entries

def parse_line(line_number, line):
    """Parses a single line, of the format [citation_key DOI]"""

    if line.isspace():
        return None

    if line.startswith("#"):
        # Found a comment
        return None

    parsed_line = key_file_matcher.match(line)

    if not parsed_line:
        error("Line {}: Syntax error ({})".format(line_number, line.strip()))

    citation_key, doi = parsed_line.group(1, 2)

    # Limit the length of citation keys
    if len(citation_key) > 20:
        error("Line {}: Citation key '{}' is over 20 characters".format(line_number, citation_key))

    # if not doi_matcher.match(doi):
        # error("Line {}: Invalid DOI {}".format(line_number, doi))

    return citation_key, doi

def get_all_bibliography_data(doi_entries, driver, verbose):
    """Takes a dict in the format {citation_key: doi_key}* and expands each doi key into its bibliography text"""

    if verbose:<commit_msg>Remove limit on citation key length<commit_after>import re
import os
import sys
import requests

# Matches a valid DOI
doi_matcher = re.compile('\b(10[.][0-9]{4,}(?:[.][0-9]+)*/(?:(?!["&\'<>])\S)+)\b')

# Matches a [citation_key DOI] pair
key_file_matcher = re.compile('(\S+)\s+(\S+)')

# Matches the beginning of a BibTeX entry
bibtex_key_matcher = re.compile('@\w+{(.+?),')


def convert_file(filename, driver, out_to_file, verbose):
    """Convert a file in the format [citation_key doi_key]* into a valid bibtex file"""

    if verbose:
        print("Retrieving entries from {}".format(driver.url))

    doi_entries = parse_file(filename, verbose)

    bib_entries = get_all_bibliography_data(doi_entries, driver, verbose)

    bib_entries = replace_citation_keys(bib_entries)

    writefn = write_to_file if out_to_file else write_to_stdout

    writefn(bib_entries, filename, verbose)

def write_to_file(entries, filename, verbose):
    (root, ext) = os.path.splitext(filename)
    bibtex_file = root + ".bib"

    with open(bibtex_file, 'w') as bib:
        for citation in entries:
            print(citation, file=bib)

    if verbose:
        print("Wrote generated BibTeX file as {}".format(bibtex_file))

def write_to_stdout(entries, filename, verbose):
    for citation in entries:
        print(citation)

def parse_file(filename, verbose):
    """Parses a file in the format [citation_key doi_key]* into a dictionary {citation_key: doi_key}*"""

    doi_entries = {}

    with open(filename) as f:
        lines = f.readlines()

    for i, line in enumerate(lines, start=1):

        parsed_line = parse_line(i, line)

        if parsed_line is None:
            # Line was blank, or a comment
            continue

        citation_key, doi = parsed_line

        # Don't allow duplicate citation keys
        if citation_key in doi_entries:
            error("Line {}: duplicate citation key: {}".format(i, citation_key))

        doi_entries[citation_key] = doi

    if verbose:
        print("Found {} citations in {}".format(len(doi_entries), filename))

    if not doi_entries:
        error("Nothing to do")

    return doi_entries

def parse_line(line_number, line):
    """Parses a single line, of the format [citation_key DOI]"""

    if line.isspace():
        return None

    if line.startswith("#"):
        # Found a comment
        return None

    parsed_line = key_file_matcher.match(line)

    if not parsed_line:
        error("Line {}: Syntax error ({})".format(line_number, line.strip()))

    citation_key, doi = parsed_line.group(1, 2)

    if not doi_matcher.match(doi):
        error("Line {}: Invalid DOI {}".format(line_number, doi))

    return citation_key, doi

def get_all_bibliography_data(doi_entries, driver, verbose):
    """Takes a dict in the format {citation_key: doi_key}* and expands each doi key into its bibliography text"""

    if verbose:
{'fraction_non_alphanumeric': 0.121263016459523, 'fraction_numerical': 0.004030903594222372, 'mean_word_length': 3.3246187363834423, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '602019', 'n_tokens_mistral': 2057, 'n_tokens_neox': 1830, 'n_words': 539}