<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import (division, print_function, absolute_import,
                        unicode_literals)

__all__ = []

import re


p = re.compile(r"([0-9]{4}\.[0-9]{4})"
               r"|([a-zA-Z\-\.]+?/[0-9]+)"
               r"|([a-zA-Z\-\.]+?\?[0-9]+)"
               r"|([a-zA-Z\-\.]+?/\?[0-9]+)"
               r"|([a-zA-Z\-\.]+?\?=[0-9]+)"
               r"|([a-zA-Z\-\.]+?%2F[0-9]+)"
               r"|([a-zA-Z\-\.]+?%2f[0-9]+)"
               r"|([a-zA-Z\-\.]+?\$/\$[0-9]+)"
               r"|([a-zA-Z\-\.]+?\.[0-9]+)"
               r"|([a-zA-Z\-\.]+?\?papernum=[0-9]+)")
p_pref = re.compile(r"([0-9]{4}\.[0-9]{4})")


def get_id(r):
    for i, el in enumerate(r):
        if len(el):
            if i == 0 or i == 1:
                return el
            elif i == 2:
                return "/".join(el.split("?"))
            elif i == 3:
                return "/".join(el.split("/?"))
            elif i == 4:
                return "/".join(el.split("?="))
            elif i == 5:
                return "/".join(el.split("%2F"))
            elif i == 6:
                return "/".join(el.split("%2f"))
            elif i == 7:
                return "/".join(el.split("$/$"))
            elif i == 8:
                return "/".join(["".join(el.split(".")[:-1]),
                                 el.split(".")[-1]])
            else:
                return "/".join(el.split("?papernum="))
    return None


def parse_linkouts(fn):
    final = dict()
    for line in open(fn):
        article_id = line.split("|")[0]

        results = p.findall(line)
        ids = map(get_id, results)
        if len(ids) == 0:
            continue

        if all([i == ids[0] for i in ids[1:]]):
            final[article_id] = ids[0]

        else:
            for i in ids:
                if len(p_pref.findall(i)):
                    final[article_id] = i
                    break

    return final


if __name__ == "__main__":
    parse_linkouts("citeulike/arxiv")
<commit_msg>typography<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import (division, print_function, absolute_import,
                        unicode_literals)

__all__ = ["parse_linkouts"]

import re


p = re.compile(r"([0-9]{4}\.[0-9]{4})"
               r"|([a-zA-Z\-\.]+?/[0-9]+)"
               r"|([a-zA-Z\-\.]+?\?[0-9]+)"
               r"|([a-zA-Z\-\.]+?/\?[0-9]+)"
               r"|([a-zA-Z\-\.]+?\?=[0-9]+)"
               r"|([a-zA-Z\-\.]+?%2F[0-9]+)"
               r"|([a-zA-Z\-\.]+?%2f[0-9]+)"
               r"|([a-zA-Z\-\.]+?\$/\$[0-9]+)"
               r"|([a-zA-Z\-\.]+?\.[0-9]+)"
               r"|([a-zA-Z\-\.]+?\?papernum=[0-9]+)")
p_pref = re.compile(r"([0-9]{4}\.[0-9]{4})")


def get_id(r):
    for i, el in enumerate(r):
        if len(el):
            if i == 0 or i == 1:
                return el
            elif i == 2:
                return "/".join(el.split("?"))
            elif i == 3:
                return "/".join(el.split("/?"))
            elif i == 4:
                return "/".join(el.split("?="))
            elif i == 5:
                return "/".join(el.split("%2F"))
            elif i == 6:
                return "/".join(el.split("%2f"))
            elif i == 7:
                return "/".join(el.split("$/$"))
            elif i == 8:
                return "/".join(["".join(el.split(".")[:-1]),
                                 el.split(".")[-1]])
            else:
                return "/".join(el.split("?papernum="))
    return None


def parse_linkouts(fn):
    final = dict()
    for line in open(fn):
        article_id = line.split("|")[0]

        results = p.findall(line)
        ids = map(get_id, results)
        if len(ids) == 0:
            continue

        if all([i == ids[0] for i in ids[1:]]):
            final[article_id] = ids[0]

        else:
            for i in ids:
                if len(p_pref.findall(i)):
                    final[article_id] = i
                    break
    return final


if __name__ == "__main__":
    parse_linkouts("citeulike/arxiv")

{'fraction_non_alphanumeric': 0.23569350145489815, 'fraction_numerical': 0.024733268671193017, 'mean_word_length': 1.2590361445783131, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5284459', 'n_tokens_mistral': 1582, 'n_tokens_neox': 1464, 'n_words': 259}