<commit_before>from __future__ import print_function

import os
import re
import sys
import json
import warnings
import requests
from tqdm import tqdm
from utils import LogUtil
from ioutils import read_lines
from tika_parser import TikaParser
from collections import OrderedDict

# Always printing matching warnings
warnings.filterwarnings('always')


class AdsParser(TikaParser):
    """ The Ads parser class utilizes the RESTful API from the Astrophysics
    Data System (ADS) to extract the primary author, author list, author
    affiliations, publication venue.
    """
    def __init__(self, ads_token, ads_base_url, tika_server_url):
        super(AdsParser, self).__init__(tika_server_url)
        self.ads_token = ads_token
        self.ads_base_url = ads_base_url

    @staticmethod
    def escape_solr_chars(text):
        escape_rules = OrderedDict()
        escape_rules['\\'] = r'\\'  # Note "\" must be escaped first.
        escape_rules['+'] = r'\+'
        escape_rules['-'] = r'\-'
        escape_rules['&&'] = r'\&&'
        escape_rules['||'] = r'\||'
        escape_rules['!'] = r'\!'
        escape_rules['('] = r'\('
        escape_rules[')'] = r'\)'
        escape_rules['{'] = r'\{'
        escape_rules['}'] = r'\}'
        escape_rules['['] = r'\['
        escape_rules[']'] = r'\]'
        escape_rules['^'] = r'\^'
        escape_rules['~'] = r'\~'
        escape_rules['*'] = r'\*'
        escape_rules[':'] = r'\:'
        escape_rules['/'] = r'\/'
        escape_rules['"'] = r'\"'

        for c, r in escape_rules.items():
            text = text.replace(c, r)

        return text

    @staticmethod
    def special_rules(text):
        # Rule 1: remove ? in the title
        if '?' in text:
            text = text.replace('?', '')

        # Rule 2: convert title to all lower cases
        text = text.lower()

        # Rule 3: When a title contains a period at the end, grobid tool
        # consistently extract the title with the period and the first character
        # of the author name.
        # For example, the title, author names, and affiliations of a LPSC
        # abstract (1998_1589.pdf) look like below:
        # IMPLICATIONS OF THE APPARENT ABSENCE OF MAARS IN VIKING ORBITER
        # IMAGERY. <NAME> and <NAME>, Environmental Science
        # Department, Institute of Environmental and Natural Sciences,
        # Lancaster University, Lancaster LA1 4YQ, UK
        # (<EMAIL>).
        #
        # The title extracted by the grobid tool is:
        # IMPLICATIONS OF THE APPARENT ABSENCE OF MAARS IN VIKING ORBITER
        # IMAGERY. K
        #
        # We need to remove ". K" before searching the ADS database.
        text = re.sub(r'\. \w$', '', text)

        # Rule 4: Remove LPSC header in grobid titles
        text = re.sub(r'Lunar and Planetary Science [CDILVXM]+ .+\.pdf$', '',
                      text, flags=re.IGNORECASE)

        return text

    def query_ads_database(self, title):
        ads_dict = dict()

        headers = {
            'Authorization': 'Bearer %s' % self.ads_token
        }

        title = self.escape_solr_chars(title)
        title = self.special_rules(title)
        params = (
            ('q', 'title:"%s"' % title),
            ('fl', 'first_author,author,aff,pubdate,year,pub')
        )<commit_msg>escape Lucene reserved keyword NEAR and NOT #27<commit_after>from __future__ import print_function

import os
import re
import sys
import json
import warnings
import requests
from tqdm import tqdm
from utils import LogUtil
from ioutils import read_lines
from tika_parser import TikaParser
from collections import OrderedDict

# Always printing matching warnings
warnings.filterwarnings('always')


class AdsParser(TikaParser):
    """ The Ads parser class utilizes the RESTful API from the Astrophysics
    Data System (ADS) to extract the primary author, author list, author
    affiliations, publication venue.
    """
    def __init__(self, ads_token, ads_base_url, tika_server_url):
        super(AdsParser, self).__init__(tika_server_url)
        self.ads_token = ads_token
        self.ads_base_url = ads_base_url

    @staticmethod
    def escape_solr_chars(text):
        escape_rules = OrderedDict()
        escape_rules['\\'] = r'\\'  # Note "\" must be escaped first.
        escape_rules['+'] = r'\+'
        escape_rules['-'] = r'\-'
        escape_rules['&&'] = r'\&&'
        escape_rules['||'] = r'\||'
        escape_rules['!'] = r'\!'
        escape_rules['('] = r'\('
        escape_rules[')'] = r'\)'
        escape_rules['{'] = r'\{'
        escape_rules['}'] = r'\}'
        escape_rules['['] = r'\['
        escape_rules[']'] = r'\]'
        escape_rules['^'] = r'\^'
        escape_rules['~'] = r'\~'
        escape_rules['*'] = r'\*'
        escape_rules[':'] = r'\:'
        escape_rules['/'] = r'\/'
        escape_rules['"'] = r'\"'

        for c, r in escape_rules.items():
            text = text.replace(c, r)

        return text

    @staticmethod
    def special_rules(text):
        # Rule 1: remove ? in the title
        if '?' in text:
            text = text.replace('?', '')

        # Rule 2: convert title to all lower cases
        text = text.lower()

        # Rule 3: When a title contains a period at the end, grobid tool
        # consistently extract the title with the period and the first character
        # of the author name.
        # For example, the title, author names, and affiliations of a LPSC
        # abstract (1998_1589.pdf) look like below:
        # IMPLICATIONS OF THE APPARENT ABSENCE OF MAARS IN VIKING ORBITER
        # IMAGERY. <NAME> and <NAME>, Environmental Science
        # Department, Institute of Environmental and Natural Sciences,
        # Lancaster University, Lancaster LA1 4YQ, UK
        # (<EMAIL>).
        #
        # The title extracted by the grobid tool is:
        # IMPLICATIONS OF THE APPARENT ABSENCE OF MAARS IN VIKING ORBITER
        # IMAGERY. K
        #
        # We need to remove ". K" before searching the ADS database.
        text = re.sub(r'\. \w$', '', text)

        # Rule 4: Remove LPSC header in grobid titles
        text = re.sub(r'Lunar and Planetary Science [CDILVXM]+ .+\.pdf$', '',
                      text, flags=re.IGNORECASE)

        # Rule 5: If the title contains the keyword "NEAR" and "NOT", we add
        # double quotes around the title to restrict the Solr query to do exact
        # match.
        if 'near' in text or 'not' in text:
            text = '"%s"' % text

        return text

    def query_ads_database(self, title):
        ads_dict = dict()

        headers = {
            'Authorization': 'Bearer %s' % self.ads_token
        }

        title = self.escape_solr_chars(title)
        title = self.special_rules(title)
        params = (
            ('q', 'title:%s' % title),
            ('fl', 'first_author,author,aff,pubdate,year,pub')
        )
{'fraction_non_alphanumeric': 0.12221579404107608, 'fraction_numerical': 0.004483656349435927, 'mean_word_length': 2.2819174181300426, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '222864', 'n_tokens_mistral': 2143, 'n_tokens_neox': 2055, 'n_words': 657}