<commit_before>#!/usr/bin/env python
#
# Script to extract and plot Google Scholar citations. Useful for reporting track record in grant applications, etc.
#

import urllib.request
from html.parser import HTMLParser
import re

import seaborn as sea
import matplotlib.pyplot as plt
sea.set(font_scale=1.2)


class GoogleScholarHTMLParser(HTMLParser):
    """Parser to extract citation information from Google Scholar page."""

    def __init__(self):
        super(GoogleScholarHTMLParser, self).__init__()

        self.insideSpanClass = []
        self.insideTdClass = []
        self.citeYears = []
        self.citeCounts = []
        self.citeSummaryData = []

    @staticmethod
    def get_attr(attrs, key='class'):
        for (k, v) in attrs:
            if k == key:
                return v

        return None

    def handle_starttag(self, tag, attrs):
        if (tag == 'span'):
            self.insideSpanClass.append(self.get_attr(attrs, 'class'))
        if (tag == 'td'):
            self.insideTdClass.append(self.get_attr(attrs, 'class'))

    def handle_data(self, data):
        if len(self.insideSpanClass) > 0:
            if self.insideSpanClass[-1] == 'gsc_g_t':
                self.citeYears.append(int(data.strip()))
            if self.insideSpanClass[-1] == 'gsc_g_al':
                self.citeCounts.append(int(data.strip()))
        if len(self.insideTdClass) > 0:
            if self.insideTdClass[-1] == 'gsc_rsb_std':
                self.citeSummaryData.append(int(data.strip()))

    def handle_endtag(self, tag):
        if (tag == 'span'):
            self.insideSpanClass.pop()
        if (tag == 'td'):
            self.insideTdClass.pop()


if __name__ == "__main__":

    # Google Scholar page to process
    URL = r"https://scholar.google.com.au/citations?user=YvdzeM8AAAAJ&hl=en"

    print("Fetching Google Scholar page...")
    response = urllib.request.urlopen(URL)
    html = str(response.read())
    response.close()

    html = re.sub("\\\\t|\\\\r\\\\n", "", html)

    print("Parsing HTML...")
    parser = GoogleScholarHTMLParser()
    parser.feed(html)
    print("...{} total citations".format(parser.citeSummaryData[0]))
    #print(parser.citeCounts)

    print("Plotting Citations...")

    plt.figure()
    width = 0.8
    plt.bar(parser.citeYears, parser.citeCounts, width)
    plt.plot([year + 0.5 * width for year in parser.citeYears], parser.citeCounts, 'ro-', lw=2)
    plt.xticks([year + 0.5 * width for year in parser.citeYears], parser.citeYears)
    plt.xlabel('Year'); plt.ylabel('Citations')
    plt.show()<commit_msg>improved formatting<commit_after>#!/usr/bin/env python
#
# Script to extract and plot Google Scholar citations. Useful for reporting track record in grant applications, etc.
#

import urllib.request
from html.parser import HTMLParser
import re

import seaborn as sea
import matplotlib.pyplot as plt
sea.set(font_scale=1.4)

from datetime import datetime


class GoogleScholarHTMLParser(HTMLParser):
    """Parser to extract citation information from Google Scholar page."""

    def __init__(self):
        super(GoogleScholarHTMLParser, self).__init__()

        self.insideSpanClass = []
        self.insideTdClass = []
        self.citeYears = []
        self.citeCounts = []
        self.citeSummaryData = []

    @staticmethod
    def get_attr(attrs, key='class'):
        for (k, v) in attrs:
            if k == key:
                return v

        return None

    def handle_starttag(self, tag, attrs):
        if (tag == 'span'):
            self.insideSpanClass.append(self.get_attr(attrs, 'class'))
        if (tag == 'td'):
            self.insideTdClass.append(self.get_attr(attrs, 'class'))

    def handle_data(self, data):
        if len(self.insideSpanClass) > 0:
            if self.insideSpanClass[-1] == 'gsc_g_t':
                self.citeYears.append(int(data.strip()))
            if self.insideSpanClass[-1] == 'gsc_g_al':
                self.citeCounts.append(int(data.strip()))
        if len(self.insideTdClass) > 0:
            if self.insideTdClass[-1] == 'gsc_rsb_std':
                self.citeSummaryData.append(int(data.strip()))

    def handle_endtag(self, tag):
        if (tag == 'span'):
            self.insideSpanClass.pop()
        if (tag == 'td'):
            self.insideTdClass.pop()


if __name__ == "__main__":

    # Google Scholar page to process
    URL = r"https://scholar.google.com.au/citations?user=YvdzeM8AAAAJ&hl=en"

    print("Fetching Google Scholar page...")
    response = urllib.request.urlopen(URL)
    html = str(response.read())
    response.close()

    html = re.sub("\\\\t|\\\\r\\\\n", "", html)

    print("Parsing HTML...")
    parser = GoogleScholarHTMLParser()
    parser.feed(html)
    print("...{} total citations".format(parser.citeSummaryData[0]))
    #print(parser.citeCounts)

    year_fraction = datetime.now().timetuple().tm_yday / 365.0
    year_prediction = int(parser.citeCounts[-1] / year_fraction)
    print("{:0.1f}% of year with {} citations ({} predicted)".format(100.0 * year_fraction, parser.citeCounts[-1], year_prediction))

    print("Plotting Citations...")

    plt.figure()
    width = 0.8

    #plt.bar(parser.citeYears[-1], year_prediction, width, color=[1.0, 1.0, 1.0])
    #plt.plot([year + 0.5 * width for year in parser.citeYears[-2:]], [parser.citeCounts[-2], year_prediction], 'ko--', lw=2)

    plt.bar(parser.citeYears, parser.citeCounts, width, color=[0.75, 0.75, 0.75])
    plt.plot([year + 0.5 * width for year in parser.citeYears], parser.citeCounts, 'ko-', lw=2)
    plt.xticks([year + 0.5 * width for year in parser.citeYears], parser.citeYears)
    plt.xlabel('Year'); plt.ylabel('Citations')
    plt.tight_layout(pad=0.2)
    plt.show()
{'fraction_non_alphanumeric': 0.14596760146315974, 'fraction_numerical': 0.0116704406897753, 'mean_word_length': 2.74071661237785, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '462244', 'n_tokens_mistral': 1952, 'n_tokens_neox': 1841, 'n_words': 396}