<commit_before>from optparse import OptionParser
import pkg_resources
from graph import Graph
from grabber import Grabber

class Geneagrapher:
    """
    A class for building Graphviz "dot" files for math genealogies
    extracted from the Mathematics Genealogy Project website.
    """
    def __init__(self):
        self.graph = Graph()
        self.leaf_ids = []
        self.get_ancestors = False
        self.get_descendants = False
        self.verbose = False
        self.write_filename = None

    def parse_input(self):
        """
        Parse command-line information.
        """
        pkg_env = pkg_resources.Environment()
        self.parser = OptionParser(version="{}".format(
            pkg_env[self.__module__.split('.')[0]][0].version))

        self.parser.set_usage("%prog [options] ID [ID...]")
        self.parser.set_description('Create a Graphviz "dot" file for a mathematics genealogy, where ID is a record identifier from the Mathematics Genealogy Project. Multiple IDs may be passed.')

        self.parser.add_option("-f", "--file", dest="filename",
			       help="write output to FILE [default: stdout]",
			       metavar="FILE", default=None)
        self.parser.add_option("-a", "--with-ancestors", action="store_true",
			       dest="get_ancestors", default=False,
			       help="retrieve ancestors of IDs and include in graph")
        self.parser.add_option("-d", "--with-descendants", action="store_true",
			       dest="get_descendants", default=False,
			       help="retrieve descendants of IDs and include in graph")
        self.parser.add_option("--verbose", "-v", action="store_true", dest="verbose",
			       default=False, help="list nodes being retrieved")

        (options, args) = self.parser.parse_args()
        
        if len(args) == 0:
            self.parser.error("no record IDs given")

        self.get_ancestors = options.get_ancestors
        self.get_descendants = options.get_descendants
        self.verbose = options.verbose
        self.write_filename = options.filename
        self.leaf_ids = [int(arg) for arg in args]
        
    def build_graph(self):
        """
        Populate the graph member by grabbing the mathematician
        pages and extracting relevant data.
        """
        leaf_grab_queue = list(self.leaf_ids)
        ancestor_grab_queue = []
        descendant_grab_queue = []

        # Grab "leaf" nodes.
        while len(leaf_grab_queue) != 0:
            id = leaf_grab_queue.pop()
            if not self.graph.has_node(id):
                # Then this information has not yet been grabbed.
                grabber = Grabber(id)
                if self.verbose:
                    print "Grabbing record #{}".format(id)
                [name, institution, year, advisors, descendants] = grabber.extract_node_information()
                self.graph.add_node(name, institution, year, id, advisors, descendants, True)
                if self.get_ancestors:
                    ancestor_grab_queue += advisors
                if self.get_descendants:
                    descendant_grab_queue += descendants

        # Grab ancestors of leaf nodes.
        if self.get_ancestors:
            while len(ancestor_grab_queue) != 0:
                id = ancestor_grab_queue.pop()
                if not self.graph.has_node(id):
                    # Then this information has not yet been grabbed.
                    grabber = Grabber(id)
                    if self.verbose:
                        print "Grabbing record #{}".format(id)
                    [name, institution, year, advisors, descendants] = grabber.extract_node_information()
                    self.graph.add_node(name, institution, year, id, advisors, descendants)
                    ancestor_grab_queue += advisors
                        
        # Grab descendants of leaf nodes.
        if self.get_descendants:
            while len(descendant_grab_queue) != 0:
                id = descendant_grab_queue.pop()
                if not self.graph.has_node(id):
                    # Then this information has not yet been grabbed.
                    grabber = Grabber(id)
                    if self.verbose:
                        print "Grabbing record #{}".format(id)
                    [name, institution, year, advisors, descendants] = grabber.extract_node_information()
                    self.graph.add_node(name, institution, year, id, advisors, descendants)
                    descendant_grab_queue += descendants
                    
    def generate_dot_file(self):
        dotfile = self.graph.generate_dot_file(self.get_ancestors, self.get_descendants)
        dotfile = dotfile.encode('utf-8', 'replace')
        if self.write_filename is not None:
            outfile = open(self.write_filename, "w")
            outfile.write(dotfile)
            outfile.close()
        else:
            print dotfile,
        
def ggrapher():
    """Function to run the Geneagrapher. This is the function called when
    the ggrapher script is run."""
    ggrapher = Geneagrapher()
    ggrapher.parse_input()
    ggrapher.build_graph()
    ggrapher.generate_dot_file()
<commit_msg>Changed queue pop order so that it is actually a queue.<commit_after>from optparse import OptionParser
import pkg_resources
from graph import Graph
from grabber import Grabber

class Geneagrapher:
    """
    A class for building Graphviz "dot" files for math genealogies
    extracted from the Mathematics Genealogy Project website.
    """
    def __init__(self):
        self.graph = Graph()
        self.leaf_ids = []
        self.get_ancestors = False
        self.get_descendants = False
        self.verbose = False
        self.write_filename = None

    def parse_input(self):
        """
        Parse command-line information.
        """
        pkg_env = pkg_resources.Environment()
        self.parser = OptionParser(version="{}".format(
            pkg_env[self.__module__.split('.')[0]][0].version))

        self.parser.set_usage("%prog [options] ID [ID...]")
        self.parser.set_description('Create a Graphviz "dot" file for a mathematics genealogy, where ID is a record identifier from the Mathematics Genealogy Project. Multiple IDs may be passed.')

        self.parser.add_option("-f", "--file", dest="filename",
			       help="write output to FILE [default: stdout]",
			       metavar="FILE", default=None)
        self.parser.add_option("-a", "--with-ancestors", action="store_true",
			       dest="get_ancestors", default=False,
			       help="retrieve ancestors of IDs and include in graph")
        self.parser.add_option("-d", "--with-descendants", action="store_true",
			       dest="get_descendants", default=False,
			       help="retrieve descendants of IDs and include in graph")
        self.parser.add_option("--verbose", "-v", action="store_true", dest="verbose",
			       default=False, help="list nodes being retrieved")

        (options, args) = self.parser.parse_args()
        
        if len(args) == 0:
            self.parser.error("no record IDs given")

        self.get_ancestors = options.get_ancestors
        self.get_descendants = options.get_descendants
        self.verbose = options.verbose
        self.write_filename = options.filename
        self.leaf_ids = [int(arg) for arg in args]
        
    def build_graph(self):
        """
        Populate the graph member by grabbing the mathematician
        pages and extracting relevant data.
        """
        leaf_grab_queue = list(self.leaf_ids)
        ancestor_grab_queue = []
        descendant_grab_queue = []

        # Grab "leaf" nodes.
        while len(leaf_grab_queue) != 0:
            id = leaf_grab_queue.pop(0)
            if not self.graph.has_node(id):
                # Then this information has not yet been grabbed.
                grabber = Grabber(id)
                if self.verbose:
                    print "Grabbing record #{}".format(id)
                [name, institution, year, advisors, descendants] = grabber.extract_node_information()
                self.graph.add_node(name, institution, year, id, advisors, descendants, True)
                if self.get_ancestors:
                    ancestor_grab_queue += advisors
                if self.get_descendants:
                    descendant_grab_queue += descendants

        # Grab ancestors of leaf nodes.
        if self.get_ancestors:
            while len(ancestor_grab_queue) != 0:
                id = ancestor_grab_queue.pop(0)
                if not self.graph.has_node(id):
                    # Then this information has not yet been grabbed.
                    grabber = Grabber(id)
                    if self.verbose:
                        print "Grabbing record #{}".format(id)
                    [name, institution, year, advisors, descendants] = grabber.extract_node_information()
                    self.graph.add_node(name, institution, year, id, advisors, descendants)
                    ancestor_grab_queue += advisors
                        
        # Grab descendants of leaf nodes.
        if self.get_descendants:
            while len(descendant_grab_queue) != 0:
                id = descendant_grab_queue.pop(0)
                if not self.graph.has_node(id):
                    # Then this information has not yet been grabbed.
                    grabber = Grabber(id)
                    if self.verbose:
                        print "Grabbing record #{}".format(id)
                    [name, institution, year, advisors, descendants] = grabber.extract_node_information()
                    self.graph.add_node(name, institution, year, id, advisors, descendants)
                    descendant_grab_queue += descendants
                    
    def generate_dot_file(self):
        dotfile = self.graph.generate_dot_file(self.get_ancestors, self.get_descendants)
        dotfile = dotfile.encode('utf-8', 'replace')
        if self.write_filename is not None:
            outfile = open(self.write_filename, "w")
            outfile.write(dotfile)
            outfile.close()
        else:
            print dotfile,
        
def ggrapher():
    """Function to run the Geneagrapher. This is the function called when
    the ggrapher script is run."""
    ggrapher = Geneagrapher()
    ggrapher.parse_input()
    ggrapher.build_graph()
    ggrapher.generate_dot_file()

{'fraction_non_alphanumeric': 0.09523350987000481, 'fraction_numerical': 0.0016369764082811748, 'mean_word_length': 2.0457478005865104, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '747777', 'n_tokens_mistral': 2963, 'n_tokens_neox': 2748, 'n_words': 722}