<commit_before>import os
from banner_entity import *
import csv
import codecs

# #Non-faculty academic
# 28101 --> ['Uv Sr Rsch Scientist FT', 'Senior Research Scientist', 'Senior Research Scientist FT', 'Mc Sr Rsch Scientist FT', 'Sr Rsch Scientist', 'Uv Sr Rsch Scientist Ft', ' Senior Research Scientist']
# 28301 --> ['Uv Rsch Scientist FT', 'Mc Rsch Scientist FT', 'Research Scientist', 'Uv Research Scientist Ft', 'Mc Rsh Scient Ft', 'Uv Rsh Scient Ft', 'Mc Research Scientist Ft', 'Sr. Research Scientist']
# 28302 --> ['Mc Rsch Scientist PT', 'Research Scientist', 'Uv Rsch Scientist PT']
# 28502 --> ['Uv Rsch Assoc PT', 'Research Associate PT', 'Uv Rsch Assoc', 'Research Associate', 'Uv Rsh Assoc Pt', 'Mc Rsch Assoc PT', 'Postdoctoral Scientist']
# 283R2 --> ['Mc Lead Research Scientist Pt', 'Uv Lead Research Scientist PT', 'Uv Lead Research Scientist FT']
# 283R1 --> ['Lead Research Scientist', 'Deputy Director CHCQ', ' Lead Research Scientist', 'Uv Lead Research Scientist FT', 'Sr Research Network Engineer', 'Mc Lead Research Scientist Ft']
# 28102 --> ['Mc Sr Rsch Scientist FT', 'Sr. Research Scientist']
# 19S01 --> ['Research Project Director', 'Programs&Development Director', 'Director of Research']
# 28501 --> ['Uv Rsh Assoc Ft', 'Uv Rsch Assoc FT', 'Policy Associate', 'Research Asso. Study Coodntr', 'Research Associate', 'Bioinformatics Analyst', 'Mc Rsch Assoc FT', 'Rsch Assoc', 'Mc Rsh Assoc Ft', 'Uv Rsch Assoc FT/SAS Pgrmr']
# 27401 --> ['Uv Rsch Dir Non-Fa Ft']
#
# #Postdoc
# 289A1 --> ['Uv Post-Doctoral Scientist Ft', ' Post-Doctoral Scientist', 'Post-Doctoral Scientist', 'Post Doctoral Scientist', 'Research Scientist', 'Mc Post-Doctoral Scientist Ft', 'Uv Post-Doctoral Scientist PT']
# 289A2 --> ['Uv Post-Doctoral Scientist Pt']
#
# #Librarian
# 307A1 --> ['Uv Librarian 4 FT', 'Uv Lib 4 Ft']
# 30601 --> ['Lib 1', 'Uv Librarian 1 FT', 'Uv Librarian II FT', 'Mc Lib 1 Ft']
# 30602 --> ['Uv Lib 1 Ft', 'Systems Librarian']
# 30402 --> ['Uv Lib 3 Pt', 'Uv Librarian 3 PT', 'Uv Librarian IV FT']
# 30401 --> ['Uv Lib 3 Ft', 'Mc Lib 3 Ft', 'Uv Librarian 4FT', 'Uv Lib 4 Ft', 'Uv Lib 2 Ft', 'Uv Librarian 3 FT']
# 01001 --> ['UV Librarian & Vice Provost']
# 30501 --> ['Uv Lib 3 Ft', 'Uv Librarian 2 FT', 'Uv Librarian 3 FT', 'Mc Lib 2 Ft', 'Uv Lib 2 Ft', 'Uv Lib  Ft', 'Systems Librarian', 'Uv Lib 4 Ft', 'Librarian 2']

pos_code_to_classes = {
    "28101": "NonFacultyAcademic",
    "28301": "NonFacultyAcademic",
    "28302": "NonFacultyAcademic",
    "28502": "NonFacultyAcademic",
    "283R2": "NonFacultyAcademic",
    "283R1": "NonFacultyAcademic",
    "28102": "NonFacultyAcademic",
    "19S01": "NonFacultyAcademic",
    "28501": "NonFacultyAcademic",
    "27401": "NonFacultyAcademic",
    "289A1": "Postdoc",
    "289A2": "Postdoc",
    "307A1": "Librarian",
    "30601": "Librarian",
    "30602": "Librarian",
    "30402": "Librarian",
    "30401": "Librarian",
    "01001": "Librarian",
    "30501": "Librarian",
}


def get_non_faculty_gwids(data_dir):
    gwids = []
    with codecs.open(os.path.join(data_dir, "vivo_emplappt.txt"), 'r', encoding="utf-8") as csv_file:
        reader = csv.reader(csv_file)
        for row in reader:
            if row[2] in pos_code_to_classes:
                gwids.append(row[0])
    return gwids


def get_faculty_gwids(data_dir):
    gwids = set()
    with codecs.open(os.path.join(data_dir, "vivo_acadappt.txt"), 'r', encoding="utf-8") as csv_file:
        reader = csv.reader(csv_file)
        for row in reader:
            gwids.add(row[0])
    return gwids


def print_position_code_to_name(data_dir):
    """
    Prints map of position code to position names.
    """
    positions = {}
    with codecs.open(os.path.join(data_dir, "vivo_emplappt.txt"), 'r', encoding="utf-8") as csv_file:
        reader = csv.reader(csv_file)
        for row in reader:
            pos_code = row[2]
            pos_name = row[1]
            if not pos_code in positions:
                positions[pos_code] = []
            if not pos_name in positions[pos_code]:
                positions[pos_code].append(pos_name)
    for pos_code in positions:
        print "%s --> %s" % (pos_code, positions[pos_code])


def load_demographic(data_dir, limit=None):
    #"G37176643","Sabina","M","Alkire","Institute for International Economic Policy","1900 F Street NW",,"Washington","DC",,"20052","<EMAIL>","sabina_alkire","202-994-5320"
    print """<commit_msg>Added intersection of banner demographic entries to limit.<commit_after>import os
from banner_entity import *
import csv
import codecs

# #Non-faculty academic
# 28101 --> ['Uv Sr Rsch Scientist FT', 'Senior Research Scientist', 'Senior Research Scientist FT', 'Mc Sr Rsch Scientist FT', 'Sr Rsch Scientist', 'Uv Sr Rsch Scientist Ft', ' Senior Research Scientist']
# 28301 --> ['Uv Rsch Scientist FT', 'Mc Rsch Scientist FT', 'Research Scientist', 'Uv Research Scientist Ft', 'Mc Rsh Scient Ft', 'Uv Rsh Scient Ft', 'Mc Research Scientist Ft', 'Sr. Research Scientist']
# 28302 --> ['Mc Rsch Scientist PT', 'Research Scientist', 'Uv Rsch Scientist PT']
# 28502 --> ['Uv Rsch Assoc PT', 'Research Associate PT', 'Uv Rsch Assoc', 'Research Associate', 'Uv Rsh Assoc Pt', 'Mc Rsch Assoc PT', 'Postdoctoral Scientist']
# 283R2 --> ['Mc Lead Research Scientist Pt', 'Uv Lead Research Scientist PT', 'Uv Lead Research Scientist FT']
# 283R1 --> ['Lead Research Scientist', 'Deputy Director CHCQ', ' Lead Research Scientist', 'Uv Lead Research Scientist FT', 'Sr Research Network Engineer', 'Mc Lead Research Scientist Ft']
# 28102 --> ['Mc Sr Rsch Scientist FT', 'Sr. Research Scientist']
# 19S01 --> ['Research Project Director', 'Programs&Development Director', 'Director of Research']
# 28501 --> ['Uv Rsh Assoc Ft', 'Uv Rsch Assoc FT', 'Policy Associate', 'Research Asso. Study Coodntr', 'Research Associate', 'Bioinformatics Analyst', 'Mc Rsch Assoc FT', 'Rsch Assoc', 'Mc Rsh Assoc Ft', 'Uv Rsch Assoc FT/SAS Pgrmr']
# 27401 --> ['Uv Rsch Dir Non-Fa Ft']
#
# #Postdoc
# 289A1 --> ['Uv Post-Doctoral Scientist Ft', ' Post-Doctoral Scientist', 'Post-Doctoral Scientist', 'Post Doctoral Scientist', 'Research Scientist', 'Mc Post-Doctoral Scientist Ft', 'Uv Post-Doctoral Scientist PT']
# 289A2 --> ['Uv Post-Doctoral Scientist Pt']
#
# #Librarian
# 307A1 --> ['Uv Librarian 4 FT', 'Uv Lib 4 Ft']
# 30601 --> ['Lib 1', 'Uv Librarian 1 FT', 'Uv Librarian II FT', 'Mc Lib 1 Ft']
# 30602 --> ['Uv Lib 1 Ft', 'Systems Librarian']
# 30402 --> ['Uv Lib 3 Pt', 'Uv Librarian 3 PT', 'Uv Librarian IV FT']
# 30401 --> ['Uv Lib 3 Ft', 'Mc Lib 3 Ft', 'Uv Librarian 4FT', 'Uv Lib 4 Ft', 'Uv Lib 2 Ft', 'Uv Librarian 3 FT']
# 01001 --> ['UV Librarian & Vice Provost']
# 30501 --> ['Uv Lib 3 Ft', 'Uv Librarian 2 FT', 'Uv Librarian 3 FT', 'Mc Lib 2 Ft', 'Uv Lib 2 Ft', 'Uv Lib  Ft', 'Systems Librarian', 'Uv Lib 4 Ft', 'Librarian 2']

pos_code_to_classes = {
    "28101": "NonFacultyAcademic",
    "28301": "NonFacultyAcademic",
    "28302": "NonFacultyAcademic",
    "28502": "NonFacultyAcademic",
    "283R2": "NonFacultyAcademic",
    "283R1": "NonFacultyAcademic",
    "28102": "NonFacultyAcademic",
    "19S01": "NonFacultyAcademic",
    "28501": "NonFacultyAcademic",
    "27401": "NonFacultyAcademic",
    "289A1": "Postdoc",
    "289A2": "Postdoc",
    "307A1": "Librarian",
    "30601": "Librarian",
    "30602": "Librarian",
    "30402": "Librarian",
    "30401": "Librarian",
    "01001": "Librarian",
    "30501": "Librarian",
}


def get_non_faculty_gwids(data_dir):
    gwids = []
    with codecs.open(os.path.join(data_dir, "vivo_emplappt.txt"), 'r', encoding="utf-8") as csv_file:
        reader = csv.reader(csv_file)
        for row in reader:
            if row[2] in pos_code_to_classes:
                gwids.append(row[0])
    return demographic_intersection(gwids, data_dir)


def get_faculty_gwids(data_dir):
    gwids = set()
    with codecs.open(os.path.join(data_dir, "vivo_acadappt.txt"), 'r', encoding="utf-8") as csv_file:
        reader = csv.reader(csv_file)
        for row in reader:
            gwids.add(row[0])
    return demographic_intersection(gwids, data_dir)


def demographic_intersection(gwids, data_dir):
    demo_gwids = set()
    with codecs.open(os.path.join(data_dir, "vivo_demographic.txt"), 'r', encoding="utf-8") as csv_file:
        reader = csv.reader(csv_file)
        for row in reader:
            demo_gwids.add(row[0])
    return demo_gwids.intersection(gwids)


def print_position_code_to_name(data_dir):
    """
    Prints map of position code to position names.
    """
    positions = {}
    with codecs.open(os.path.join(data_dir, "vivo_emplappt.txt"), 'r', encoding="utf-8") as csv_file:
        reader = csv.reader(csv_file)
        for row in reader:
            pos_code = row[2]
            pos_name = row[1]
            if not pos_code in positions:
                positions[pos_code] = []
            if not pos_name in positions[pos_code]:
                positions[pos_code].append(pos_name)
    for pos_code in positions:
        print "%s --> %s" % (pos_code, positions[pos_code])


def load_demographic(data_dir, limit=None):
    #"G37176643","Sabina","M","Alkire","Institute for International Economic Policy","1900 F Street NW",,"Washington","DC",,"20052","<EMAIL>","sabina_alkire","202-994-5320"
    print """
{'fraction_non_alphanumeric': 0.15327604726100966, 'fraction_numerical': 0.050483351235230935, 'mean_word_length': 3.736012207527976, 'pattern_counts': {'":': 38, '<': 5, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6090132', 'n_tokens_mistral': 3750, 'n_tokens_neox': 3291, 'n_words': 988}