<jupyter_start><jupyter_text>Scripted Primo Search Assessment Tool Description:   Load required libraries<jupyter_code>import pandas as pd
import os
import time
from urllib.request import Request, urlopen
from urllib.parse import urlencode, quote_plus
import io
os.chdir('/Volumes/jwa_drive1/git/spst/ScriptedPrimoSearchAssessmentTool/dev')
from spst import SPST,CompResults
test = True
type(test)
test = str(test)
type(test)<jupyter_output><empty_output><jupyter_text>Define Some Variables<jupyter_code>## files with search strings
f = '/Volumes/jwa_drive1/git/spst/search_strings.csv'
#f = '/Volumes/jwa_drive1/git/spst/cr_records.xml'
p = SPST()
s = SPST()
# search strings are shared by each instance of the class
p.set_search_strings(f,'title_author')
# set parameters sets the number of results per search, the 'onCampus' variable, and the 'institution' variable
# these variables are shared by each instance of the class
p.set_params('2','true','BOSU')

## the platform and scope are assigned by the set_platform_scope() function which requires a dict as the parameter.
## the platform, scope, and/or url_base can be assigned per instance. If the url_base is not passed, it is adjusted
## based on the default value for the platform
p_dict = {'platform':'production',
          'scope':'&loc=local,scope:(ALMA_BOSU1)&loc=adaptor,primo_central_multiple_fe&onCampus=true&json=true'}
s_dict  = {'platform':'stage',
           'scope':'&loc=local,scope:(ALMA_BOSU1)&loc=adaptor,primo_central_multiple_fe&onCampus=true&json=true',
          'url_base':'http://bu-primostage.hosted.exlibrisgroup.com:1701/PrimoWebServices/xservice/search/brief'}
s.set_platform_scope(s_dict)
p.set_platform_scope(p_dict)
search1 = p.data(start=0,end=10)
search1 = search1.sort_values(by=['Search_No','ResultNumber'], ascending=[1,1])
search2 = s.data(start=0,end=10)
search2 = search2.sort_values(by=['Search_No','ResultNumber'], ascending=[1,1])
comp = CompResults()
r_df = comp.Compare2ResultSets(search1,search2,columns = ['title','creator','recordid','fulltext'])
r_df.head(10)


# Define the aggregation calculations
aggregations = {
    'match' : {
        'match' : 'count'
    },
    'sType' : {
        'stype' : 'count'
    },
    'pType' : {
        'pType' : 'count'
        
    },

    'sToplevel' : {
        'pToplevel' : 'count'
        
    },
    'pToplevel' : {
        'pToplevel': 'count'
        
    }
}
search1.columns
#cr_df = cr_df[['title', 'collection', 'creator', 'type','fulltext','delcategory','doi','OpenURL']]
cr_df.columns
alma_id = ''
counter = 0
doi_cnt = 0
for i,r in cr_df.iterrows():
    length = len(i[0]) -1
    #print(i)
    #print('Match: ', i[0][:-1].lower() == r['title'][:length].lower())
    #print(i[0][:-1].lower())
    #print(r['title'][:length].lower())
    #print(r['collection'])
    if 'ALMA_BOSU' in i[1]:
        alma_id = i[1]
        author = r['creator']
        title = r['title'].lower()
        counter += 1
    if (alma_id > '' and 'ALMA_BOSU' not in i[1]):
        if i[0][:-1].lower() == r['title'][:length].lower():
            doi_cnt += 1
            print(str(doi_cnt),' of ',str(counter))
            print(alma_id)
            print(author)
            print(title)
            print('Search: ',i[0][:-1].lower())
            print('Title:  ',r['title'][:length].lower())
            print('Creator:  ',r['creator'][:length].lower())
            print('Collection: ',r['collection'])
            print('Fulltext: ',r['fulltext'])
            try:
                if r['collection'] != 'PsycARTICLES (American Psychological Association)':
                    print(r['doi'].replace('.org/','.org.ezproxy.bu.edu/').replace('https','http'))
                print(r['OpenURL'])
            except Exception as e:
                print(r['OpenURL'])
        
            print()
d = {}
c = 0
for k,group in by_search:
    print()
    print(k)
    print(group.columns)
    group = group[['pFulltext', 'pToplevel','pType','sType']]
    c+= 1
    print(group.groupby('pType').count())<jupyter_output><empty_output><jupyter_text>Code below has been migrated to a class<jupyter_code># build_url 
    # takes base_url and search_string, returns url properly formated for searching
def build_url(base_url,search_string):
    url = base_url+query_Params1 + search_string.replace(' ','+') + query_Params2 + query_Params3
    #print(url)
    return(url)
    
# parse_response 
    # takes response (as json) from search, returns response_dict (dict of our responses)
def parse_response(search,response,platform,search_no):
    response_dict = {}
    # define the elements to grab from the json response. These become the columns in the dataframe
    elements = []
    #elements.append('Platform')
    #elements.append('Search')
    #elements.append('ResultNumber')
    #elements.append('TotalHits')
    #elements.append('Rank')
    #elements.append('SearchEngine')
    elements.append('control.recordid')
    elements.append('display.type')
    elements.append('display.creator')
    elements.append('display.title')
    elements.append('display.ispartof')
    elements.append('delivery.delcategory')
    elements.append('delivery.fulltext')
    elements.append('search.searchscope')
    elements.append('search.general')
    elements.append('facets.collection')
    elements.append('facets.frbrtype')
    elements.append('facets.toplevel')
    elements.append('facets.prefilter')
    elements.append('sort')
    # parse the response to return the data desired elements
    response = json.loads(response.decode('utf8'))
    # need to extract the desired elements and add them to a dict
    response = response['SEGMENTS']
    response = response['JAGROOT']
    response = response['RESULT']
    response_dict['facet'] = response['FACETLIST']
    response_dict['docset'] = response['DOCSET']
    docs = response_dict['docset']['DOC']
    facets = response_dict['facet']
        
    ## define a dataframe to return the results
    doc_columns = ['Platform','Search_No','Search','ResultNumber','TotalHits','FirstHit','LastHit','Rank','SearchEngine','recordid', \
           'type','creator','title','ispartof','delcategory','fulltext','searchscope','general','collection', \
          'frbrtype','toplevel','prefilter','sort']
    df_doc = pd.DataFrame(columns = doc_columns)
    fac_columns = ['Platform','Search_No','Search','TotalHits','SearchEngine','facet','values']
    df_facets = pd.DataFrame(columns = fac_columns)
    #bib = doc['PrimoNMBib']['record']
    #print(type(bib)
    ## iterate through the results (normally 10) to populate a dictionary that will be added to the dataframe
    for doc in docs:
        doc_dict = {}
        doc_dict['Platform'] = platform
        doc_dict['Search_No'] = search_no
        doc_dict['Search'] = search
        doc_dict['ResultNumber'] = doc['@NO']
        doc_dict['TotalHits'] = response['DOCSET']['@TOTALHITS']
        doc_dict['FirstHit'] = response['DOCSET']['@FIRSTHIT']
        doc_dict['LastHit'] = response['DOCSET']['@LASTHIT']
        doc_dict['Rank'] = doc['@RANK']
        doc_dict['SearchEngine'] = doc['@SEARCH_ENGINE']
        #doc_dict['Bib'] = doc['PrimoNMBib']['record']
        Bib = doc['PrimoNMBib']['record']
        
        for element in elements:
            x = element.split('.')
            if len(x) == 1:
                try:
                    doc_dict[x[-1]] = Bib[x[0]]
                except:
                    pass
            if len(x) == 2:
                try:
                    doc_dict[x[-1]] = Bib[x[0]][x[1]]
                except:
                    pass
            if len(x) == 3:
                try:
                    doc_dict[x[-1]] = Bib[x[0]][x[1]][x[2]]
                except:
                    pass
            if len(x) == 4:
                try:
                    doc_dict[x[-1]] = Bib[x[0]][x[1]][x[2]][x[3]]
                except:
                    pass
            if len(x) == 5:
                try:
                    doc_dict[x[-1]] = Bib[x[0]][x[1]][x[2]][x[3]][x[4]]
                except:
                    pass   
        df_doc = df_doc.append(doc_dict, ignore_index=True)
        #print(doc_dict)
        f_dict = parse_facets(facets['FACET'])
        for k,v in f_dict.items() :
            if doc['@NO'] == '1':
                facet_dict = {}
                facet_dict['Platform'] = platform
                facet_dict['Search_No'] = search_no
                facet_dict['Search'] = search
                #facet_dict['ResultNumber'] = doc['@NO']
                facet_dict['TotalHits'] = response['DOCSET']['@TOTALHITS']
                facet_dict['SearchEngine'] = doc['@SEARCH_ENGINE']
                facet_dict['facet'] = k
                facet_dict['values'] = v
                df_facets = df_facets.append(facet_dict,ignore_index=True)

    return(df_doc,Bib,df_facets)


def parse_facets(facets):
    return_dict = {}
    for facet in facets:
        f = facet['@NAME']
        fv = facet['FACET_VALUES']
        #print(f)
        l = []
        for x in fv :
            try:
                l.append((x['@KEY'],int(x['@VALUE'])))
            except Exception as e:
                pass
        sorted_by_second = sorted(l, key=lambda tup: tup[1],reverse=True)
        return_dict[f] = sorted_by_second
    return return_dict<jupyter_output><empty_output><jupyter_text>create a DataFrame. Expand the columns to match the desired elements to be returned by the function 'parse_response' Create a dataframe with - column for search string  - column for response (returned from parse_response fuction)columns = ['Platform','Search','ResultNumber','TotalHits','FirstHit','LastHit','Rank','SearchEngine','recordid', \           'type','creator','title','ispartof','delcategory','fulltext','searchscope','general','collection', \          'frbrtype','toplevel','prefilter','sort']df = pd.DataFrame(columns = columns)fac_columns = ['Platform','Search_No','Search','TotalHits','SearchEngine','facet','values']df_f = pd.DataFrame(columns = fac_columns) Loop through search_strings, execute search, ... [to be done?]counter = 0p_dict = {'platform':'production','scope':'&loc=local,scope:(ALMA_BOSU1)&loc=local,scope:(BOSU)&loc=adaptor,primo_central_multiple_fe&onCampus=true&json=true'}p = SPST()s = SPST()p.set_search_strings(f,'title_author')p.set_params('2','true','BOSU')for search_string in p.search_strings[:]:    time.sleep(1)    print(counter)    print(search_string)    prod_search_url = p.build_url('prod', search_string) prod_url_base,search_string)    prod_request = Request(prod_search_url)    stage_search_url = s.build_url('stage',search_string) stage_url_base,search_string)    stage_request = Request(stage_search_url)        try:        response_body = urlopen(prod_request).read()  reads request into response_body var        prod = response_body        d = p.parse_response(search_string,response_body,'production',counter)[0]        b = p.parse_response(search_string,response_body,'production',counter)[1]        f = p.parse_response(search_string,response_body,'production',counter)[2]        df = df.append(d, ignore_index=True)        df_f = df_f.append(f,ignore_index=True)        response_body = urlopen(stage_request).read()  reads request into response_body var        stage = response_body        d = s.parse_response(search_string,response_body,'stage',counter)[0]        b = s.parse_response(search_string,response_body,'stage',counter)[1]        f = s.parse_response(search_string,response_body,'stage',counter)[2]        df = df.append(d, ignore_index=True)        df_f = df_f.append(f,ignore_index=True)    prints errors     except Exception as e:        pass        print('Exception: ',e)    counter += 1<jupyter_code>## here we are going to combine values from the search on production and stage to compare the results for each search
columns = ['Search','match','pRecordid','sRecordid','pRank','sRank','pCollection','sCollection', 
                        'pCreator','sCreator', 'pDelcategory','sDelcategory','pFrbrtype','sFrbrtype', 
                        'pFulltext','sFulltext', 'pTitle','sTitle', 'pToplevel','sToplevel', 'pType','sType']
#comp_results = pd.DataFrame(columns = columns)
mask = search1['Platform'] == 'production'
production = search1[mask] 
mask = search1['Platform'] == 'stage'
stage = search1[mask]
stage= stage[['Rank', 'ResultNumber', 'Search', 'SearchEngine',
       'Search_No', 'TotalHits', 'collection', 'creator', 'delcategory',
       'frbrtype', 'fulltext', 'general', 'ispartof', 'prefilter', 'recordid',
       'searchscope', 'sort', 'title', 'toplevel', 'type']]
production = production[['Rank', 'ResultNumber', 'Search', 'SearchEngine',
       'Search_No', 'TotalHits', 'collection', 'creator', 'delcategory',
       'frbrtype', 'fulltext', 'general', 'ispartof', 'prefilter', 'recordid',
       'searchscope', 'sort', 'title', 'toplevel', 'type','doi','OpenURL']]
stage= stage.set_index(['Search_No', 'ResultNumber'])
#cr_df = production[['pRecordid', 'ResultNumber','Search', 'pTitle', 'pCollection',  'pType','pFulltext','pDelcategory']]
cr_df = production.set_index(['Search', 'recordid','ResultNumber'])
production = production.set_index(['Search_No', 'ResultNumber'])
production = production[['recordid','Rank', 'Search','collection', 'creator','delcategory', 'frbrtype', 'fulltext','title', 'toplevel','type']]
stage = stage[['recordid','Rank', 'Search','collection', 'creator','delcategory', 'frbrtype', 'fulltext','title', 'toplevel','type']]
production.columns = ['pRecordid','pRank', 'Search', 'pCollection', 'pCreator', 'pDelcategory', 'pFrbrtype',
       'pFulltext', 'pTitle', 'pToplevel', 'pType']
stage.columns = ['sRecordid','sRank', 'Search', 'sCollection', 'sCreator', 'sDelcategory', 'sFrbrtype',
       'sFulltext', 'sTitle', 'sToplevel', 'sType']

if (stage.shape[1] == production.shape[1]):
    rows = set(range(0,min(stage.shape[0],production.shape[0])))
    cols = set(range(0,stage.shape[1]))
    scolumns = stage.columns
    pcolumns = production.columns
    for row in rows:
        d = {}
        for col in cols:
            if scolumns[col] == 'sRecordid':
                if stage.iloc[row][scolumns[col]] == production.iloc[row][pcolumns[col]]:
                    d['match'] = True
                else:
                    d['match'] = False
            d[scolumns[col]] = stage.iloc[row][scolumns[col]]
            d[pcolumns[col]] = production.iloc[row][pcolumns[col]]
        comp_results = comp_results.append(d,ignore_index=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15938744677928857, 'fraction_numerical': 0.007279219887378108, 'mean_word_length': 3.458971218616044, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '414208', 'license': 'MIT', 'n_tokens_mistral': 4702, 'n_tokens_neox': 4556, 'n_words': 865, 'path': 'ScriptedPrimoSearchAssessmentTool.ipynb', 'repo_name': 'bulib/ScriptedPrimoSearchAssessmentTool', 'size': '14562'}