<jupyter_start><jupyter_text>Prosodic parsing  Constants<jupyter_code>STONES = ['parse_by_line', 'parse_by_slice', 'parse_by_ngram', 'parse_by_phrase', 'parse_by_nsyll']
eg_path1='/Volumes/Present/DH/corpora/chadwyck_poetry/txt/african-american/beadlesa/Z200265018.txt'
eg_path1='/Volumes/Present/DH/corpora/chadwyck_poetry/txt/english/shakespe/Z300482603.txt'

eg_path1='/Volumes/Present/DH/corpora/chadwyck_poetry/txt/english/miscell2/Z200439011.txt'
eg_path2='/Users/ryan/llp/corpora/dottxt/txt/Unknown.King James Bible.Genesis.txt'
eg_path1='/Users/ryan/DH/Prose-Verse/experiments/lsa_abstract/data_ryan/dibble-O.txt'

eg_path3='/Volumes/Present/DH/corpora/chadwyck_poetry/txt/faber/fa0101/Z200557409.txt'
MAXLINES=5000<jupyter_output><empty_output><jupyter_text>Imports<jupyter_code>import prosodic as p
p.config['print_to_screen']=False<jupyter_output><empty_output><jupyter_text>Functions<jupyter_code>def read_file(fnfn):
    try:
        if fnfn.endswith('.gz'):
            import gzip
            with gzip.open(fnfn,'rb') as f:
                return f.read().decode('utf-8')
        else:
            with open(fnfn) as f:
                return f.read()
    except IOError as e:
        print("!! error:",e, end=' ')
        print("!! opening:",fnfn)
        print()
        return ''
#print(read_file(eg_path1))
def counts(string, sub):
    count = start = 0
    while True:
        start = string.find(sub, start) + 1
        if start > 0:
            count+=1
        else:
            return count<jupyter_output><empty_output><jupyter_text>Parsing functions  Raw string parsing function<jupyter_code>def get_data_from_line(line,meter,require_parse_data=True):
    """
    Get data from the prosodic line object, with its meter.
    """
    
    # get phonological info
    weight_str=line.str_weight()
    sonority_str=line.str_sonority()
    stress_str=line.str_stress()
    stress_str_wordbounds=line.str_stress(word_sep="#")
    #print(stress_str)

    # store metrical constraint stats
    bp=line.bestParse(meter)
    
    # require 
    if require_parse_data:
        if not bp:
            return {}
        
    ap=line.allParses(meter)
    output_dict={}
    output_dict['prosodic_line']=line.txt #.encode('utf-8',errors='ignore')
    output_dict['parse']=bp.posString(viols=True) if bp else ''
    #output_dict['parse']=output_dict['parse'] #.encode('utf-8',errors='ignore')
    meter_str=output_dict['meter']=bp.str_meter() if bp else ''
    output_dict['num_parses']=len(ap)
    output_dict['num_viols'] = bp.totalCount if bp else ''
    output_dict['score_viols'] = bp.score() if bp else ''
    output_dict['num_sylls']=bp.num_sylls if bp else ''
    output_dict['num_words']=len(line.words())
    for c in meter.constraints:
        sumviol = sum([parse.constraintCounts[c] if c in parse.constraintCounts else 0 for parse in ap])
        output_dict['mviol_'+c.name.replace('.','_').replace('=>-','_is_')+'_bestparse']=bp.constraintCounts[c] if bp and c in bp.constraintCounts else 0
        output_dict['mviol_'+c.name.replace('.','_').replace('=>-','_is_')+'_allparse_sum']=sumviol if sumviol else 0
    
    ## store phonological constraint stats
    output_dict['prosodic_stress']=stress_str
    output_dict['prosodic_weight']=weight_str
    output_dict['prosodic_sonority']=sonority_str
    output_dict['num_monosylls']=len([w for w in line.words() if w.numSyll==1])
    output_dict['viol_clash_across']=counts(stress_str_wordbounds,'P#P') + counts(stress_str_wordbounds,'P#S') + counts(stress_str_wordbounds,'S#P') + counts(stress_str_wordbounds,'S#S')
    output_dict['viol_clash_within]']=counts(stress_str_wordbounds,'PP') + counts(stress_str_wordbounds,'PS') + counts(stress_str_wordbounds,'SP') + counts(stress_str_wordbounds,'SS')
    output_dict['viol_clash_across_primary']=counts(stress_str_wordbounds,'P#P')
    output_dict['viol_clash_within_primary']=counts(stress_str_wordbounds,'PP')
    output_dict['viol_lapse_across']=counts(stress_str_wordbounds,'U#U')
    output_dict['viol_lapse_within']=counts(stress_str_wordbounds,'UU')
    output_dict['viol_clash']=counts(stress_str,'PP') + counts(stress_str,'PS') + counts(stress_str,'SP') + counts(stress_str,'SS')
    output_dict['viol_lapse']=counts(stress_str,'UU')
    output_dict['viol_WSP']=0
    output_dict['viol_PEAKPROM']=0
    output_dict['viol_High_Stress']=0
    output_dict['viol_Low_Unstress']=0
    output_dict['viol_High_Strong']=0
    output_dict['viol_Low_Weak']=0
    for s,w,hml,mtr in zip(stress_str,weight_str,sonority_str,meter_str):
        if s=='U' and w=='H':
                output_dict['viol_WSP']+=1
        if (s=='P' or s=='S') and w=='L':
                output_dict['viol_PEAKPROM']+=1
        if hml=='H' and s in {'P','S'}:
                output_dict['viol_High_Stress']+=1
        if hml=='L' and s=="U":
                output_dict['viol_Low_Unstress']+=1
        if hml=='H' and mtr == 's':
                output_dict['viol_High_Strong']+=1
        if hml=='L' and mtr == 'w':
                output_dict['viol_Low_Weak']+=1
    return output_dict
def parse_string(text_str_or_list, meter='default_english', num_processes=1, maxlines=MAXLINES):
    """
    Parse the string, assuming line as unit
    """
    # prosodic parse
    
    if maxlines:
        import random
        lines = text_str_or_list if type(text_str_or_list) in {list,tuple} else text_str_or_list.split('\n')
        random.shuffle(lines)
        text_str='\n'.join(lines[:maxlines])
    else:
        text_str = '\n'.join(text_str_or_list) if type(text_str_or_list) in {list,tuple} else text_str_or_list
    
    text = p.Text(text_str)
    meter = text.get_meter(meter)

    out_ld=[]
    for i,line in enumerate(text.iparse(meter=meter, num_processes=num_processes)):
        if not i%100: print(i,line)
        line_d=get_data_from_line(line,meter)
        if not line_d or not 'score_viols' in line_d: continue
        line_d['line_id']=i+1
        out_ld.append(line_d)
    return out_ld<jupyter_output><empty_output><jupyter_text>Testing<jupyter_code># import pandas as pd
# pd.DataFrame(parse_string("""With what attractive charms this goodly frame 
# Of nature touches the consenting hearts 
# Of mortal men; and what the pleasing stores 
# Which beauteous imitation thence derives 
# To deck the poet's, or the painter's toil; 
# My verse unfolds.
# Rock bog sweat steam liberty"""))[['prosodic_line','viol_clash','viol_clash_across','viol_lapse','viol_lapse_across']]<jupyter_output><empty_output><jupyter_text>By line in text<jupyter_code>def parse_by_line(path_to_txt_or_xml_file, meter='default_english', num_processes=1):
    # get txt
    txt=read_file(path_to_txt_or_xml_file)
    
    # return parse
    return parse_string(txt, meter=meter, num_processes=num_processes)<jupyter_output><empty_output><jupyter_text>Testing<jupyter_code># import pandas as pd
# df_line = pd.DataFrame(parse_by_line(eg_path1)).sort_values('score_viols')
# df_line.head()
# import pandas as pd
# df_line1=pd.DataFrame(parse_by_line(eg_path1))
# df_line2=pd.DataFrame(parse_by_line(eg_path2))
# #print(df_line1.score_viols.mean(), df_line2.score_viols.mean())
# df_line1.sort_values('num_viols').head()<jupyter_output><empty_output><jupyter_text>By window<jupyter_code>def slice(l,num_slices=None,slice_length=None,runts=True,random=False):
    """
    Returns a new list of n evenly-sized segments of the original list
    """
    if random:
        import random
        random.shuffle(l)
    if not num_slices and not slice_length: return l
    if not slice_length: slice_length=int(len(l)/num_slices)
    newlist=[l[i:i+slice_length] for i in range(0, len(l), slice_length)]
    if runts: return newlist
    return [lx for lx in newlist if len(lx)==slice_length]

def ngram(l,n=3):
    grams=[]
    gram=[]
    for x in l:
        gram.append(x)
        if len(gram)<n: continue
        g=tuple(gram)
        grams.append(g)
        gram.reverse()
        gram.pop()
        gram.reverse()
    return grams
# Test slicing
#slice(read_file(eg_path2).split(), slice_length=5, runts=False)
# num_ngrams=len(ngram(read_file(eg_path2).split(), n=5))
# num_slices=len(slice(read_file(eg_path2).split(), slice_length=5, runts=False))

# num_ngrams, num_slices
def zeroPunc(s):
    import string
    return s.translate(str.maketrans('', '', string.punctuation))
def parse_by_window(path_to_txt_or_xml_file, meter='default_english', window_size=5,
                    overlapping_windows=False,max_slices=100000,num_processes=1,
                   phrase_splitter='[?.,;:]'):
    # start slices
    word_slices=[]
    
    # get txt
    fulltxt=read_file(path_to_txt_or_xml_file)
    
    import re
    txts= re.split(phrase_splitter, fulltxt) if phrase_splitter else [fulltxt]
    
    for txt in txts:
        # remove punc
        txt=zeroPunc(txt)

        # tokenize
        words=txt.split()
        
        # remove nums?
        words=[w for w in words if not any(c.isdigit() for c in w)]

        # windows?
        if overlapping_windows:
            word_slices += ngram(words,n=window_size)
        else:
            word_slices += slice(words,slice_length=window_size)
    
    if max_slices:
        import random
        random.shuffle(word_slices)
        word_slices = word_slices[:max_slices]

    # rejoin
    #txt = '\n'.join([' '.join(slicex) for slicex in word_slices])
    print('found slices:',len(word_slices))
    lines = [' '.join(slicex) for slicex in word_slices]
    return parse_string(lines,num_processes=num_processes)
def parse_by_ngram(path_to_txt_or_xml_file, n=5, meter='default_english', num_processes=1):
    return parse_by_window(path_to_txt_or_xml_file, window_size=n, meter=meter, overlapping_windows=True, num_processes=num_processes)
parse_by_slice = parse_by_window<jupyter_output><empty_output><jupyter_text>Testing<jupyter_code># import pandas as pd
# df_window1_slice=pd.DataFrame(parse_by_window(eg_path1))
# # df_window2_slice=pd.DataFrame(parse_by_window(eg_path2))
# # df_window1_ngram=pd.DataFrame(parse_by_window(eg_path1,overlapping_windows=True))
# # df_window2_ngram=pd.DataFrame(parse_by_window(eg_path2,overlapping_windows=True))
# # print(df_window1_slice.mean()['score_viols'], df_window1_slice.mean()['score_viols'])
# # print(df_window1_ngram.mean()['score_viols'], df_window2_ngram.mean()['score_viols'])
# df_window1_slice.sort_values('score_viols')<jupyter_output><empty_output><jupyter_text>By phrase<jupyter_code># Requires NLTK
def parse_by_phrase(path_to_txt_or_xml_file, meter='default_english', minword=5):
    # get txt
    txt=read_file(path_to_txt_or_xml_file)
    
    # phrases
    import re
    phrases=re.split('[?.,;:\n]', txt)
    
    # recombine for minword
    if minword:
        phrases2=[]
        phrase=[]
        for px in phrases:
            phrase+=px.split()
            if len(phrase)>=minword:
                phrases2+=[' '.join(phrase)]
                phrase=[]
        phrases=phrases2
    
    # make txt
    txt = '\n'.join(phrases)

    # return parsed
    return parse_string(txt)<jupyter_output><empty_output><jupyter_text>Testing<jupyter_code># import pandas as pd
# df_window1_phrase=pd.DataFrame(parse_by_phrase(eg_path1))
# df_window2_phrase=pd.DataFrame(parse_by_phrase(eg_path2))
# print(df_window1_phrase.mean()['score_viols'], df_window2_phrase.mean()['score_viols'])
# df_window1_phrase.sort_values('score_viols',ascending=False).head()<jupyter_output><empty_output><jupyter_text>N-Syllable Project<jupyter_code>def loopsum(l,sumval=10,overlapping=True,overlapping_offset=1):
    stack=[]
    for i,x in enumerate(l):
        stack.append((i,x))
        stack_sum=sum([y for x,y in stack])
        #print(stack_sum,stack)
        if stack_sum < sumval: continue
        
        while sum([y for x,y in stack]) > sumval:
            gone=stack.pop(0)
            _sum=sum([y for x,y in stack])
            #print('-',gone,_sum,stack)
            #print(_sum,stack,'--')
            

        #print('>>',stack_sum,stack)
        stack_sum=sum([y for x,y in stack])
        if stack_sum == sumval:
            _sum=sum([y for x,y in stack])
            #print(_sum,stack,'>>')
            yield stack
            stack=stack[overlapping_offset:] if overlapping else []
            
    if stack and sum([y for x,y in stack])==sumval:
        yield stack
            
def tokenize_fast(line):
    import re
    return re.findall("[A-Z]{2,}(?![a-z])|[A-Z][a-z]+(?=[A-Z])|[\'\w\-]+",line.lower())
import random
# def n_syll_project(path_to_txt_or_xml_file, n_syll=10,meter='default_english',
#                    within_phrases=False,within_lines=False, maxlines=MAXLINES,
#                    overlapping=False,overlapping_offset=2,shuffle_lines=True,
#                    can_end_on_unstressed=False,can_end_on_maybe_stressed=False,
#                    return_postprocessed=False):
    
def n_syll_project(path_to_txt_or_xml_file, n_syll=10,meter='default_english',
                   within_phrases=True,within_lines=False, maxlines=MAXLINES,
                   overlapping=True,overlapping_offset=2,shuffle_lines=True,
                   can_end_on_unstressed=True,can_end_on_maybe_stressed=True,
                   return_postprocessed=False):
    # get txt
    txt=read_file(path_to_txt_or_xml_file)
    
    # words
    all_tokens=tokenize_fast(txt)
    all_types=set(all_tokens)
    
    # dictionary
    pDict = p.dict['en']
    
    # getting the numsyll
    word2numsyll=dict((w,pDict.get(w)[0].getNumSyll()) for w in all_types)
    
    # phrases
    import re
    if within_lines:
        phrase_splitter='[?.,;:\n]' if within_phrases else '[\n]'
    else:
        phrase_splitter='[?.,;:]' if within_phrases else ''
    
    if phrase_splitter:
        phrases=re.split(phrase_splitter, txt)
    else:
        phrases = [txt]
        
    if not can_end_on_unstressed: unstressed = set(p.dict['en'].unstressedWords)
    if not can_end_on_maybe_stressed: maybestressed = set(p.dict['en'].unstressedWords)
        
    
    # loop over phrases to produce the lines
    LINES = []
    for phrase in phrases:
        phrase_words=tokenize_fast(phrase)
        phrase_nsylls=[word2numsyll.get(w,0) for w in phrase_words]
        
        for stack in loopsum(phrase_nsylls,sumval=n_syll,overlapping=overlapping,overlapping_offset=overlapping_offset):
            stack_words=[phrase_words[i] for i,x in stack]
            if not can_end_on_unstressed and stack_words[-1] in unstressed: continue
            if not can_end_on_maybe_stressed and stack_words[-1] in maybestressed: continue
            line = ' '.join(stack_words)
            #print(line)
            LINES.append(line)
    
    # make txt
    
    print('>> # lines:',len(LINES),'in',path_to_txt_or_xml_file)
    if shuffle_lines: random.shuffle(LINES)
    LINES=LINES[:MAXLINES]
    
    #print('>> # lines:',len(LINES))
    txt = '\n'.join(LINES)
    #print(txt)

    # return parsed
    data=parse_string(txt,meter=meter)
    if return_postprocessed:
        data=postprocess_avg((path_to_txt_or_xml_file,data))
        #print(data)
    return data
    

parse_by_nsyll = n_syll_project
# import pandas as pd
# df_window1_nsyllproj=pd.DataFrame(n_syll_project(eg_path1,return_postprocessed=False))
# df_window2_nsyllproj=pd.DataFrame(n_syll_project(eg_path2,return_postprocessed=False))
# df_window1_nsyllproj_overlap=pd.DataFrame(n_syll_project(eg_path1,return_postprocessed=False,overlapping=True))
# df_window2_nsyllproj_overlap=pd.DataFrame(n_syll_project(eg_path2,return_postprocessed=False,overlapping=True))
# df_window1_nsyllproj.score_viols.mean(), df_window2_nsyllproj.score_viols.mean()
# #print(df_line1.score_viols.mean(),df_line2.score_viols.mean())
# print(df_window1_ngram.score_viols.mean(),df_window2_ngram.score_viols.mean())
# print(df_window1_slice.score_viols.mean(),df_window2_slice.score_viols.mean(), len(df_window1_slice), len(df_window2_slice))
# #print(df_window1_phrase.score_viols.mean(),df_window2_phrase.score_viols.mean())
# print(df_window1_nsyllproj.score_viols.mean(),df_window2_nsyllproj.score_viols.mean())
# print(df_window1_nsyllproj_overlap.score_viols.mean(),df_window2_nsyllproj_overlap.score_viols.mean())
#print(read_file(eg_path1))
#n_syll_project(eg_path3)<jupyter_output><empty_output><jupyter_text>Postprocess<jupyter_code>def postprocess_avg(individual_slingshot_result, split_bys=['/xml/','/txt/','/txt_tcp/','/_txt_chadwyck/','/_txt_ecco_tcp/','/_txt_sellars/']):
    # imports
    import os, pandas as pd

    # Split into two components
    path,data=individual_slingshot_result

    # make pandas dataframe
    df=pd.DataFrame(data)
    #df2=pd.DataFrame(index=df.index)

    # normalize
    for col in df.columns:
        try:
            df[col+'_per_10_sylls'] = df[col] / df['num_sylls'] * 10
        except (TypeError,ValueError) as e:
            pass

    # summarize
    d=dict(df.mean())

    # outof
    d['num_lines_parsed']=len(data)

    # add id
    #split_fn_by='/txt/'
    idx=os.path.splitext(path)[0]
    corpus=None
    for split_fn_by in split_bys:
        if split_fn_by in idx:
            idxdat=idx.split(split_fn_by)
            corpus,idx=os.path.basename(idxdat[0]),idxdat[-1]

    d['id'] = idx
    d['corpus'] = corpus
    d['path']=path

    # return dict
    return d
def postprocess_jsonl(jsonl_fn):
    import mpi_slingshot as sl,os
    out=[]
    for path,data in sl.stream_jsonl(jsonl_fn):
        data_d=postprocess_avg((path,data))
        #print(data_d)
        out+=[data_d]
    return out
def postprocess(cache_folder,ofn='data.metrical_parsing.postprocessed.txt',sep='\t'):
    import mpi_slingshot as sl,os

    def _writegen():
        for path,data in sl.stream_results(cache_folder):
            data_d=postprocess_avg((path,data))
            #print(data_d)
            yield data_d

    sl.writegen(ofn, _writegen)
    print('>> saved:',ofnfn)

#n_syll_project(eg_path1,return_postprocessed=True)
#postprocess('/Users/ryan/DH/Prose-Verse/experiments/language_article/small_corpus/data_slingshot/new_parse_by_nsyll/cache')
# eg_path='/Users/ryan/Dropbox/PHD/Prose-Verse/experiments/language_article/small_corpus/txt/browne-O.txt'
# parse_by_nsyll(eg_path)
# import mpi_slingshot as sl
# ??sl.writegen<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12890902144845276, 'fraction_numerical': 0.009987447470392404, 'mean_word_length': 3.935092916778885, 'pattern_counts': {'":': 1, '<': 55, '<?xml version=': 0, '>': 68, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '535752', 'license': 'MIT', 'n_tokens_mistral': 6939, 'n_tokens_neox': 6658, 'n_words': 963, 'path': 'slings/metrical_parsing.ipynb', 'repo_name': 'quadrismegistus/mpi-slingshot', 'size': '18323'}