<jupyter_start><jupyter_text>Index<jupyter_code>import numpy as np
import datetime as dt

wd = 'downloads'
def cleanhtml(raw_html):
    cleanr = re.compile('<.*?>')
    cleantext = re.sub(cleanr, '', raw_html)
    return cleantext
def cleanAutogenSubtitles(subtitles):
    line = [item for item in subtitles[2].split('\n') if item is not ' ']
    try:
        if(len(line[0].split(' --> ')[1]) > 12):
            cleaned = []
            for x in subtitles: 
                line = [item for item in x.split('\n') if item is not ' ']
                line[0] = line[0][:29]
                if(len(line) == 2 and len(cleanhtml(line[1])) == len(line[1])):
                    cleaned.append(line)
            subtitles = cleaned
    except:
        return subtitles
    
    return subtitles
# accepts data as dictionary
def pushCaption(data, user, pwd):
    location = 'captions'
    url = 'http://127.0.0.1:8000/'+location+'/'
    r = requests.post(url, json=data, auth=(user, pwd))
    try:
        return r.json()
    except: 
        print(r)
        return r<jupyter_output><empty_output><jupyter_text>Gather Data<jupyter_code>next(os.walk('.'))[1]

subdir_ls = []
for subdir in os.walk(wd):
    if len(subdir[2]) > 0:
        for file in subdir[2]:
            subdir_ls += [(subdir[0], file)]
#part 0: Create dict of files

"""
Data Structure:
{
    
        title: [string],
        captions: [txt],
        description: [file path],
        thumbnail: [file path],
        uploader: [string],
        id: [string]

}
""" 

file_content_dict = {}
for i, file_snippet in enumerate(subdir_ls):
    print(str(i+1)+' of '+str(len(subdir_ls)), end='\r')
    
    file = file_snippet[1]
    if file.split('.')[-1] == 'vtt' or file.split('.')[-1] == 'json':
        file_label = "'".join(file.split('.')[0:-2])
    else:
        file_label = "'".join(file.split('.')[0:-1])
    
    try: 
        file_content_dict[file_label]
    except:
        file_content_dict[file_label] = {}
    
    # captions
    if file[-4:] == '.vtt':
        vtt0 = open(wd+"/"+file, "r", encoding='utf-8').read()
        vtt1 = cleanAutogenSubtitles(vtt0.split('\n\n'))
        if vtt1 != ['']:
            vtt2 = []
            for x in vtt1:
                if(len(x)>0):
                    try:
                        tmp = x.split('\n')
                        timepoint = tmp[0].split(' --> ')
                        timepoint = timepoint[0] + '\t' + timepoint[1]   #convert to string
#                         label = file + '\t' + timepoint
                        label = timepoint
                        vtt2.append((label, ' '.join(tmp[1:]).lower()))
                    except:
                        continue

            vtt1 = vtt2
        file_content_dict[file_label]['captions'] = vtt1
        
    # thumbnail
    elif file[-4:] == '.jpg':
        file_content_dict[file_label]['thumbnail'] = file_snippet[0] + '/' + file
    
    # description
    elif file[-12:] == '.description':
        file_content_dict[file_label]['description'] = file_snippet[0] + '/' + file
    
    # additional info
    elif file[-10:] == '.info.json':
        contents = json.loads(open(wd+"/"+file, "r", encoding='utf-8').read())
        file_content_dict[file_label]['uploader'] = contents['uploader']
        file_content_dict[file_label]['id'] = contents['id']
        
for item in file_content_dict:
    try:
        data = {
            'title': item,
            'uploader': file_content_dict[item]['uploader'],
            'videoid': file_content_dict[item]['id'],
            'thumbnail': file_content_dict[item]['thumbnail'],
            'description': file_content_dict[item]['description'],
            'captions': json.dumps(file_content_dict[item]['captions'])
        }
        pushCaption(data, user='[ enter username here ]', pwd='[ enter password here ]')
#         break
    except Exception as e:
        print(e)
%pylab inline
import matplotlib.pyplot as plt
import matplotlib.image as mpimg

for item in file_content_dict:
    print(file_content_dict[item]['thumbnail'])
    img=mpimg.imread(file_content_dict[item]['thumbnail'])
    imgplot = plt.imshow(img)
    plt.show()
    break<jupyter_output>Populating the interactive namespace from numpy and matplotlib
downloads/'Empty Bottles' by <NAME> _ MIT 21M.342 Composing for Jazz Orchestra, Fall 2008-Yp67PhU7Jt0.jpg<jupyter_text>Create a new index<jupyter_code>schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT)
if not os.path.exists("indexdir"):
    os.mkdir("indexdir")
ix = create_in("indexdir", schema)
writer = ix.writer()
length = len(file_content_dict)
for i, file_label in enumerate(file_content_dict):
    print(f'{i+1} of {length}', end='\r')
    captions = ''
    try:
        for caption in file_content_dict[file_label]['captions']:
            captions += caption[1] + ' '
    except:
        continue
    
    writer.add_document(title=file_label, content=captions)

writer.commit()<jupyter_output>5813 of 5813<jupyter_text>Open an existing index<jupyter_code>import whoosh.index as index

ix = index.open_dir("indexdir")<jupyter_output><empty_output><jupyter_text>Run Search<jupyter_code>from whoosh.qparser import QueryParser
def toseconds(timepoint):
    tmp = timepoint.split('.')[0].split(':')
    hours = int(tmp[0])
    minutes = int(tmp[1])
    seconds = int(tmp[2])
    return str(hours*360 + minutes*60 + seconds)

def searchCorpus(q):
    with ix.searcher() as searcher:
        query = QueryParser('content', ix.schema).parse(q)
        results = searcher.search(query, limit=10)
        results.fragmenter.charlimit = None
        # Show more context before and after
        results.fragmenter.surround = 400
        for hit in results:
            img=mpimg.imread(file_content_dict[hit['title']]['thumbnail'])
            imgplot = plt.imshow(img)
            plt.show()
            print(hit['title'][:-12] + '\n')

            filecontents = ''
            for line in file_content_dict[hit['title']]['captions']:
                filecontents += '\n' + line[0] + '\t' + line[1]
            
            snippets = hit.highlights('content', text=filecontents, top=5).split('...')
            snippets = [cleanhtml(x) for x in snippets]
            for snippet in snippets:
                surrounding_lines = snippet.split('\n')[1:-1]
                for i, line in enumerate(surrounding_lines):
                    tmp = line.split('\t')
                    print('\t' + tmp[3])
                    
                    if(i == 0):
                        video_id = tmp[0][-18:-7]
                        time_start = toseconds(tmp[1])
                
                try:
                    if(len(surrounding_lines) > 0):
                        print('\thttps://app.chimeraeditor.com/player?privacy=private&v=' + video_id + '&t=' + time_start + '\n')
                except:
                    continue
                
            
        return results
query = input('Search: ')
searchCorpus(query)<jupyter_output>Search: biology
{'fraction_non_alphanumeric': 0.147413182140326, 'fraction_numerical': 0.016017009213323885, 'mean_word_length': 2.230769230769231, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 26, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '573430', 'license': 'Apache-2.0', 'n_tokens_mistral': 2334, 'n_tokens_neox': 2255, 'n_words': 453, 'path': 'index/.ipynb_checkpoints/index-checkpoint.ipynb', 'repo_name': 'joshuafinkelstein/chimera-education', 'size': '7057'}