<jupyter_start><jupyter_text><jupyter_code>#Method - April, modified - William
from functools import reduce
import numpy as np

import operator
import pandas as pd

book_by_page = [["Mary", "Lamb", "Had", "A", "A"],["A","Little", "Lamb"],["Joseph","Had","A","Baby"]]

#remove duplicate in a page
book_by_page = list(map(lambda x: set(x), book_by_page))

#see tuples in testing eg:[(0, {'A', 'Had', 'Mary', 'Lamb'})], this ste is to assign page number to each word on each page {i:[(x[0]+1)]} => {'Had': [1]}
map_list = list(map(lambda x: list({i:[(x[0]+1)]} for i in x[1]), enumerate(book_by_page)))
print("map_list", map_list)
print("--------")

#map_list = list(map(lambda x: {i:[(x[0]+1)] for i in x[1]} , enumerate(book_by_page)))

flat_map_list = reduce(operator.concat, map_list)
print("flat_map_list", flat_map_list)
print("--------")

flat_map_list

def merge_dict(x,y):
    if list(y.keys())[0] in list(x.keys()):
        x[list(y.keys())[0]].append(list(y.values())[0][0])
    else:
        x.update(y)
    return x

reduce(merge_dict,flat_map_list)
# Method1 - William
from functools import reduce 
book_by_page = [['Mary', 'Lamb', 'Had', 'A', "A"],['Little', 'Lamb'],["Joseph","Had","A","Baby"]]

#get unique words 
words = reduce(lambda x, y: x+y, book_by_page)
unique_words = list(set(words))
print(f'unique words in the book are: \n{unique_words}')

def map_fn(data):
    idx, el = data
    dict = {}
    dict[idx+1]=el
    return dict
  
book = list(map(map_fn, enumerate(book_by_page)))
# [{1: ['A', 'Had', 'Mary', 'Lamb']}, {2: ['Lamb', 'Little']}]
print(book)

# create result dictionary
result = {}
for x in unique_words:
  result[x] = []
# print(result)

def search_words(word):
  # count is used as key of each item in the book list
  page_num = 1
  for item in book: 
    if word in item[page_num]:
      #if word occurs in the page then append page number to result list of the word
      result[word].append(page_num)
      page_num += 1
    else:
      page_num += 1

     
list(map(search_words, unique_words))

print(f"index of each word is: \n{result}")
#testing
a = {"mary":[1]}
b=[1,2,2,3]
a["bob"]=b
print(a)

print(list(a.keys())[0])

#testing
b.index(2)
#get unique values in a list
set(b)

def check_keys(dict): 
    if "mary" not in dict.keys():
        print(1)
    else:
        print(2)
    
check_keys(a)  

#testing - get index in map using emulate
my_list = [2,3,5,7]

def map_fn(data):
    idx, el = data
    dict = {}
    dict[el]=[idx+1]
    return dict

list(map(map_fn, enumerate(my_list)))

# tuples
tuples = list(enumerate(book_by_page))
print("tuples", tuples)

print("-----")

print(tuples[0][0], "and", tuples[0][1])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19125277983691624, 'fraction_numerical': 0.014455151964418088, 'mean_word_length': 4.2407766990291265, 'pattern_counts': {'":': 1, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '374618', 'license': 'MIT', 'n_tokens_mistral': 1076, 'n_tokens_neox': 1044, 'n_words': 272, 'path': 'MapReduce.ipynb', 'repo_name': 'Barneybean/Text_Indexing', 'size': '2698'}