<commit_before>import logging
import os
from src.boolean_query_parser import BooleanQueryParser
from src.boolean_ir import BooleanIR
from src.boolean_ir import Literal
from src.term import Term
from src.indexterm_postings_reader import IndextermPostingsReader

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    
    #===========================================================================
    # logging.info(BooleanQueryParser().parse_query("Hexe"))
    # logging.info('')
    # logging.info(BooleanQueryParser().parse_query("NOT Hexe AND NOT Prinzessin AND HalloWelt"))
    # logging.info('')
    # logging.info(BooleanQueryParser().parse_query("Hexe OR Prinzessin OR (A AND NOT B AND NOT 78) OR NOT test OR (NOT A AND NOT B)"))
    # logging.info('')
    # logging.info(BooleanQueryParser().parse_query("(Hexe AND Prinzessin) OR (Frosch AND Koenig AND Tellerlein)"))
    # logging.info('')
    # logging.info(BooleanQueryParser().parse_query("(Hexe AND Prinzessin) OR (NOT Hexe AND Koenig)"))
    # logging.info('')
    # logging.info(Tokenizer().tok_lowercase('data/myfile.txt', ' |\t|\n|\.|,|;|:|!|\?|"|-'))
    # logging.info('')
    # logging.info(BooleanIR().intersect([4,5,6,9,10], [3,4,5,6,11,12]))
    # logging.info('')
    # logging.info(BooleanIR().union([4,5,6,9,10], [3,4,5,6,11,12]))
    # logging.info('')
    # logging.info(BooleanIR().complement([4,5,6,9,10], [3,4,5,6,9,10,11,12]))
    # logging.info('')
    # logging.info(BooleanIR().intersect_complement([4,5,6,9,10], [3,4,5,6,11,12]))   
    # 
    # # Hexe OR Prinzessin OR (A AND NOT B AND NOT 78) OR NOT test OR (NOT A AND NOT B)
    # # and_clause = [([1,5,6], True), ([6,8,9, True), [([2,3], True), ([7]], False), ([6,10], False)], ([1,9], False), [([3,4,6], False), ([8,9], False)]]
    # # UNIVERSE = {1,2,3,4,5}
    # universe = list(range(1,6))
    # # {1,2} AND {1,2,3,4} AND NOT {2} = {1}
    # and_clause = [Literal([1,2],True), Literal([1,2,3,4],True), Literal([2],False)]
    # print(BooleanIR().intersect_literals(and_clause, universe))
    # # {1} AND {1,2} AND NOT {1,2,3} = {}
    # and_clause = [Literal([1],True), Literal([1,2],True), Literal([1,2,3],False)]
    # print(BooleanIR().intersect_literals(and_clause, universe))
    # # NOT {} AND {1,2,4} AND {1,2,3,5} = {1,2}
    # and_clause = [Literal([],False), Literal([1,2,4],True), Literal([1,2,3,5],True)]
    # print(BooleanIR().intersect_literals(and_clause, universe))
    # 
    # # {1,2} OR {4,5} OR {1,4} = {1,2,4,5}
    # or_clause = [Literal([1,2],True), Literal([4,5],True), Literal([1,4],True)]
    # print(BooleanIR().union_literals(or_clause, universe))
    # # {} OR {1,2,3} OR NOT {} = UNIVERSE
    # or_clause = [Literal([],True), Literal([1,2,3],True), Literal([],False)]
    # print(BooleanIR().union_literals(or_clause, universe))
    # # NOT {1,2,3,4,5} OR NOT {1,2,3} OR {1} = {1,4,5}
    # or_clause = [Literal([1,2,3,4,5],False), Literal([1,2,3],False), Literal([1],True)]
    # print(BooleanIR().union_literals(or_clause, universe))
    #===========================================================================
    
    
    
#===============================================================================
#     dictionary = {} # matches a Term with an occurrence mylist
#     docsDict = {}   # matches DocID and DocName
# #    dictionary = defaultdict(SingleList)    # this does not do what I want!
#     docID = 0
#     # Reading Files
#     # This works even if subfolders are used
#     print("Start:")
#     data_folder = os.getcwd() + "/data/"
#     for root, dirs, files in os.walk(data_folder):
#         for file in files:
#             if file.endswith(".txt"):    # Is there anything else?
#                 if docID not in docsDict:
#                     docsDict[docID] = file
#                 terms = Tokenizer().tok_lowercase(os.path.join(root, file), ' |\t|\n|\.|,|;|:|!|\?|"|-')
#                 for t in terms:
#                     if t not in dictionary:
#                         dictionary[t] = TermPostings()
#                     dictionary[t].postings.append(Posting(docID))
# #                   dictionary[Term(t)].append(docID)    # class Term would need to be immutable
#                 docID += 1
#===============================================================================
    
    data_folder = os.path.join(os.getcwd(), "data")
    dictionary, docsDict = IndextermPostingsReader().read_from_folder(data_folder)
     
    print("number of dict entries:", len(dictionary))
    print(docsDict)
    print(dictionary)
#    alaList = dictionary["und"]
#    print(alaList)
#    print(dictionary["und"].len)
    print("Done.")
    
    # currently the classes 'term' and 'posting' only contain a String and an int respectively but as per the task the can now be extended
    
    #===========================================================================
    # from singly_linked_list import SingleList
    # mylist = SingleList()
    # for i in range(1, 20):
    #     mylist.append(i)
    # print(mylist)
    # for ele in mylist:
    #     print(ele)
    #===========================================================================
    
    
    
    
    mode = "bool"
    print("Boolean logic activated.")
    while True: # user input loop
        print("Please enter a query or command:")
        query = input().strip()
        if len(query) < 1:
            continue    # ask for input again
        else:
            if query.startswith("/"):    # execute COMMAND
                if query == "/bool":<commit_msg>removed comments<commit_after>import logging
import os
from src.boolean_query_parser import BooleanQueryParser
from src.boolean_ir import BooleanIR
from src.boolean_ir import Literal
from src.term import Term
from src.indexterm_postings_reader import IndextermPostingsReader

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    
    #===========================================================================
    # logging.info(BooleanQueryParser().parse_query("Hexe"))
    # logging.info('')
    # logging.info(BooleanQueryParser().parse_query("NOT Hexe AND NOT Prinzessin AND HalloWelt"))
    # logging.info('')
    # logging.info(BooleanQueryParser().parse_query("Hexe OR Prinzessin OR (A AND NOT B AND NOT 78) OR NOT test OR (NOT A AND NOT B)"))
    # logging.info('')
    # logging.info(BooleanQueryParser().parse_query("(Hexe AND Prinzessin) OR (Frosch AND Koenig AND Tellerlein)"))
    # logging.info('')
    # logging.info(BooleanQueryParser().parse_query("(Hexe AND Prinzessin) OR (NOT Hexe AND Koenig)"))
    # logging.info('')
    # logging.info(Tokenizer().tok_lowercase('data/myfile.txt', ' |\t|\n|\.|,|;|:|!|\?|"|-'))
    # logging.info('')
    # logging.info(BooleanIR().intersect([4,5,6,9,10], [3,4,5,6,11,12]))
    # logging.info('')
    # logging.info(BooleanIR().union([4,5,6,9,10], [3,4,5,6,11,12]))
    # logging.info('')
    # logging.info(BooleanIR().complement([4,5,6,9,10], [3,4,5,6,9,10,11,12]))
    # logging.info('')
    # logging.info(BooleanIR().intersect_complement([4,5,6,9,10], [3,4,5,6,11,12]))   
    # 
    # # Hexe OR Prinzessin OR (A AND NOT B AND NOT 78) OR NOT test OR (NOT A AND NOT B)
    # # and_clause = [([1,5,6], True), ([6,8,9, True), [([2,3], True), ([7]], False), ([6,10], False)], ([1,9], False), [([3,4,6], False), ([8,9], False)]]
    # # UNIVERSE = {1,2,3,4,5}
    # universe = list(range(1,6))
    # # {1,2} AND {1,2,3,4} AND NOT {2} = {1}
    # and_clause = [Literal([1,2],True), Literal([1,2,3,4],True), Literal([2],False)]
    # print(BooleanIR().intersect_literals(and_clause, universe))
    # # {1} AND {1,2} AND NOT {1,2,3} = {}
    # and_clause = [Literal([1],True), Literal([1,2],True), Literal([1,2,3],False)]
    # print(BooleanIR().intersect_literals(and_clause, universe))
    # # NOT {} AND {1,2,4} AND {1,2,3,5} = {1,2}
    # and_clause = [Literal([],False), Literal([1,2,4],True), Literal([1,2,3,5],True)]
    # print(BooleanIR().intersect_literals(and_clause, universe))
    # 
    # # {1,2} OR {4,5} OR {1,4} = {1,2,4,5}
    # or_clause = [Literal([1,2],True), Literal([4,5],True), Literal([1,4],True)]
    # print(BooleanIR().union_literals(or_clause, universe))
    # # {} OR {1,2,3} OR NOT {} = UNIVERSE
    # or_clause = [Literal([],True), Literal([1,2,3],True), Literal([],False)]
    # print(BooleanIR().union_literals(or_clause, universe))
    # # NOT {1,2,3,4,5} OR NOT {1,2,3} OR {1} = {1,4,5}
    # or_clause = [Literal([1,2,3,4,5],False), Literal([1,2,3],False), Literal([1],True)]
    # print(BooleanIR().union_literals(or_clause, universe))
    #===========================================================================
    
    
    data_folder = os.path.join(os.getcwd(), "data")
    dictionary, docsDict = IndextermPostingsReader().read_from_folder(data_folder)
     
    print("number of dict entries:", len(dictionary))
    print(docsDict)
    print(dictionary)
#    alaList = dictionary["und"]
#    print(alaList)
#    print(dictionary["und"].len)
    print("Done.")
    
    # currently the classes 'term' and 'posting' only contain a String and an int respectively but as per the task the can now be extended
    
    mode = "bool"
    print("Boolean logic activated.")
    while True: # user input loop
        print("Please enter a query or command:")
        query = input().strip()
        if len(query) < 1:
            continue    # ask for input again
        else:
            if query.startswith("/"):    # execute COMMAND
                if query == "/bool":
{'fraction_non_alphanumeric': 0.2389232219545407, 'fraction_numerical': 0.03739394574211794, 'mean_word_length': 3.668948655256724, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '25733', 'n_tokens_mistral': 3373, 'n_tokens_neox': 3243, 'n_words': 670}