<jupyter_start><jupyter_text>Sensitivy Analysis Data Creation<jupyter_code>cursor = FBW.aql.execute('FOR doc IN Businesses RETURN doc', ttl=5000)
for document in tqdm(cursor, total=cursor.count()):
    if (("HTML_R1_NOV20" in document) or ("HTML_R2_NOV20" in document)):
        if(document["HTML_R1_NOV20"] is not None):
            document["Clean_Text_SA"] = clean_pop_cookie_frame(document["HTML_R1_NOV20"])
        elif(document["HTML_R2_NOV20"] is not None):
            document["Clean_Text_SA"] = clean_pop_cookie_frame(document["HTML_R2_NOV20"])
        BusinessCollection.update(document) 
#Sensitivity Parameters

processing_id = "SA16"
stem_bool = False
remove_sequence_length = 2
stopwords_bool = False
minimum_document_length = 20
cursor = FBW.aql.execute('FOR doc IN Businesses RETURN doc', ttl=5000)
for document in tqdm(cursor, total=cursor.count()):
    if (("Clean_Text_SA" in document) and ("Language" in document)):
        document[processing_id] = lower_punct_number_clean(document["Clean_Text_SA"], remove_sequence_length)
        if(stopwords_bool): 
            temp_text = remove_stopwords(document[processing_id], document["Language"])
        else:
            temp_text = document[processing_id]
        if((temp_text is not None) and (count_words(temp_text) >= minimum_document_length)):
            if(stem_bool):
                document[processing_id] = stem_text(temp_text, document["Language"])
            else:
                document[processing_id] = temp_text
        BusinessCollection.update(document) 
BusinessCollection["421138663"]["SA01"]
BusinessCollection["421138663"]["SA12"]
BusinessCollection["421138663"]["SA16"]<jupyter_output><empty_output><jupyter_text>Transformer Dataset Raw Output<jupyter_code>def database_to_transformers(revision):
    text = []
    innovation = []
    cursor = FBW.aql.execute('FOR doc IN Businesses RETURN doc', ttl=5000)
    for document in cursor:
        if((revision in document) and ("Innovation" in document)):
            text.append(document[revision])
            innovation.append(document["Innovation"])
    return pd.DataFrame({'text': text,'innovation': innovation})

transformer_dataset = database_to_transformers('Clean_Text_SA')
transformer_dataset.to_csv("transformer_dataset_raw.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11505190311418685, 'fraction_numerical': 0.029411764705882353, 'mean_word_length': 4.768079800498753, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '222453', 'license': 'MIT', 'n_tokens_mistral': 765, 'n_tokens_neox': 719, 'n_words': 132, 'path': 'Notebooks/Website Text Process.ipynb', 'repo_name': 'nusretipek/Flanders_Innovation', 'size': '2312'}