<jupyter_start><jupyter_text>Function that reads the export directories, at the export base directory<jupyter_code>def read_exports_directories(export_directories, exports_basedir):
    exports = []
    
    for directory in exports_directories:
        export_dirpath = os.path.join(exports_basedir, directory)
        
        # Lê os exports do diretório
        export = fm.read_csv_directory(dirpath=export_dirpath, header=1)
        
        exports.append(export)
        
    return exports<jupyter_output><empty_output><jupyter_text>Function that do the pre-processing for each of the exports passed<jupyter_code># Realiza o pré-processamento e armazenamento dos exports
def pre_processing(exports):
    clean_exports = []
    
    for export in exports:
        # Realiza o pré-processamento do conteúdo dos exports
        clean_export = ppc.clean_reviews(export)
        
        clean_exports.append(clean_export)
        
    return clean_exports<jupyter_output><empty_output><jupyter_text>Function that saves the clean exports in CSV files, at the clean exports directory<jupyter_code># Realiza o pré-processamento e armazenamento dos exports
def save_clean_exports(exports, export_directories, clean_exports_basedir):
    # Colunas dos exports a serem consideradas
    columns = ["Content", "CleanContent", "Source", "Rating"]
    # Novos nomes de colunas para os exports pré-processados
    header = ["content", "clean_content", "source", "rating"]
    
    fm.delete_directory(clean_exports_basedir)
    
    for directory, export in zip(exports_directories, exports):
        filename = directory + ".csv"

        # Salva os exports já pré-processados em um arquivo .csv
        fm.data_to_csv(data=export,\
                       dirpath=clean_exports_basedir,\
                       filename=filename,\
                       columns=columns,\
                       header=header)<jupyter_output><empty_output><jupyter_text>Function that reads the clean reviews CSV files in the clean exports directories<jupyter_code># Lê os diretórios com os exports pré-processados
def read_clean_exports_directories(export_directories, clean_exports_basedir):
    exports = []
    
    for directory in export_directories:
        export_filename = directory + ".csv"

        export = fm.read_csv_file(dirpath=clean_exports_basedir, filename=export_filename)
        
        exports.append(export)
        
    return exports<jupyter_output><empty_output><jupyter_text>Does the K-medoids clusteringAlso has the function to turn dataframe tuples into a list of dicts, with the passed columns<jupyter_code># Função que transforma uma tupla do dataframe em um dict()
# A função considera os atributos: Content, CleanContent, Source e Rating
def dataframe_to_dicts(dataframe):
    tuples = dataframe.itertuples() # Traz as linhas do dataframe em formato de tupla
    columns = dataframe.keys() # Recupera o nome das columas do dataframe de exports
    indexes = range(1, len(columns) + 1) # Cria um vetor com os indices a serem considerados
    
    dicts = []
    
    for tuple_t in tuples:
        enumeration = None
    
        if not indexes:
            enumeration = enumerate(columns)
        else:
            enumeration = zip(indexes, columns)

        d = dict()

        for index, column in enumeration:
            d[column] = tuple_t[index]
            
        dicts.append(d)
        
    return dicts

def clustering_kmedoids(export, distance_calculator, k_min=3, k_max=15, iteractions=20, n=20):
    # Dict que armazenará as reviews como documentos a serem clusterizados
    reviews = dict()
    reviews["reviews"] = dataframe_to_dicts(export)
    
    result = dict()
    result["k"] = []        
    result["sse"] = []
    result["iteraction"] = []
    result["clusters"] = []
    result["medoids"] = []
    result["most_similar"] = []

    for k in range(k_min, k_max + 1):
        min_sse = float("inf")
        min_itr = None        
        min_clusters = None
        min_medoids = None
        min_most_similar = None

        for itr in range(1, iteractions + 1):
            # Simulação de passagem de parâmetro po cópia.
            # Necessário pois é preciso uma cópia para cada instância do kmedoids, para evitar inconsistências.
            documents = copy.deepcopy(reviews["reviews"])

            kmedoids_instance = kmedoids.Kmedoids(k=k,\
                                                  documents=documents,\
                                                  distance_calculator=distance_calculator,\
                                                  collection_field="reviews", text_field_name="clean_content")

            kmedoids_instance.clustering()                
            
            sse = kmedoids_instance.calculate_sse()

            if sse < min_sse:
                min_itr = itr
                min_sse = sse
                min_clusters = copy.deepcopy(kmedoids_instance.get_clusters())
                min_medoids = copy.deepcopy(kmedoids_instance.get_medoids())
                min_most_similar = copy.deepcopy(kmedoids_instance.n_most_similar_for_clusters_medoid(n))                

        result["k"].append(k)
        result["sse"].append(min_sse)
        result["iteraction"].append(min_itr)
        result["clusters"].append(min_clusters)
        result["medoids"].append(min_medoids)
        result["most_similar"].append(min_most_similar)
        
    return result<jupyter_output><empty_output><jupyter_text>Saves the clustering results in the results directorySaves overall results, clusters generated, most similar reviews and word clouds of each cluster, for all the K values<jupyter_code>def save_clustering_result(result, directory, results_basedir, collection_field="reviews", text_field_name="clean_content", overall_results_filename="clustering_overall_results.csv"):
    result_directory = os.path.join(results_basedir, directory)

    fm.delete_directory(result_directory)

    overall_results_columns = ["k", "iteraction", "sse"]

    # Save overall clustering results
    fm.overall_results_to_csv(results=result,\
                             dirpath=result_directory,\
                             filename=overall_results_filename,\
                             columns=overall_results_columns)

    clusters_results_columns = ["content", "clean_content", "cluster", "rating"]

    # Save cluster contents
    fm.clusters_to_csv(result=result,\
                       dirpath=result_directory,\
                       collection_field="reviews",\
                       columns=clusters_results_columns)

    # Save the clusters' most similar documents
    fm.most_similar_to_csv(result=result, dirpath=result_directory)
    
    fm.medoids_to_csv(result=result, dirpath=result_directory)
def plot_graphs(overall_results, results_basedir, directory, k_min=3, k_max=20):
    results_dirpath = os.path.join(results_basedir, directory)
    
    gp.plot_overall_clustering_results(overall_results, results_dirpath)
    
    clusters = fm.clusters_dataframe_from_csv(results_dirpath)

    for k in range(k_min, k_max + 1):
        k_subdir = os.path.join(results_dirpath, 'clustering', 'K_' + str(k))
        
        itr_subdirs = os.listdir(k_subdir)
        
        for itr_subdir in itr_subdirs:
            graphs_dirpath = os.path.join(k_subdir, itr_subdir, "graphs")
            
            gp.plot_clusters_mean_ratings(clusters, k=k, dirpath=graphs_dirpath, xoffset=1) 
            gp.plot_clusters_document_count(clusters, k=k, dirpath=graphs_dirpath, xoffset=1)
            gp.plot_clusters_document_percentage(clusters, k=k, dirpath=graphs_dirpath, xoffset=1)
            
            ratings_frequencies_dirpath = os.path.join(graphs_dirpath, "mean_ratings")
            
            gp.plot_clusters_rating_frequencies(clusters, k, ratings_frequencies_dirpath)<jupyter_output><empty_output><jupyter_text>Application variables (mostly workspace directories)Also defines the exports directories considered in the application execution<jupyter_code># Diretórios base
data_basedir = os.path.join(os.getcwd(), "data") # Diretório geral de dados

exports_basedir = os.path.join(data_basedir, "exports") # Subdiretório dos exports

clean_exports_basedir = os.path.join(data_basedir, "clean_exports") # Subdiretório os exports pré-processados
results_jaccard_basedir = os.path.join(data_basedir, "results", "jaccard") # Subdiretório dos resultados
results_levenshtein_basedir = os.path.join(data_basedir, "results", "levenshtein") # Subdiretório dos resultados

# Lista dos nomes dos diretórios que contém os exports
exports_directories = ["facebook", "telegram"]<jupyter_output><empty_output><jupyter_text>Reads the exports directories, and pre-process the reviews, and saves them in the results directory for each export directory defined<jupyter_code>exports = read_exports_directories(exports_directories, exports_basedir)
clean_exports = pre_processing(exports)
save_clean_exports(clean_exports, exports_directories, clean_exports_basedir)<jupyter_output><empty_output><jupyter_text>Runs the K-medoids clustering algorithm using the Jaccard distance calculator<jupyter_code>print("K-medoids with Jaccard distance started!")
clean_exports = read_clean_exports_directories(exports_directories, clean_exports_basedir)

for directory, export in zip(exports_directories, clean_exports):
    print("Directory: ", directory)
    result = clustering_kmedoids(export, distance.JaccardCalculatorDistance(), k_min=3, k_max=80, iteractions=40, n=100)
    
    save_clustering_result(result, directory, results_jaccard_basedir)<jupyter_output><empty_output><jupyter_text>Runs the K-medoids clustering algorithm using the Levenshtein distance calculator<jupyter_code>print("K-medoids with Levenshtein distance started!")
clean_exports = read_clean_exports_directories(exports_directories, clean_exports_basedir)

for directory, export in zip(exports_directories, clean_exports):
    print("Directory: ", directory)
    result = clustering_kmedoids(export, distance.LevenshteinCalculator(), k_min=3, k_max=40, iteractions=20, n=100)
    
    save_clustering_result(result, directory, results_levenshtein_basedir)<jupyter_output><empty_output><jupyter_text>Plots the graphs with clusters statistics for the best K values<jupyter_code>for distance_name, results_basedir in zip(["Jaccard", "Levenshtein"], [results_jaccard_basedir, results_levenshtein_basedir]):
    for directory in exports_directories:
        result_dirpath = os.path.join(results_basedir, directory)
        
        overall_results = fm.overall_results_from_csv(result_dirpath)
        minimal_difference = gp.calculate_minimal_difference(overall_results, interval=5, max_err_increase=7.5)
        
        difference = minimal_difference["difference"]
        results = minimal_difference["results"]
        
        print("\n", distance_name, "results")
        print("Minimal difference for", directory, "is", difference, "\n")
        
        if difference != float("inf"):
            k_min = min(results["k"].astype(int))
            k_max = max(results["k"].astype(int))
            
            min_sse = results['sse'].min()
            min_k = results.loc[results['sse'] == min_sse]['k'].astype(int).tolist()[0]
                
            print("The best K value is", min_k, ", with a SSE of", min_sse)

            plot_graphs(overall_results, results_basedir, directory, k_min=min_k, k_max=min_k)
        else:
            print("No interval satisfied the max error condition.")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08529005524861878, 'fraction_numerical': 0.003453038674033149, 'mean_word_length': 3.358540255831452, 'pattern_counts': {'":': 0, '<': 46, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '608149', 'license': 'MIT', 'n_tokens_mistral': 3532, 'n_tokens_neox': 3399, 'n_words': 740, 'path': 'main.ipynb', 'repo_name': 'CaioMelo8/android-comments-miner', 'size': '11584'}