<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# dramavis by <NAME> (@umblaetterer) & <NAME> (@chris_kittel)

__author__ = "<NAME> <web at christopherkittel.eu>, <NAME> <ffischer at hse.ru>"
__copyright__ = "Copyright 2017"
__license__ = "MIT"
__version__ = "0.4 (beta)"
__maintainer__ = "<NAME> <ffischer at hse.ru>"
__status__ = "Development" # 'Development', 'Production' or 'Prototype'

import os
import glob
import pandas as pd
import networkx as nx
import csv
from lxml import etree
from itertools import chain, zip_longest
from collections import Counter
import argparse
from superposter import plotGraph, plot_superposter
import logging
import numpy
from tqdm import tqdm

from linacorpus import LinaCorpus, Lina


class CorpusAnalyzer(LinaCorpus):

    def __init__(self, inputfolder, outputfolder, logpath):
        super(CorpusAnalyzer, self).__init__(inputfolder, outputfolder)
        self.logger = logging.getLogger("corpusAnalyzer")
        formatter = logging.Formatter('%(asctime)-15s %(name)s [%(levelname)s]'
                                      '%(message)s')
        fh = logging.FileHandler(logpath)
        fh.setFormatter(formatter)
        self.logger.addHandler(fh)
        self.logpath = logpath


    def analyze_dramas(self, action):
        """
        Reads all XMLs in the inputfolder,
        returns an iterator of lxml.etree-objects created with lxml.etree.parse("dramafile.xml").
        """
        # dramas = {}
        for dramafile in tqdm(self.dramafiles, desc="Dramas", mininterval=0.5):
            # ID, ps = parse_drama(tree, filename)
            # dramas[ID] = ps
            drama = DramaAnalyzer(dramafile, self.outputfolder, self.logpath,
                                  action)
            yield drama

    def get_char_metrics(self):
        self.logger.info("Exporting character metrics.")
        dramas = self.analyze_dramas(action="char_metrics")
        header = [
                    'ID', 'author', 'title', 'year',
                    'frequency', 'degree', 'betweenness', 'closeness'
                 ]
        dfs = []
        for drama in dramas:
            temp_df = pd.DataFrame(index=[drama.ID])
            for m in header[1:3]:
                temp_df[m] = drama.metadata.get(m)
            temp_df['year'] = drama.metadata.get('date_definite')
            for m in header[4:]:
                temp_df[m] = drama.get_top_ranked_chars()[m]
            temp_df['ID'] = drama.ID
            dfs.append(temp_df)
        df = pd.concat(dfs)
        df = df[header]
        df.index = df['ID']
        df.index.name = 'index'
        df.to_csv(os.path.join(self.outputfolder,
                               "central_characters.csv"), sep=";")

    def get_graph_metrics(self):
        self.logger.info("Exporting corpus metrics.")
        dramas = self.analyze_dramas(action="corpus_metrics")
        df = pd.concat([d.graph_metrics for d in dramas])
        header = [
                'ID', 'author', 'title', 'subtitle', 'year', 'genretitle', 'filename',
                'charcount', 'edgecount', 'maxdegree', 'avgdegree',
                'clustering_coefficient', 'clustering_coefficient_random', 'avgpathlength', 'average_path_length_random', 'density',
                'segment_count', 'count_type', 'all_in_index', 'central_character_entry_index', 'change_rate_mean', 'change_rate_std', 'final_scene_size_index',
                'central_character', 'characters_last_in',
                'connected_components'
                ]
        df.index = df["ID"]
        df.index.name = "index"
        df[header].to_csv(os.path.join(self.outputfolder, "corpus_metrics.csv"), sep=";")

    def get_both_metrics(self):
        self.logger.info("Exporting character metrics.")
        dramas = self.analyze_dramas(action="both")
        header = [
                    'ID', 'author', 'title', 'year',
                    'frequency', 'degree', 'betweenness', 'closeness'
                 ]
        char_dfs = []
        graph_dfs = []
        for drama in dramas:
            temp_df = pd.DataFrame(index=[drama.ID])
            for m in header[1:3]:
                temp_df[m] = drama.metadata.get(m)
            temp_df['year'] = drama.metadata.get('date_definite')
            for m in header[4:]:
                temp_df[m] = drama.get_top_ranked_chars()[m]
            temp_df['ID'] = drama.ID
            char_dfs.append(temp_df)
            graph_dfs.append(d.graph_metrics)
        df = pd.concat(char_dfs)
        df = df[header]
        df.index = df['ID']
        df.index.name = 'index'
        df.to_csv(os.path.join(self.outputfolder,
                               "central_characters.csv"), sep=";")
        self.logger.info("Exporting corpus metrics.")
        df = pd.concat(graph_dfs)
        header = [
                'ID', 'author', 'title', 'subtitle', 'year', 'genretitle', 'filename',
                'charcount', 'edgecount', 'maxdegree', 'avgdegree',
                'clustering_coefficient', 'clustering_coefficient_random', 'avgpathlength', 'average_path_length_random', 'density',
                'segment_count', 'count_type', 'all_in_index', 'central_character_entry_index', 'change_rate_mean', 'change_rate_std', 'final_scene_size_index',
                'central_character', 'characters_last_in',
                'connected_components'
                ]
        df.index = df["ID"]
        df.index.name = "index"
        df[header].to_csv(os.path.join(self.outputfolder, "corpus_metrics.csv"), sep=";")


class DramaAnalyzer(Lina):

    def __init__(self, dramafile, outputfolder, logpath,
                 action, randomization=1000):
        super(DramaAnalyzer, self).__init__(dramafile, outputfolder)
        self.logger = logging.getLogger("dramaAnalyzer")
        formatter = logging.Formatter('%(asctime)-15s %(name)s [%(levelname)s]'
                                      '%(message)s')
        fh = logging.FileHandler(logpath)
        fh.setFormatter(formatter)
        self.logger.addHandler(fh)
        self.n_personae = len(self.personae)
        self.centralities = pd.DataFrame(index = [p for p in self.personae])
        self.centralities.index.name = "name"
        self.randomization = randomization
        self.metrics = pd.DataFrame()
        self.G = self.create_graph()
        if action == "char_metrics":
            self.analyze_characters()
            self.get_character_frequencies()
            self.get_character_speech_amounts()
            self.get_character_ranks()
            self.get_centrality_ranks()
            self.get_ranking_stability_measures()
            self.export_char_metrics()
        if action == "corpus_metrics":
            self.graph_metrics = self.get_graph_metrics()
            self.export_graph_metrics()
        if action == "both":
            self.graph_metrics = self.get_graph_metrics()
            self.analyze_characters()
            self.get_character_frequencies()
            self.get_character_speech_amounts()
            self.get_character_ranks()
            self.get_centrality_ranks()
            self.get_ranking_stability_measures()
            self.export_char_metrics()
            self.export_graph_metrics()

    def get_final_scene_size(self):
        last_scene_size = len(self.segments[-1])
        return last_scene_size / self.n_personae

    def get_drama_change_rate_metrics(self):
        change_rates = self.get_drama_change_rate()
        cr_mean = numpy.mean(change_rates)
        cr_std = numpy.std(change_rates)
        return cr_mean, cr_std

    def get_drama_change_rate(self):
        change_rates = []
        for x, y in zip_longest(self.segments[:-1], self.segments[1:]):
            s = set(x)
            t = set(y)
            u = s.intersection(t)
            cr = abs(len(s)-len(u)) + abs(len(u)-len(t))
            cr_sum = len(s.union(t))
            change_rates.append(cr/cr_sum)
        return change_rates

    def get_central_character_entry(self):
        central_character = self.get_central_character()
        for i, segment in enumerate(self.segments):
            if central_character in segment:
                i += 1
                central_character_entry_index = float(i/len(self.segments))
                return central_character_entry_index

    def get_central_character(self):
        # either that or hardcode
        ranks = [c for c in self.centralities.columns if c.endswith("rank")]
        # sum up all rank values per character, divide by nr. of rank metrics
        avg_ranks = self.centralities[ranks].sum(axis=1)/len(ranks)
        min_rank = min(avg_ranks)
        central_chars = avg_ranks[avg_ranks == min_rank].index.tolist()
        if len(central_chars) == 1:
            return central_chars[0]
        else:
            return "SEVERAL"

    def get_character_frequencies(self):
        self.centralities['frequency'] = 0
        frequencies = Counter(list(chain.from_iterable(self.segments)))
        for char, freq in frequencies.items():
            self.centralities.loc[char, 'frequency'] = freq

    def get_character_speech_amounts(self):
        for amount in ["speech_acts", "words", "lines", "chars"]:
            self.centralities[amount] = 0
            for name, person in self.personae.items():
                self.centralities.loc[person.name, amount] = person.amounts.get(amount)

    def get_top_ranked_chars(self):
        top_ranked = {}
        # check whether metric should be sorted ascending(min) or descending(max)
        for metric in ['degree', 'closeness', 'betweenness', 'frequency']:
            cent_max = self.centralities[metric].max()
            top_char = self.centralities[self.centralities['closeness'] == cent_max].index.tolist()
            if len(top_char) != 1:
                top_ranked[metric] = "SEVERAL"
            else:
                top_ranked[metric] = top_char[0]
        # top_ranked['central'] = self.get_central_character()
        return top_ranked

    def get_character_ranks(self):
        for metric in ['degree', 'closeness', 'betweenness',
                       'strength', 'eigenvector_centrality',
                       'frequency', 'speech_acts', 'words']:
            # ascending: False for ranks by high (1) to low (N)
            # check ascending value for each metric
            self.centralities[metric+"_rank"] = self.centralities[metric].rank(method='min', ascending=False)

    def get_centrality_ranks(self):
        ranks = [c for c in self.centralities.columns if c.endswith("rank")]
        self.centralities['centrality_rank_avg'] = self.centralities[ranks].sum(axis=1)/len(ranks)
        self.centralities['centrality_rank_std'] = self.centralities[ranks].std(axis=1)/len(ranks)
        for metric in ['centrality_rank_avg', 'centrality_rank_std']:
            self.centralities[metric+"_rank"] = self.centralities[metric].rank(method='min', ascending=False)

    def get_central_characters(self):
        self.centralities['composite_centrality'] = self.centralities.apply(
                                        lambda x: (x['frequency_rank'] +
                                                   x['avg_centrality_rank'])/2,
                                        axis=1)

    def get_ranking_stability_measures(self):
        ranks = [c for c in self.centralities.columns if c.endswith("rank")][:8]
        self.ranking_stability = self.centralities[ranks].corr(method='kendall')

    def get_characters_all_in_index(self):
        appeared = set()
        for i, speakers in enumerate(self.segments):
            for sp in speakers:
                appeared.add(sp)
            if len(appeared) >= self.num_chars_total:
                i += 1
                all_in_index = float(i/len(self.segments))
                # print(all_in_index, len(self.segments), len(appeared), self.num_chars_total)
                return all_in_index

    def export_char_metrics(self):
        self.centralities.index.name = "name"
        self.centralities.to_csv(
                os.path.join(
                            self.outputfolder,
                            "%s_%s_chars.csv" % (self.ID, self.title)
                            ))
        self.ranking_stability.to_csv(
                os.path.join(
                            self.outputfolder,
                            "%s_%s_kendalltaus.csv" % (self.ID, self.title)
                            ))

    def export_graph_metrics(self):
        self.graph_metrics.index.name = "ID"
        self.graph_metrics.index = self.graph_metrics["ID"]
        self.graph_metrics.to_csv(os.path.join(
                                    self.outputfolder,
                                    "%s_%s_graph.csv" % (self.ID, self.title)),
                                   sep=";")
        self.export_table(
                self.get_drama_change_rate(),
                "_".join([self.filepath, self.title,"change_rates"])+".csv")
        nx.write_edgelist(
                self.G,
                os.path.join(self.outputfolder,
                             "_".join([str(self.ID),
                             self.title,"edgelist"])+".csv"),
                delimiter=";",
                data=["weight"])
        plotGraph(
                self.G,
                filename=os.path.join(self.outputfolder,
                                      "_".join([str(self.ID),
                                      self.title])+".svg"))

    def export_table(self, t, filepath):
        with open(filepath, 'w') as f:  # Just use 'w' mode in 3.x
            csvwriter = csv.writer(f, delimiter=';')
            csvwriter.writerow(["segment", "change_rate"])
            for i, t in enumerate(t):
                csvwriter.writerow([i+1, t])

    def get_graph_metrics(self):
        graph_metrics = self.analyze_graph()
        graph_metrics["ID"] = self.ID
        graph_metrics["average_path_length_random"], graph_metrics["clustering_coefficient_random"] = self.randomize_graph(graph_metrics.get("charcount"), graph_metrics.get("edgecount"))
        graph_metrics["year"] = self.metadata.get("date_definite")
        graph_metrics["author"] = self.metadata.get("author")
        graph_metrics["title"] = self.title
        graph_metrics["filename"] = self.metadata.get("filename")
        graph_metrics["genretitle"] = self.metadata.get("genretitle")
        graph_metrics["subtitle"] = self.metadata.get("subtitle")
        graph_metrics["segment_count"] = self.metadata.get("segment_count")
        graph_metrics["count_type"] = self.metadata.get("count_type")
        graph_metrics["all_in_index"] = self.get_characters_all_in_index()
        graph_metrics["change_rate_mean"], graph_metrics["change_rate_std"] = self.get_drama_change_rate_metrics()
        graph_metrics["final_scene_size_index"] = self.get_final_scene_size()
        # graph_metrics["central_character"] = self.get_central_character()
        # graph_metrics["central_character_entry_index"] = self.get_central_character_entry()
        graph_metrics["characters_last_in"] = self.get_characters_last_in()
        return pd.DataFrame.from_dict(graph_metrics, orient='index').T

    def get_characters_last_in(self):
        last_chars = self.segments[-1]
        return ",".join(last_chars)<commit_msg>ranking stability metric output added<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# dramavis by <NAME> (@umblaetterer) & <NAME> (@chris_kittel)

__author__ = "<NAME> <web at christopherkittel.eu>, <NAME> <ffischer at hse.ru>"
__copyright__ = "Copyright 2017"
__license__ = "MIT"
__version__ = "0.4 (beta)"
__maintainer__ = "<NAME> <ffischer at hse.ru>"
__status__ = "Development" # 'Development', 'Production' or 'Prototype'

import os
import glob
import pandas as pd
import networkx as nx
import csv
from lxml import etree
from itertools import chain, zip_longest
from collections import Counter
import argparse
from superposter import plotGraph, plot_superposter
import logging
import numpy as np
from tqdm import tqdm

from linacorpus import LinaCorpus, Lina


class CorpusAnalyzer(LinaCorpus):

    def __init__(self, inputfolder, outputfolder, logpath):
        super(CorpusAnalyzer, self).__init__(inputfolder, outputfolder)
        self.logger = logging.getLogger("corpusAnalyzer")
        formatter = logging.Formatter('%(asctime)-15s %(name)s [%(levelname)s]'
                                      '%(message)s')
        fh = logging.FileHandler(logpath)
        fh.setFormatter(formatter)
        self.logger.addHandler(fh)
        self.logpath = logpath


    def analyze_dramas(self, action):
        """
        Reads all XMLs in the inputfolder,
        returns an iterator of lxml.etree-objects created with lxml.etree.parse("dramafile.xml").
        """
        # dramas = {}
        for dramafile in tqdm(self.dramafiles, desc="Dramas", mininterval=0.5):
            # ID, ps = parse_drama(tree, filename)
            # dramas[ID] = ps
            drama = DramaAnalyzer(dramafile, self.outputfolder, self.logpath,
                                  action)
            yield drama

    def get_char_metrics(self):
        self.logger.info("Exporting character metrics.")
        dramas = self.analyze_dramas(action="char_metrics")
        header = [
                    'ID', 'author', 'title', 'year',
                    'frequency', 'degree', 'betweenness', 'closeness'
                 ]
        dfs = []
        for drama in dramas:
            temp_df = pd.DataFrame(index=[drama.ID])
            for m in header[1:3]:
                temp_df[m] = drama.metadata.get(m)
            temp_df['year'] = drama.metadata.get('date_definite')
            for m in header[4:]:
                temp_df[m] = drama.get_top_ranked_chars()[m]
            temp_df['ID'] = drama.ID
            dfs.append(temp_df)
        df = pd.concat(dfs)
        df = df[header]
        df.index = df['ID']
        df.index.name = 'index'
        df.to_csv(os.path.join(self.outputfolder,
                               "central_characters.csv"), sep=";")

    def get_graph_metrics(self):
        self.logger.info("Exporting corpus metrics.")
        dramas = self.analyze_dramas(action="corpus_metrics")
        df = pd.concat([d.graph_metrics for d in dramas])
        header = [
                'ID', 'author', 'title', 'subtitle', 'year', 'genretitle', 'filename',
                'charcount', 'edgecount', 'maxdegree', 'avgdegree',
                'clustering_coefficient', 'clustering_coefficient_random', 'avgpathlength', 'average_path_length_random', 'density',
                'segment_count', 'count_type', 'all_in_index', 'change_rate_mean', 'change_rate_std', 'final_scene_size_index',
                'characters_last_in',
                'connected_components', 'kendall_tau_avg', 'kendall_tau_std'
                ]
        df.index = df["ID"]
        df.index.name = "index"
        df[header].to_csv(os.path.join(self.outputfolder, "corpus_metrics.csv"), sep=";")

    def get_both_metrics(self):
        self.logger.info("Exporting character metrics.")
        dramas = self.analyze_dramas(action="both")
        header = [
                    'ID', 'author', 'title', 'year',
                    'frequency', 'degree', 'betweenness', 'closeness'
                 ]
        char_dfs = []
        graph_dfs = []
        for drama in dramas:
            temp_df = pd.DataFrame(index=[drama.ID])
            for m in header[1:3]:
                temp_df[m] = drama.metadata.get(m)
            temp_df['year'] = drama.metadata.get('date_definite')
            for m in header[4:]:
                temp_df[m] = drama.get_top_ranked_chars()[m]
            temp_df['ID'] = drama.ID
            char_dfs.append(temp_df)
            graph_dfs.append(drama.graph_metrics)
        df = pd.concat(char_dfs)
        df = df[header]
        df.index = df['ID']
        df.index.name = 'index'
        df.to_csv(os.path.join(self.outputfolder,
                               "central_characters.csv"), sep=";")
        self.logger.info("Exporting corpus metrics.")
        df = pd.concat(graph_dfs)
        header = [
                'ID', 'author', 'title', 'subtitle', 'year', 'genretitle', 'filename',
                'charcount', 'edgecount', 'maxdegree', 'avgdegree',
                'clustering_coefficient', 'clustering_coefficient_random', 'avgpathlength', 'average_path_length_random', 'density',
                'segment_count', 'count_type', 'all_in_index', 'change_rate_mean', 'change_rate_std', 'final_scene_size_index',
                'characters_last_in',
                'connected_components', 'kendall_tau_avg', 'kendall_tau_std'
                ]
        df.index = df["ID"]
        df.index.name = "index"
        df.to_csv(os.path.join(self.outputfolder, "corpus_metrics.csv"), sep=";")


class DramaAnalyzer(Lina):

    def __init__(self, dramafile, outputfolder, logpath,
                 action, randomization=1000):
        super(DramaAnalyzer, self).__init__(dramafile, outputfolder)
        self.logger = logging.getLogger("dramaAnalyzer")
        formatter = logging.Formatter('%(asctime)-15s %(name)s [%(levelname)s]'
                                      '%(message)s')
        fh = logging.FileHandler(logpath)
        fh.setFormatter(formatter)
        self.logger.addHandler(fh)
        self.n_personae = len(self.personae)
        self.centralities = pd.DataFrame(index = [p for p in self.personae])
        self.centralities.index.name = "name"
        self.randomization = randomization
        self.metrics = pd.DataFrame()
        self.G = self.create_graph()
        if action == "char_metrics":
            self.analyze_characters()
            self.get_character_frequencies()
            self.get_character_speech_amounts()
            self.get_character_ranks()
            self.get_centrality_ranks()
            self.get_ranking_stability_measures()
            self.export_char_metrics()
        if action == "corpus_metrics":
            self.graph_metrics = self.get_graph_metrics()
            self.export_graph_metrics()
        if action == "both":
            self.graph_metrics = self.get_graph_metrics()
            self.analyze_characters()
            self.get_character_frequencies()
            self.get_character_speech_amounts()
            self.get_character_ranks()
            self.get_centrality_ranks()
            self.get_ranking_stability_measures()
            self.add_ranking_stability_metrics()
            self.export_char_metrics()
            self.export_graph_metrics()

    def add_ranking_stability_metrics(self):
        self.graph_metrics["kendall_tau_avg"] = self.ranking_stability.stack().mean()
        self.graph_metrics["kendall_tau_std"] = self.ranking_stability.stack().std()
        (self.graph_metrics["top_rank_char_count"],
         self.graph_metrics["top_rank_char_avg"],
         self.graph_metrics["top_rank_char_std"]) = self.get_top_ranked_char_count()

    def get_final_scene_size(self):
        last_scene_size = len(self.segments[-1])
        return last_scene_size / self.n_personae

    def get_drama_change_rate_metrics(self):
        change_rates = self.get_drama_change_rate()
        cr_mean = np.mean(change_rates)
        cr_std = np.std(change_rates)
        return cr_mean, cr_std

    def get_drama_change_rate(self):
        change_rates = []
        for x, y in zip_longest(self.segments[:-1], self.segments[1:]):
            s = set(x)
            t = set(y)
            u = s.intersection(t)
            cr = abs(len(s)-len(u)) + abs(len(u)-len(t))
            cr_sum = len(s.union(t))
            change_rates.append(cr/cr_sum)
        return change_rates

    def get_central_character_entry(self):
        central_character = self.get_central_character()
        for i, segment in enumerate(self.segments):
            if central_character in segment:
                i += 1
                central_character_entry_index = float(i/len(self.segments))
                return central_character_entry_index

    def get_central_character(self):
        # either that or hardcode
        ranks = [c for c in self.centralities.columns if c.endswith("rank")]
        # sum up all rank values per character, divide by nr. of rank metrics
        avg_ranks = self.centralities[ranks].sum(axis=1)/len(ranks)
        min_rank = min(avg_ranks)
        central_chars = avg_ranks[avg_ranks == min_rank].index.tolist()
        if len(central_chars) == 1:
            return central_chars[0]
        else:
            return "SEVERAL"

    def get_character_frequencies(self):
        self.centralities['frequency'] = 0
        frequencies = Counter(list(chain.from_iterable(self.segments)))
        for char, freq in frequencies.items():
            self.centralities.loc[char, 'frequency'] = freq

    def get_character_speech_amounts(self):
        for amount in ["speech_acts", "words", "lines", "chars"]:
            self.centralities[amount] = 0
            for name, person in self.personae.items():
                self.centralities.loc[person.name, amount] = person.amounts.get(amount)

    def get_top_ranked_chars(self):
        top_ranked = {}
        # check whether metric should be sorted ascending(min) or descending(max)
        for metric in ['degree', 'closeness', 'betweenness', 'frequency']:
            cent_max = self.centralities[metric].max()
            top_char = self.centralities[self.centralities[metric] == cent_max].index.tolist()
            if len(top_char) != 1:
                top_ranked[metric] = "SEVERAL"
            else:
                top_ranked[metric] = top_char[0]
        # top_ranked['central'] = self.get_central_character()
        return top_ranked

    def get_top_ranked_char_count(self):
        avg_min = self.centralities['centrality_rank_avg'].min()
        top_chars = self.centralities[self.centralities['centrality_rank_avg'] == avg_min].index.tolist()
        top_std = self.centralities[self.centralities['centrality_rank_avg'] == avg_min]['centrality_rank_std']
        return len(top_chars), avg_min, top_std

    def get_character_ranks(self):
        for metric in ['degree', 'closeness', 'betweenness',
                       'strength', 'eigenvector_centrality',
                       'frequency', 'speech_acts', 'words']:
            # ascending: False for ranks by high (1) to low (N)
            # check ascending value for each metric
            self.centralities[metric+"_rank"] = self.centralities[metric].rank(method='min', ascending=False)

    def get_centrality_ranks(self):
        ranks = [c for c in self.centralities.columns if c.endswith("rank")]
        self.centralities['centrality_rank_avg'] = self.centralities[ranks].sum(axis=1)/len(ranks)
        self.centralities['centrality_rank_std'] = self.centralities[ranks].std(axis=1)/len(ranks)
        for metric in ['centrality_rank_avg', 'centrality_rank_std']:
            self.centralities[metric+"_rank"] = self.centralities[metric].rank(method='min', ascending=False)

    def get_ranking_stability_measures(self):
        ranks = [c for c in self.centralities.columns if c.endswith("rank")][:8]
        self.ranking_stability = self.centralities[ranks].corr(method='kendall')
        np.fill_diagonal(self.ranking_stability.values, np.nan)
        self.ranking_stability.index.name = "rank_name"

    def get_characters_all_in_index(self):
        appeared = set()
        for i, speakers in enumerate(self.segments):
            for sp in speakers:
                appeared.add(sp)
            if len(appeared) >= self.num_chars_total:
                i += 1
                all_in_index = float(i/len(self.segments))
                # print(all_in_index, len(self.segments), len(appeared), self.num_chars_total)
                return all_in_index

    def export_char_metrics(self):
        self.centralities.index.name = "name"
        self.centralities.to_csv(
                os.path.join(
                            self.outputfolder,
                            "%s_%s_chars.csv" % (self.ID, self.title)
                            ))
        self.ranking_stability.to_csv(
                os.path.join(
                            self.outputfolder,
                            "%s_%s_kendalltaus.csv" % (self.ID, self.title)
                            ))

    def export_graph_metrics(self):
        self.graph_metrics.index.name = "ID"
        self.graph_metrics.index = self.graph_metrics["ID"]
        self.graph_metrics.to_csv(os.path.join(
                                    self.outputfolder,
                                    "%s_%s_graph.csv" % (self.ID, self.title)),
                                   sep=";")
        self.export_table(
                self.get_drama_change_rate(),
                "_".join([self.filepath, self.title,"change_rates"])+".csv")
        nx.write_edgelist(
                self.G,
                os.path.join(self.outputfolder,
                             "_".join([str(self.ID),
                             self.title,"edgelist"])+".csv"),
                delimiter=";",
                data=["weight"])
        plotGraph(
                self.G,
                filename=os.path.join(self.outputfolder,
                                      "_".join([str(self.ID),
                                      self.title])+".svg"))

    def export_table(self, t, filepath):
        with open(filepath, 'w') as f:  # Just use 'w' mode in 3.x
            csvwriter = csv.writer(f, delimiter=';')
            csvwriter.writerow(["segment", "change_rate"])
            for i, t in enumerate(t):
                csvwriter.writerow([i+1, t])

    def get_graph_metrics(self):
        graph_metrics = self.analyze_graph()
        graph_metrics["ID"] = self.ID
        graph_metrics["average_path_length_random"], graph_metrics["clustering_coefficient_random"] = self.randomize_graph(graph_metrics.get("charcount"), graph_metrics.get("edgecount"))
        graph_metrics["year"] = self.metadata.get("date_definite")
        graph_metrics["author"] = self.metadata.get("author")
        graph_metrics["title"] = self.title
        graph_metrics["filename"] = self.metadata.get("filename")
        graph_metrics["genretitle"] = self.metadata.get("genretitle")
        graph_metrics["subtitle"] = self.metadata.get("subtitle")
        graph_metrics["segment_count"] = self.metadata.get("segment_count")
        graph_metrics["count_type"] = self.metadata.get("count_type")
        graph_metrics["all_in_index"] = self.get_characters_all_in_index()
        graph_metrics["change_rate_mean"], graph_metrics["change_rate_std"] = self.get_drama_change_rate_metrics()
        graph_metrics["final_scene_size_index"] = self.get_final_scene_size()
        graph_metrics["characters_last_in"] = self.get_characters_last_in()
        return pd.DataFrame.from_dict(graph_metrics, orient='index').T

    def get_characters_last_in(self):
        last_chars = self.segments[-1]
        return ",".join(last_chars)
{'fraction_non_alphanumeric': 0.12035901078323756, 'fraction_numerical': 0.002841092529218054, 'mean_word_length': 2.2896134239592185, 'pattern_counts': {'":': 6, '<': 19, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3545228', 'n_tokens_mistral': 9589, 'n_tokens_neox': 9191, 'n_words': 1613}