# Path: poetry/settings.py
# BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))


import itertools
import os
import re
import xml.etree.ElementTree as etree
from dicttoxml import dicttoxml
from rupo.util.preprocess import to_cyrrilic, normilize_line
from poetry.settings import BASE_DIR
                        if len(element['themes']) == 0:
                            theme_list = "[]"
                        else:
                            theme_list = "["
                            for theme in element['themes']:
                                theme_list += str(themes_to_pk[theme]) + ","
                            theme_list = theme_list[:-1] + "]"
                        content += '"' + str(field) + '": ' + theme_list + ','
                content = content[:-1] + '}},'
                pk += 1
            content = content[:-1] + ']'
            f.write(content)

    def print_statistics(self):
        print("Example:", self.data[0])
        print("Invalid: ", self.invalid_count)
        print("Duplicates: ", self.duplicates_count)
        print("Concatenated: ", self.concatenated_count)
        print("Total chars: ", sum([len(elem['text']) for elem in self.data]))
        print("Total words: ", sum([len(elem['text'].split()) for elem in self.data]))
        print("Total poems after deduplication: ", len(self.data))
        print("Total poems with themes: ", len([elem for elem in self.data if len(elem['themes']) != 0]))
        authors = set()
        for elem in self.data:
            authors.add(normilize_line(elem['author']))
        print("Total authors: ", len(authors))


def main():
    corpus = Corpus()

# Path: poetry/settings.py
# BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))


# Path: poetry/apps/corpus/scripts/unite.py
import itertools
import os
import re
import xml.etree.ElementTree as etree
from dicttoxml import dicttoxml
from rupo.util.preprocess import to_cyrrilic, normilize_line
from poetry.settings import BASE_DIR

                        if len(element['themes']) == 0:
                            theme_list = "[]"
                        else:
                            theme_list = "["
                            for theme in element['themes']:
                                theme_list += str(themes_to_pk[theme]) + ","
                            theme_list = theme_list[:-1] + "]"
                        content += '"' + str(field) + '": ' + theme_list + ','
                content = content[:-1] + '}},'
                pk += 1
            content = content[:-1] + ']'
            f.write(content)

    def print_statistics(self):
        print("Example:", self.data[0])
        print("Invalid: ", self.invalid_count)
        print("Duplicates: ", self.duplicates_count)
        print("Concatenated: ", self.concatenated_count)
        print("Total chars: ", sum([len(elem['text']) for elem in self.data]))
        print("Total words: ", sum([len(elem['text'].split()) for elem in self.data]))
        print("Total poems after deduplication: ", len(self.data))
        print("Total poems with themes: ", len([elem for elem in self.data if len(elem['themes']) != 0]))
        authors = set()
        for elem in self.data:
            authors.add(normilize_line(elem['author']))
        print("Total authors: ", len(authors))


def main():
    corpus = Corpus()

    text_files = [os.path.join(BASE_DIR, "datasets", "web", "rupoem.xml"), os.path.join(BASE_DIR, "datasets", "web", "strofa.xml"),