# Path: poetry/settings.py
# BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))


import itertools
import os
import re
import xml.etree.ElementTree as etree
from dicttoxml import dicttoxml
from rupo.util.preprocess import to_cyrrilic, normilize_line
from poetry.settings import BASE_DIR
                    continue
                data[id] = element
                themes = element['themes']
                data[id]['themes'] = set()
                if themes is not None:
                    for theme in themes:
                        data[id]['themes'].add(theme)
            else:
                if element['text'] is None:
                    if element['themes'] is not None:
                        for theme in element['themes']:
                            data[id]['themes'].add(theme)
                elif data[id]['text'] is None:
                    element['themes'] = data[id]['themes']
                    data[id] = element
                else:
                    is_duplicate = False
                    for i in range(3):
                        for j in range(3):
                            flag1 = get_actual_line(element['text'], i) in get_actual_line(data[id]['text'], j)
                            flag2 = get_actual_line(data[id]['text'], i) in get_actual_line(element['text'], j)
                            is_duplicate = is_duplicate or flag1 or flag2
                    if not is_duplicate:
                        # TODO: handle this case
                        self.concatenated_count += 1
                    else:
                        for field, value in data[id].items():
                            if value is None:
                                data[id][field] = element[field]
                        if len(element['text']) > len(data[id]['text']):

# Path: poetry/settings.py
# BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))


# Path: poetry/apps/corpus/scripts/unite.py
import itertools
import os
import re
import xml.etree.ElementTree as etree
from dicttoxml import dicttoxml
from rupo.util.preprocess import to_cyrrilic, normilize_line
from poetry.settings import BASE_DIR

                    continue
                data[id] = element
                themes = element['themes']
                data[id]['themes'] = set()
                if themes is not None:
                    for theme in themes:
                        data[id]['themes'].add(theme)
            else:
                if element['text'] is None:
                    if element['themes'] is not None:
                        for theme in element['themes']:
                            data[id]['themes'].add(theme)
                elif data[id]['text'] is None:
                    element['themes'] = data[id]['themes']
                    data[id] = element
                else:
                    is_duplicate = False
                    for i in range(3):
                        for j in range(3):
                            flag1 = get_actual_line(element['text'], i) in get_actual_line(data[id]['text'], j)
                            flag2 = get_actual_line(data[id]['text'], i) in get_actual_line(element['text'], j)
                            is_duplicate = is_duplicate or flag1 or flag2
                    if not is_duplicate:
                        # TODO: handle this case
                        self.concatenated_count += 1
                    else:
                        for field, value in data[id].items():
                            if value is None:
                                data[id][field] = element[field]
                        if len(element['text']) > len(data[id]['text']):

                            data[id]['text'] = element['text']