# Path: poetry/settings.py
# BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))


import itertools
import os
import re
import xml.etree.ElementTree as etree
from dicttoxml import dicttoxml
from rupo.util.preprocess import to_cyrrilic, normilize_line
from poetry.settings import BASE_DIR
            'path': './/text',
            'handler': lambda x: x if x is None else clean_text(x.text),
            'required': True
        },
        'date_from': {
            'path': './/date_from',
            'handler': lambda x: x if x is None else int(x.text),
            'required': False
        },
        'date_to': {
            'path': './/date_to',
            'handler': lambda x: x if x is None else int(x.text),
            'required': False
        },
        'themes': {
            'path': './/themes',
            'handler': lambda x: [] if x is None else [value.text for value in x.findall(".//value")],
            'required': False
        },
    }
    for filename in text_files:
        corpus.process_file(filename, './/item', fields)

    fields['text']['required'] = False
    fields['themes']['required'] = True
    for filename in themes_files:
        corpus.process_file(filename, './/item', fields)

    corpus.clean_duplicates()
    corpus.print_statistics()

# Path: poetry/settings.py
# BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))


# Path: poetry/apps/corpus/scripts/unite.py
import itertools
import os
import re
import xml.etree.ElementTree as etree
from dicttoxml import dicttoxml
from rupo.util.preprocess import to_cyrrilic, normilize_line
from poetry.settings import BASE_DIR

            'path': './/text',
            'handler': lambda x: x if x is None else clean_text(x.text),
            'required': True
        },
        'date_from': {
            'path': './/date_from',
            'handler': lambda x: x if x is None else int(x.text),
            'required': False
        },
        'date_to': {
            'path': './/date_to',
            'handler': lambda x: x if x is None else int(x.text),
            'required': False
        },
        'themes': {
            'path': './/themes',
            'handler': lambda x: [] if x is None else [value.text for value in x.findall(".//value")],
            'required': False
        },
    }
    for filename in text_files:
        corpus.process_file(filename, './/item', fields)

    fields['text']['required'] = False
    fields['themes']['required'] = True
    for filename in themes_files:
        corpus.process_file(filename, './/item', fields)

    corpus.clean_duplicates()
    corpus.print_statistics()

    corpus.to_xml(os.path.join(BASE_DIR, "datasets", "corpus", "all.xml"))