<commit_before># Python 3.3 or higher
import shelve
import os
import datetime
import time
import uuid
import glob
import sys

if sys.version_info.minor == 3:
    from contextlib import closing
    shelve_open = lambda file, flag="c", protocol=None, writeback=False: closing(shelve.open(file, flag))
else:
    shelve_open = shelve.open
from multiprocessing import Pipe
from zashel.utils import threadize
from zrest.basedatamodel import *
from zrest.exceptions import *
from math import ceil


class ShelveModel(RestfulBaseInterface):
    """
    ShelveModel with a double interface:
    An inner interface with new, edit, replace, drop and fetch whose take dictionaries.
    A Restful interface with post, patch, put, delete and get whose take json data.

    To use with zrest.

    """
    def __init__(self, filepath, groups=10, *, index_fields=None,
                                               headers=None,
                                               name=None,
                                               items_per_page=50,
                                               unique=None,
                                               unique_is_id=False,
                                               split_unique=0,
                                               to_block = True):
        """
        Initializes ShelveModel
        
        :param filepath: path to save the database files
        :param groups: splits to data database
        :param index_fields: fields indexed. Not indexed fields do not accept queries
        :param headers: headers of table. None by default. If None dictionaries are
        stored.
        :param name: name of the model
        :param items_per_page: amount of items each page
        :param unique: unique field. One bye the moment
        :param split_unique: number of characters of each piece in which unique is
                             splitted

        """
        try:
            assert os.path.exists(filepath)
        except AssertionError:
            os.makedirs(filepath)
        self.uuid = str(uuid.uuid4())
        self._filepath = filepath
        self._alive = False
        self._opened = True
        self._pipe_in, self._pipe_out = Pipe(False)
        self._close = False
        self._headers = headers
        self._headers_checked = False
        self._unique = unique
        self._split_unique = split_unique
        self._unique_is_id = unique_is_id
        self._name = name
        self.items_per_page = items_per_page
        self._to_block = to_block
        if index_fields is None:
            self._index_fields = list()
        else:
            assert isinstance(index_fields, list)
            self._index_fields = index_fields
        try:
            assert any([os.path.exists(file)
                    for file in glob.glob("{}.*".format(self._meta_path))]+[False])
        except AssertionError:
            with shelve_open(self._meta_path) as shelf:
                shelf["filepath"] = self._meta_path
                shelf["total"] = int()
                shelf["next"] = int()
                shelf["groups"] = groups
                shelf["class"] = self.__class__.__name__
                shelf["name"] = self._name
                shelf["ids"] = list()
            for index in self.index_fields:
                if (self._unique_is_id is True and self._unique != index) or self._unique_is_id is False:
                    with shelve_open(self._index_path(index)) as shelf:
                        shelf["filepath"] = self._index_path(index)
            for group in range(0, groups):
                with shelve_open(self._data_path(group)) as shelf:
                    shelf["filepath"] = self._data_path(group)
        self.writer = self._writer()
        with shelve_open(self._meta_path, "r") as shelf:
            self._groups = shelf["groups"]
        time.sleep(0.05)
        self._as_foreign = list()
        self._as_child = list()

    def __len__(self):
        final = None
        while True:
            try:
                with shelve_open(self._meta_path, "r") as meta:
                    final = meta["total"]
            except (KeyboardInterrupt, SystemExit):
                raise
            except:
                continue
            else:
                if final is not None:
                    break
        return final

    def __next__(self): #This is not very appropiate, but...
        final = None
        while True:
            try:
                with shelve_open(self._meta_path, "r") as meta:
                    final = meta["next"]
            except (KeyboardInterrupt, SystemExit):
                    raise
            except:
                continue
            else:
                if final is not None:
                    break
        return final

    @property
    def name(self):
        if self._name == None:
            with shelve_open(self._meta_path, "r") as shelf:
                self._name = shelf["name"]
        return self._name

    @name.setter
    def name(self, value):
        self._wait_to_block(self._meta_path)
        self._keep_alive(self._meta_path)
        if self._name == None:
            with shelve_open(self._meta_path) as shelf:
                shelf["name"] = value
        self._alive = False
        self._name = value

    @property
    def index_fields(self):
        return self._index_fields

    @property
    def indexes_files(self):
        return [self._index_path(index) for index in self.index_fields]

    @property
    def groups(self):
        return self._groups

    @property
    def data_files(self):
        return [self._data_path(index) for index in range(0, self.groups)]

    @property
    def filepath(self):
        return self._filepath
    
    @property
    def headers(self):
        if self._headers is None and self._headers_checked is False:
            try:
                with shelve_open(self._data_path(0), "r") as shelf:
                    self._headers = shelf["headers"]
            except KeyError:
                self._headers = None
            finally:
                self._headers_checked = True
        return self._headers

    @property
    def _meta_path(self):
        return os.path.join(self.filepath, "meta")

    def _index_path(self, field):
        return os.path.join(self.filepath, "index_{}".format(field))

    def _data_path(self, group):
        return os.path.join(self.filepath, "data_{}".format(str(group)))

    def _block(self, file):
        assert file in [self._meta_path, ] + self.indexes_files + self.data_files
        if self.is_blocked(file) is False:
            with open("{}.block".format(file), "w") as block:
                block.write("{}\t{}".format(self.uuid, datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
        else:
            raise BlockedFile

    def _unblock(self, file):
        if self.is_blocked(file) is False:
            try:
                os.remove("{}.block".format(file))
            except (FileNotFoundError, PermissionError):
                pass

    def is_blocked(self, file):
        """
        Checks if given file is blocked
        :param file: file to check.
        :returns: True if blocked, False if not blocked

        """
        blocked = False
        filepath = "{}.block".format(file)
        if os.path.exists(filepath):
            try:
                with open(filepath, "r") as block:
                    uuid, date = block.read().strip("\n").split("\t")
                date = datetime.datetime.strptime(date, "%Y-%m-%d %H:%M:%S")
                now = datetime.datetime.now()
                if uuid != self.uuid and date+datetime.timedelta(seconds=10)>now:
                    blocked = True
            except (FileNotFoundError, ValueError):
                pass
            except (PermissionError):
                blocked = True
        return blocked


    def _wait_to_block(self, file):
        while True:
            if self.is_blocked(file) is True:
                time.sleep(0.5)
            else:
                try:
                    self._block(file)
                except (BlockedFile, PermissionError):
                    continue
                else:
                    time.sleep(0.05)
                    if self.is_blocked(file) is False:
                        self._alive = True
                        break

    @threadize
    def _keep_alive(self, file):
        counter = int()
        while self._alive is True:
            if os.path.exists("{}.block".format(file)):
                if self._alive is True and counter%250==0:
                    try:
                        self._block(file)
                    except (BlockedFile, PermissionError, FileNotFoundError) as e:
                        print(e)
                        self._alive = False
            else:
                self._alive = False
            counter += 1
            time.sleep(0.5)
        self._unblock(file)

    def _send_pipe(self, **kwargs):
        self._pipe_out.send(kwargs)

    def fetch(self, filter, **kwargs):
        """
        Gives the result of a query.
        :param filter: dictionary with wanted coincidences
        :returns: dictionary with result of the query

        """
        filtered = self._filter(filter)
        filter = filtered["filter"]
        filter = self._get_datafile(filter)
        final = list()
        for filename in filter:
            final.extend(self._fetch(filter[filename], filename))
        new_final = list()
        for _id in filtered["filter"]:
            for index, item in enumerate(final):
                if "_id" in item and item["_id"]==_id:
                    if filtered["fields"]:
                        new = dict()
                        for field in item:
                            if field=="_id" or field in filtered["fields"]:
                                new[field] = item[field]
                        new_final.append(new)
                        break
                    else:
                        new_final.append(item)
                        break
            try:
                del(final[index])
            except UnboundLocalError:
                pass
        return ({"data": new_final,
                 "total": filtered["total"],
                 "page": filtered["page"],
                 "items_per_page": filtered["items_per_page"]})

    def _fetch(self, registries, shelf):
        if isinstance(registries, int):
            registries = {registries}
        final = list()
        with shelve_open(shelf, "r") as file:
            for item in registries:
                try:
                    data = file[str(item)]
                except KeyError:
                    data = None
                if isinstance(data, list) and self.headers is not None:
                    if data is not None and len(data)==len(self.headers):
                        data = dict(zip(self.headers, data))
                    else:
                        data = None
                if isinstance(data, dict):
                    data.update({"_id": item})
                if data is not None:
                    final.append(data)
        return final

    def _set_index(self, data, registry):
        if isinstance(data, list) and self.headers is not None and len(data) == len(self.headers):
            data = dict(zip(self.headers, data))
        for field in data:
            if (any([os.path.exists(file)
                    for file in glob.glob("{}.*".format(self._index_path(field)))]+[False]) and
                    (self.is_blocked(self._index_path(field)) is False or self._to_block is False)):
                with shelve_open(self._index_path(field)) as shelf:
                    index = str(data[field])
                    last = shelf
                    if not index in shelf:
                        if field != self._unique or self._split_unique == 0:
                            shelf[str(index)] = set()
                        elif self._unique_is_id is False:
                            offset = len(str(index))%self._split_unique
                            if offset:
                                inter = str(index)[0:offset]
                                if inter not in last:
                                    last[inter] = dict()
                                last = last[inter]
                            for x in range(ceil(len(str(index))/self._unique)):
                                inter = str(index)[offset+x*self._split_unique:offset+(x+1)*self._split_unique]
                                if inter not in last:
                                    last[inter] = dict()
                                last = last[inter]
                            last = registry
                    if field != self._unique:
                        shelf[str(index)] |= {registry}
                    else:
                        last = registry
                    
    def _del_index(self, data, registry):
        for field in data:
            if (any([os.path.exists(file)
                    for file in glob.glob("{}.*".format(self._index_path(field)))]+[False]) and
                    self.is_blocked(self._index_path(field)) is False):
                with shelve_open(self._index_path(field)) as shelf:
                    index = str(data[field])
                    if index in shelf:
                        shelf[index] -= {registry}

    def _check_child(self, data):
        if self._as_child:
            for foreign_key in self._as_child:
                if not foreign_key.field in data:
                    return 1
                else:
                    foreign = foreign_key.foreign.fetch({"_id": data[foreign_key.field]}) # Change to header
                    if not foreign:
                        return 2
        return 0

    def insert(self, data, **kwargs):
        """
        Loads new given data in the database
        Blocks until finnish
        :param data: list with a dictionary for each item to upload
        :returns: New Data
        """
        conn_in, conn_out = Pipe(False)
        if self._unique is not None:
            return {}
        self._send_pipe(action="insert", data=data, pipe=conn_out)
        recv = conn_in.recv()
        return recv

    def _insert(self, data, filename_reg):
        for filename in filename_reg:
            with shelve_open(filename) as shelf:
                for index in filename_reg[filename]:
                    new_data = data[str(index)]
                    if self.headers is not None:
                        new_data = list()
                        for header in self.headers:
                            try:
                                new_data.append(data[str(index)][header])
                            except KeyError:
                                new_data.append("")
                    shelf[str(index)] = new_data
        for index_name in self.index_fields:
             index_dict = dict()
             for index in data:
                 if str(data[index][index_name]) not in index_dict:
                     index_dict[str(data[index][index_name])] = set()
                 index_dict[str(data[index][index_name])].add(int(index))
             with shelve_open(self._index_path(index_name)) as shelf:
                 for index in index_dict:
                     if index not in shelf:
                         shelf[index] = index_dict[index]
                     else:
                         shelf[index] |= index_dict[index]
        with shelve_open(self._meta_path) as shelf:
             total, next_ = len(self), next(self)
             shelf["total"] = total + len(data)
             shelf["next"] = next_ + len(data)
             shelf["ids"] = set([int(key) for key in list(data.keys())])

    def new(self, data, **kwargs): #TODO: Errors setting new data
        """
        Set new given data in the database
        Blocks until finnish
        :param data: dictionary with given data. Saved as is if self.headers is None
        :returns: New Data

        """
        if self._check_child(data) != 0:
            return None
        conn_in, conn_out = Pipe(False)
        test = None
        if self._unique in data:
            test = self.fetch({self._unique: data[self._unique]})
        if test and "total" in test and test["total"] > 0:
            self._send_pipe(action="replace", data=data, filter={self._unique: data[self._unique]}, pipe=conn_out)
        else:
            self._send_pipe(action="new", data=data, pipe=conn_out)
        recv = conn_in.recv()
        return recv

    def _new(self, data, registry, shelf):
        with shelve_open(shelf) as file:
            if self.headers is not None:
                new_data = list()
                for header in self.headers:
                    try:
                        new_data.append(data[header])
                    except KeyError:
                        new_data.append("")
                data = new_data
            file[str(registry)] = data
        with shelve_open(self._meta_path) as file:
            total, next_ = len(self), next(self) #Bug!
            file["total"] = total + 1
            file["next"] = next_ + 1
            ids = list(file["ids"])
            ids.append(str(registry))
            file["ids"] = ids
        self._set_index(data, registry)

    def replace(self, filter, data, **kwargs):
        """
        Replaces all data which coincides with given filter with given data
        Blocks untill finnish
        :param filter: dictionary with coincidences
        :param data: dictionary with new data. It can be partial.
        :returns: Data replaced

        """
        if self._check_child(data) != 0:
            return None
        conn_in, conn_out = Pipe(False)
        test = None
        if ((self._unique in data and self._unique not in filter) or
            (self._unique in data and self._unique in filter and data[self._unique]!=filter[self._unique])):
            test = self.fetch({self._unique: data[self._unique]})
        if not test:
            self._send_pipe(action="replace", filter=filter, data=data, pipe=conn_out)
        else:
            return {"Error": "400"}
        return conn_in.recv()

    def _replace(self, data, registries, shelf):
        with shelve_open(shelf) as file:
            for reg in registries:
                try:
                    old_data = self._fetch({reg}, shelf)[0]
                except IndexError:
                    continue
                else:
                    if isinstance(old_data, dict): # Verified twice. It has to be a dict
                        if "_id" in old_data:
                            del(old_data["_id"])
                        new_data = old_data.copy()
                        new_data.update(data)
                        self._del_index(old_data, reg)
                        if self.headers is not None:
                            new_data = [new_data[item] for item in self.headers]
                        file[str(reg)] = new_data
                        self._set_index(new_data, reg)

    def edit(self, filter, data, **kwargs):
        """
        replace alias

        """
        if self._check_child(data) == 2:
            return None
        conn_in, conn_out = Pipe(False)
        test = None
        if ((self._unique in data and self._unique not in filter) or
                (self._unique in data and self._unique in filter and data[self._unique] != filter[self._unique])):
            test = self.fetch({self._unique: data[self._unique]})
        if not test:
            self._send_pipe(action="edit", filter=filter, data=data, pipe=conn_out)
        else:
            return {"Error": "400"}
        return conn_in.recv()

    def _edit(self, data, registries, shelf):
        self._replace(data, registries, shelf)

    def drop(self, filter, **kwargs):
        """
        Deletes data from database which coincides with given filter
        Blocks untill finnish
        :param filter: dictionary with given filter
        :returns: Data
        """
        conn_in, conn_out = Pipe(False)
        self._send_pipe(action="drop", filter=filter, data={}, pipe=conn_out)
        return conn_in.recv()

    def _drop(self, data, registries, shelf):
        with shelve_open(shelf) as file:
            for reg in registries:
                try:
                    old_data = self._fetch({reg}, shelf)
                except KeyError:
                    continue
                else:
                    if self._as_foreign:
                        for item in self._as_foreign:
                            children = item.children.fetch({item.field: reg})
                            if item:
                                continue
                    if old_data != list():
                        self._del_index(old_data, reg)
                        del(file[str(reg)])
                        with shelve_open(self._meta_path) as file:
                            file["total"] -= 1
                            ids = list(file["ids"])
                            del(ids[ids.index(str(reg))])
                            file["ids"] = ids

    def _filter(self, filter):
        with shelve_open(self._meta_path) as shelf:
            ids = list(shelf["ids"])
            final_set = set([int(id) for id in ids])
        #final_set = set(range(0, next(self)))
        order = str()
        fields = list()
        page = 1
        items_per_page = self.items_per_page
        if self._unique_is_id and self._unique in filter:
            filter["_id"] = filter[self._unique]
            del(filter[self._unique])
        if "order" in filter:
            order = filter["order"]
            order = order.split(",")
        if "page" in filter:
            page = filter["page"]
        if "items_per_page" in filter:
            items_per_page = filter["items_per_page"]
        if "fields" in filter:
            fields = filter["fields"].split(",")
        sub_order = dict()
        final_order = list()
        for field in filter:
            subfilter = set()
            if field == "_id" and filter[field] != "":
                subfilter = {int(filter["_id"])}
            elif field == "_id" and filter[field] == "":
                subfilter = final_set
            else:
                if any([os.path.exists(file)
                        for file in glob.glob("{}.*".format(self._index_path(field)))]+[False]):
                    with shelve_open(self._index_path(field), "r") as index:
                        if self._unique != field or self._split_unique == 0:
                            if str(filter[field]) in index:
                                subfilter = index[str(filter[field])]
                        else:
                            offset = len(str(index))%self._split_unique
                            last = index
                            if offset:
                                inter = str(index)[0:offset]
                                last = last[inter]
                            for x in range(ceil(len(str(index))/self._split_unique)):
                                inter = str(index)[offset+x*self._split_unique:offset+(x+1)*self._split_unique]
                                last = last[inter]
                            subfilter = {last}
            final_set &= subfilter
        final_set = list(final_set)
        final_set.sort()
        if len(order) > 0:
            for _id in final_set:
                field = order[0] #TODO: Accept many fields
                if field.startswith("-"):
                    sfield = field[1:]
                else:
                    sfield = field
                if any([os.path.exists(file)
                        for file in glob.glob("{}.*".format(self._index_path(sfield)))] + [False]):
                    with shelve_open(self._index_path(sfield), "r") as index:
                        sub_order[sfield] = index.copy()
                    keys = list(sub_order.keys())
                    if field.startswith("-"):
                        keys.reverse()
                    else:
                        keys.sort()
                    for indexes in keys:
                        for key in indexes:
                            if key in final_set:
                                final_order.append(key)
        else:
            final_order = final_set
        return {"filter": final_order[items_per_page*(int(page)-1):items_per_page*int(page)],
                "total": len(final_order),
                "page": page,
                "items_per_page": items_per_page,
                "fields": fields}

    def _get_datafile(self, filter):
        assert isinstance(filter, list)
        filename_reg = dict()
        for reg in filter:
            filename = self._data_path(reg % self.groups)
            if filename not in filename_reg:
                filename_reg[filename] = set()
            filename_reg[filename] |= {reg}
        return filename_reg

    @threadize
    def _writer(self):
        """
        It may receive by self._pipe_out a dictionary with:
        action: new, replace, drop or edit
        filter: if not new, a set of registries
        data: dictionary with the new data
        """
        new = int()
        while True:<commit_msg>debug implementation of LinkButton and TreeView<commit_after># Python 3.3 or higher
import shelve
import os
import datetime
import time
import uuid
import glob
import sys

if sys.version_info.minor == 3:
    from contextlib import closing
    shelve_open = lambda file, flag="c", protocol=None, writeback=False: closing(shelve.open(file, flag))
else:
    shelve_open = shelve.open
from multiprocessing import Pipe
from zashel.utils import threadize
from zrest.basedatamodel import *
from zrest.exceptions import *
from math import ceil


class ShelveModel(RestfulBaseInterface):
    """
    ShelveModel with a double interface:
    An inner interface with new, edit, replace, drop and fetch whose take dictionaries.
    A Restful interface with post, patch, put, delete and get whose take json data.

    To use with zrest.

    """
    def __init__(self, filepath, groups=10, *, index_fields=None,
                                               headers=None,
                                               name=None,
                                               items_per_page=50,
                                               unique=None,
                                               unique_is_id=False,
                                               split_unique=0,
                                               to_block = True):
        """
        Initializes ShelveModel
        
        :param filepath: path to save the database files
        :param groups: splits to data database
        :param index_fields: fields indexed. Not indexed fields do not accept queries
        :param headers: headers of table. None by default. If None dictionaries are
        stored.
        :param name: name of the model
        :param items_per_page: amount of items each page
        :param unique: unique field. One bye the moment
        :param split_unique: number of characters of each piece in which unique is
                             splitted

        """
        try:
            assert os.path.exists(filepath)
        except AssertionError:
            os.makedirs(filepath)
        self.uuid = str(uuid.uuid4())
        self._filepath = filepath
        self._alive = False
        self._opened = True
        self._pipe_in, self._pipe_out = Pipe(False)
        self._close = False
        self._headers = headers
        self._headers_checked = False
        self._unique = unique
        self._split_unique = split_unique
        self._unique_is_id = unique_is_id
        self._name = name
        self.items_per_page = items_per_page
        self._to_block = to_block
        if index_fields is None:
            self._index_fields = list()
        else:
            assert isinstance(index_fields, list)
            self._index_fields = index_fields
        try:
            assert any([os.path.exists(file)
                    for file in glob.glob("{}.*".format(self._meta_path))]+[False])
        except AssertionError:
            with shelve_open(self._meta_path) as shelf:
                shelf["filepath"] = self._meta_path
                shelf["total"] = int()
                shelf["next"] = int()
                shelf["groups"] = groups
                shelf["class"] = self.__class__.__name__
                shelf["name"] = self._name
                shelf["ids"] = list()
            for index in self.index_fields:
                if (self._unique_is_id is True and self._unique != index) or self._unique_is_id is False:
                    with shelve_open(self._index_path(index)) as shelf:
                        shelf["filepath"] = self._index_path(index)
            for group in range(0, groups):
                with shelve_open(self._data_path(group)) as shelf:
                    shelf["filepath"] = self._data_path(group)
        self.writer = self._writer()
        with shelve_open(self._meta_path, "r") as shelf:
            self._groups = shelf["groups"]
        time.sleep(0.05)
        self._as_foreign = list()
        self._as_child = list()

    def __len__(self):
        final = None
        while True:
            try:
                with shelve_open(self._meta_path, "r") as meta:
                    final = meta["total"]
            except (KeyboardInterrupt, SystemExit):
                raise
            except:
                continue
            else:
                if final is not None:
                    break
        return final

    def __next__(self): #This is not very appropiate, but...
        final = None
        while True:
            try:
                with shelve_open(self._meta_path, "r") as meta:
                    final = meta["next"]
            except (KeyboardInterrupt, SystemExit):
                    raise
            except:
                continue
            else:
                if final is not None:
                    break
        return final

    @property
    def name(self):
        if self._name == None:
            with shelve_open(self._meta_path, "r") as shelf:
                self._name = shelf["name"]
        return self._name

    @name.setter
    def name(self, value):
        self._wait_to_block(self._meta_path)
        self._keep_alive(self._meta_path)
        if self._name == None:
            with shelve_open(self._meta_path) as shelf:
                shelf["name"] = value
        self._alive = False
        self._name = value

    @property
    def index_fields(self):
        return self._index_fields

    @property
    def indexes_files(self):
        return [self._index_path(index) for index in self.index_fields]

    @property
    def groups(self):
        return self._groups

    @property
    def data_files(self):
        return [self._data_path(index) for index in range(0, self.groups)]

    @property
    def filepath(self):
        return self._filepath
    
    @property
    def headers(self):
        if self._headers is None and self._headers_checked is False:
            try:
                with shelve_open(self._data_path(0), "r") as shelf:
                    self._headers = shelf["headers"]
            except KeyError:
                self._headers = None
            finally:
                self._headers_checked = True
        return self._headers

    @property
    def _meta_path(self):
        return os.path.join(self.filepath, "meta")

    def _index_path(self, field):
        return os.path.join(self.filepath, "index_{}".format(field))

    def _data_path(self, group):
        return os.path.join(self.filepath, "data_{}".format(str(group)))

    def _block(self, file):
        assert file in [self._meta_path, ] + self.indexes_files + self.data_files
        if self.is_blocked(file) is False:
            with open("{}.block".format(file), "w") as block:
                block.write("{}\t{}".format(self.uuid, datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
        else:
            raise BlockedFile

    def _unblock(self, file):
        if self.is_blocked(file) is False:
            try:
                os.remove("{}.block".format(file))
            except (FileNotFoundError, PermissionError):
                pass

    def is_blocked(self, file):
        """
        Checks if given file is blocked
        :param file: file to check.
        :returns: True if blocked, False if not blocked

        """
        blocked = False
        filepath = "{}.block".format(file)
        if os.path.exists(filepath):
            try:
                with open(filepath, "r") as block:
                    uuid, date = block.read().strip("\n").split("\t")
                date = datetime.datetime.strptime(date, "%Y-%m-%d %H:%M:%S")
                now = datetime.datetime.now()
                if uuid != self.uuid and date+datetime.timedelta(seconds=10)>now:
                    blocked = True
            except (FileNotFoundError, ValueError):
                pass
            except (PermissionError):
                blocked = True
        return blocked


    def _wait_to_block(self, file):
        while True:
            if self.is_blocked(file) is True:
                time.sleep(0.5)
            else:
                try:
                    self._block(file)
                except (BlockedFile, PermissionError):
                    continue
                else:
                    time.sleep(0.05)
                    if self.is_blocked(file) is False:
                        self._alive = True
                        break

    @threadize
    def _keep_alive(self, file):
        counter = int()
        while self._alive is True:
            if os.path.exists("{}.block".format(file)):
                if self._alive is True and counter%250==0:
                    try:
                        self._block(file)
                    except (BlockedFile, PermissionError, FileNotFoundError) as e:
                        print(e)
                        self._alive = False
            else:
                self._alive = False
            counter += 1
            time.sleep(0.5)
        self._unblock(file)

    def _send_pipe(self, **kwargs):
        self._pipe_out.send(kwargs)

    def fetch(self, filter, **kwargs):
        """
        Gives the result of a query.
        :param filter: dictionary with wanted coincidences
        :returns: dictionary with result of the query

        """
        filtered = self._filter(filter)
        filter = filtered["filter"]
        filter = self._get_datafile(filter)
        final = list()
        for filename in filter:
            final.extend(self._fetch(filter[filename], filename))
        new_final = list()
        for _id in filtered["filter"]:
            for index, item in enumerate(final):
                if "_id" in item and item["_id"]==_id:
                    if filtered["fields"]:
                        new = dict()
                        for field in item:
                            if field=="_id" or field in filtered["fields"]:
                                new[field] = item[field]
                        new_final.append(new)
                        break
                    else:
                        new_final.append(item)
                        break
            try:
                del(final[index])
            except UnboundLocalError:
                pass
        return ({"data": new_final,
                 "total": filtered["total"],
                 "page": filtered["page"],
                 "items_per_page": filtered["items_per_page"]})

    def _fetch(self, registries, shelf):
        if isinstance(registries, int):
            registries = {registries}
        final = list()
        with shelve_open(shelf, "r") as file:
            for item in registries:
                try:
                    data = file[str(item)]
                except KeyError:
                    data = None
                if isinstance(data, list) and self.headers is not None:
                    if data is not None and len(data)==len(self.headers):
                        data = dict(zip(self.headers, data))
                    else:
                        data = None
                if isinstance(data, dict):
                    data.update({"_id": item})
                if data is not None:
                    final.append(data)
        return final

    def _set_index(self, data, registry):
        if isinstance(data, list) and self.headers is not None and len(data) == len(self.headers):
            data = dict(zip(self.headers, data))
        for field in data:
            if (any([os.path.exists(file)
                    for file in glob.glob("{}.*".format(self._index_path(field)))]+[False]) and
                    (self.is_blocked(self._index_path(field)) is False or self._to_block is False)):
                with shelve_open(self._index_path(field)) as shelf:
                    index = str(data[field])
                    last = shelf
                    if not index in shelf:
                        if field != self._unique or self._split_unique == 0:
                            shelf[str(index)] = set()
                        elif self._unique_is_id is False:
                            offset = len(str(index))%self._split_unique
                            if offset:
                                inter = str(index)[0:offset]
                                if inter not in last:
                                    last[inter] = dict()
                                last = last[inter]
                            for x in range(ceil(len(str(index))/self._unique)):
                                inter = str(index)[offset+x*self._split_unique:offset+(x+1)*self._split_unique]
                                if inter not in last:
                                    last[inter] = dict()
                                last = last[inter]
                            last = registry
                    if field != self._unique:
                        shelf[str(index)] |= {registry}
                    else:
                        last = registry
                    
    def _del_index(self, data, registry):
        for field in data:
            if (any([os.path.exists(file)
                    for file in glob.glob("{}.*".format(self._index_path(field)))]+[False]) and
                    self.is_blocked(self._index_path(field)) is False):
                with shelve_open(self._index_path(field)) as shelf:
                    index = str(data[field])
                    if index in shelf:
                        shelf[index] -= {registry}

    def _check_child(self, data):
        if self._as_child:
            for foreign_key in self._as_child:
                if not foreign_key.field in data:
                    return 1
                else:
                    foreign = foreign_key.foreign.fetch({"_id": data[foreign_key.field]}) # Change to header
                    if not foreign:
                        return 2
        return 0

    def insert(self, data, **kwargs):
        """
        Loads new given data in the database
        Blocks until finnish
        :param data: list with a dictionary for each item to upload
        :returns: New Data
        """
        conn_in, conn_out = Pipe(False)
        if self._unique is not None:
            return {}
        self._send_pipe(action="insert", data=data, pipe=conn_out)
        recv = conn_in.recv()
        return recv

    def _insert(self, data, filename_reg):
        for filename in filename_reg:
            with shelve_open(filename) as shelf:
                for index in filename_reg[filename]:
                    new_data = data[str(index)]
                    if self.headers is not None:
                        new_data = list()
                        for header in self.headers:
                            try:
                                new_data.append(data[str(index)][header])
                            except KeyError:
                                new_data.append("")
                    shelf[str(index)] = new_data
        for index_name in self.index_fields:
             index_dict = dict()
             for index in data:
                 if str(data[index][index_name]) not in index_dict:
                     index_dict[str(data[index][index_name])] = set()
                 index_dict[str(data[index][index_name])].add(int(index))
             with shelve_open(self._index_path(index_name)) as shelf:
                 for index in index_dict:
                     if index not in shelf:
                         shelf[index] = index_dict[index]
                     else:
                         shelf[index] |= index_dict[index]
        with shelve_open(self._meta_path) as shelf:
             total, next_ = len(self), next(self)
             shelf["total"] = total + len(data)
             shelf["next"] = next_ + len(data)
             shelf["ids"] = set([int(key) for key in list(data.keys())])

    def new(self, data, **kwargs): #TODO: Errors setting new data
        """
        Set new given data in the database
        Blocks until finnish
        :param data: dictionary with given data. Saved as is if self.headers is None
        :returns: New Data

        """
        if self._check_child(data) != 0:
            return None
        conn_in, conn_out = Pipe(False)
        test = None
        if self._unique in data:
            test = self.fetch({self._unique: data[self._unique]})
        if test and "total" in test and test["total"] > 0:
            self._send_pipe(action="replace", data=data, filter={self._unique: data[self._unique]}, pipe=conn_out)
        else:
            self._send_pipe(action="new", data=data, pipe=conn_out)
        recv = conn_in.recv()
        return recv

    def _new(self, data, registry, shelf):
        with shelve_open(shelf) as file:
            if self.headers is not None:
                new_data = list()
                for header in self.headers:
                    try:
                        new_data.append(data[header])
                    except KeyError:
                        new_data.append("")
                data = new_data
            file[str(registry)] = data
        with shelve_open(self._meta_path) as file:
            total, next_ = len(self), next(self) #Bug!
            file["total"] = total + 1
            file["next"] = next_ + 1
            ids = list(file["ids"])
            ids.append(str(registry))
            file["ids"] = ids
        self._set_index(data, registry)

    def replace(self, filter, data, **kwargs):
        """
        Replaces all data which coincides with given filter with given data
        Blocks untill finnish
        :param filter: dictionary with coincidences
        :param data: dictionary with new data. It can be partial.
        :returns: Data replaced

        """
        if self._check_child(data) != 0:
            return None
        conn_in, conn_out = Pipe(False)
        test = None
        if ((self._unique in data and self._unique not in filter) or
            (self._unique in data and self._unique in filter and data[self._unique]!=filter[self._unique])):
            test = self.fetch({self._unique: data[self._unique]})
        if not test:
            self._send_pipe(action="replace", filter=filter, data=data, pipe=conn_out)
        else:
            return {"Error": "400"}
        return conn_in.recv()

    def _replace(self, data, registries, shelf):
        with shelve_open(shelf) as file:
            for reg in registries:
                try:
                    old_data = self._fetch({reg}, shelf)[0]
                except IndexError:
                    continue
                else:
                    if isinstance(old_data, dict): # Verified twice. It has to be a dict
                        if "_id" in old_data:
                            del(old_data["_id"])
                        new_data = old_data.copy()
                        new_data.update(data)
                        self._del_index(old_data, reg)
                        if self.headers is not None:
                            new_data = [new_data[item] for item in self.headers]
                        file[str(reg)] = new_data
                        self._set_index(new_data, reg)

    def edit(self, filter, data, **kwargs):
        """
        replace alias

        """
        if self._check_child(data) == 2:
            return None
        conn_in, conn_out = Pipe(False)
        test = None
        if ((self._unique in data and self._unique not in filter) or
                (self._unique in data and self._unique in filter and data[self._unique] != filter[self._unique])):
            test = self.fetch({self._unique: data[self._unique]})
        if not test:
            self._send_pipe(action="edit", filter=filter, data=data, pipe=conn_out)
        else:
            return {"Error": "400"}
        return conn_in.recv()

    def _edit(self, data, registries, shelf):
        self._replace(data, registries, shelf)

    def drop(self, filter, **kwargs):
        """
        Deletes data from database which coincides with given filter
        Blocks untill finnish
        :param filter: dictionary with given filter
        :returns: Data
        """
        conn_in, conn_out = Pipe(False)
        self._send_pipe(action="drop", filter=filter, data={}, pipe=conn_out)
        return conn_in.recv()

    def _drop(self, data, registries, shelf):
        with shelve_open(shelf) as file:
            for reg in registries:
                try:
                    old_data = self._fetch({reg}, shelf)
                except KeyError:
                    continue
                else:
                    if self._as_foreign:
                        for item in self._as_foreign:
                            children = item.children.fetch({item.field: reg})
                            if item:
                                continue
                    if old_data != list():
                        self._del_index(old_data, reg)
                        del(file[str(reg)])
                        with shelve_open(self._meta_path) as file:
                            file["total"] -= 1
                            ids = list(file["ids"])
                            del(ids[ids.index(str(reg))])
                            file["ids"] = ids

    def _filter(self, filter):
        with shelve_open(self._meta_path) as shelf:
            ids = list(shelf["ids"])
            final_set = set([int(id) for id in ids])
        #final_set = set(range(0, next(self)))
        order = str()
        fields = list()
        page = 1
        items_per_page = self.items_per_page
        if self._unique_is_id and self._unique in filter:
            filter["_id"] = filter[self._unique]
            del(filter[self._unique])
        if "order" in filter:
            order = filter["order"]
            order = order.split(",")
        if "page" in filter:
            page = filter["page"]
        if "items_per_page" in filter:
            items_per_page = filter["items_per_page"]
        if "fields" in filter:
            fields = filter["fields"].split(",")
        sub_order = dict()
        final_order = list()
        for field in filter:
            subfilter = set()
            if field == "_id" and filter[field] != "":
                subfilter = {int(filter["_id"])}
            elif field == "_id" and filter[field] == "":
                subfilter = final_set
            else:
                if any([os.path.exists(file)
                        for file in glob.glob("{}.*".format(self._index_path(field)))]+[False]):
                    with shelve_open(self._index_path(field), "r") as index:
                        if self._unique != field or self._split_unique == 0:
                            if str(filter[field]) in index:
                                subfilter = index[str(filter[field])]
                        else:
                            offset = len(str(index))%self._split_unique
                            last = index
                            if offset:
                                inter = str(index)[0:offset]
                                last = last[inter]
                            for x in range(ceil(len(str(index))/self._split_unique)):
                                inter = str(index)[offset+x*self._split_unique:offset+(x+1)*self._split_unique]
                                last = last[inter]
                            subfilter = {last}
            final_set &= subfilter
        final_set = list(final_set)
        final_set.sort()
        if len(order) > 0:
            for _id in final_set:
                field = order[0] #TODO: Accept many fields
                if field.startswith("-"):
                    sfield = field[1:]
                else:
                    sfield = field
                if any([os.path.exists(file)
                        for file in glob.glob("{}.*".format(self._index_path(sfield)))] + [False]):
                    with shelve_open(self._index_path(sfield), "r") as index:
                        sub_order[sfield] = index.copy()
                    keys = list(sub_order.keys())
                    if field.startswith("-"):
                        keys.reverse()
                    else:
                        keys.sort()
                    for indexes in keys:
                        for key in indexes:
                            if key in final_set:
                                final_order.append(key)
        else:
            final_order = final_set
        return {"filter": final_order[int(items_per_page)*(int(page)-1):int(items_per_page)*int(page)],
                "total": len(final_order),
                "page": page,
                "items_per_page": items_per_page,
                "fields": fields}

    def _get_datafile(self, filter):
        assert isinstance(filter, list)
        filename_reg = dict()
        for reg in filter:
            filename = self._data_path(reg % self.groups)
            if filename not in filename_reg:
                filename_reg[filename] = set()
            filename_reg[filename] |= {reg}
        return filename_reg

    @threadize
    def _writer(self):
        """
        It may receive by self._pipe_out a dictionary with:
        action: new, replace, drop or edit
        filter: if not new, a set of registries
        data: dictionary with the new data
        """
        new = int()
        while True:
{'fraction_non_alphanumeric': 0.09505433926602615, 'fraction_numerical': 0.0022838242242872895, 'mean_word_length': 1.4093065174082156, 'pattern_counts': {'":': 30, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 38, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4990101', 'n_tokens_mistral': 13497, 'n_tokens_neox': 12893, 'n_words': 3524}