<commit_before># coding=utf-8

"""
A bunch of classes for working with data. These are to be used in your
plugins. There's several different ones to choose from, each will store
data using a different format and sometimes method. There are, so far, two
types of data storage, encapsulating four classes.

* Key-value  (Dictionary-like)
  * YAML-format with dict-like access and thread safety
  * JSON-format with dict-like access and thread safety
  * In-memory dict-like storage with tread-safety. This is for plugins to
    insert where they need to change the data object of something for some
    reason.
* Relational (SQL)
  * SQLite-format with cursors and thread safety. Also supports in-memory
    storage by specifying ":memory:" as the filename.
"""

__author__ = "<NAME>"

import json
import os
import sqlite3
import yaml

from threading import Lock
from twisted.enterprise import adbapi

from utils.log import getLogger


class Data(object):
    """
    Base class for data storage objects, mostly for type-checking.
    """
    editable = False


class YamlData(Data):
    """
    Data object that uses YAML files for storage.

    This is a standard Data object, its base type is a dictionary, so don't
    try to store anything else in this.

    The correct way to use this is with the `with` macro. This ensures both
    thread-safety and that any data edits will be saved when the block has been
    exited.

    If you're not writing any data, then you can just use this object in the
    same way you'd use a dict, but this will not guarantee thread-safety.

    Example:

    with data:
        data["x"]["y"] = "z"
        thing = data["a"]
        # Some other stuff

    This object uses dict-like access methods, including iteration, `keys`
    and `values` methods. Use it how you would a dict. Additionally, the
    following methods are supported..

    Normal dict getters and setters, length, contains, and deletion methods.
    `save()` - Force a flush to file.
    `load()` - Force a reload from file.

    Note: Data access is "jailed" - you can't load a file from outside the data
    directory.

    For sanity's sake, all YAML files should end in .yml - but this is not
    enforced.
    """

    editable = True

    data = {}

    mutex = Lock()
    context_mutex = Lock()
    _context_guarded = False

    def __init__(self, filename):
        self.logger = getLogger("Data")
        filename = filename.strip("..")

        folders = filename.split("/")
        folders.pop()
        folders = "/".join(folders)

        if not os.path.exists(folders):
            os.makedirs(folders)

        self.filename = filename
        self.load()

    def load(self):
        """
        Load or reload data from the filesystem.
        """
        if not self._context_guarded:
            with self.mutex:
                self._load()
        else:
            self._load()

    reload = load

    def _load(self):
        if not os.path.exists(self.filename):
            open(self.filename, "w").close()
        fh = open(self.filename, "r")
        self.data = yaml.load(fh)
        fh.close()
        if not self.data:
            self.data = {}

    def save(self):
        """
        Save data to the filesystem.
        """
        if not self._context_guarded:
            with self.mutex:
                self._save()
        else:
            self._save()

    def _save(self):
        data = yaml.dump(self.data)
        fh = open(self.filename, "w")
        fh.write(data)
        fh.flush()
        fh.close()

    def keys(self):
        return self.data.keys()

    def items(self):
        return self.data.items()

    def values(self):<commit_msg>YAML data storage dumps in a more user-friendly flow style.<commit_after># coding=utf-8

"""
A bunch of classes for working with data. These are to be used in your
plugins. There's several different ones to choose from, each will store
data using a different format and sometimes method. There are, so far, two
types of data storage, encapsulating four classes.

* Key-value  (Dictionary-like)
  * YAML-format with dict-like access and thread safety
  * JSON-format with dict-like access and thread safety
  * In-memory dict-like storage with tread-safety. This is for plugins to
    insert where they need to change the data object of something for some
    reason.
* Relational (SQL)
  * SQLite-format with cursors and thread safety. Also supports in-memory
    storage by specifying ":memory:" as the filename.
"""

__author__ = "<NAME>"

import json
import os
import sqlite3
import yaml

from threading import Lock
from twisted.enterprise import adbapi

from utils.log import getLogger


class Data(object):
    """
    Base class for data storage objects, mostly for type-checking.
    """
    editable = False


class YamlData(Data):
    """
    Data object that uses YAML files for storage.

    This is a standard Data object, its base type is a dictionary, so don't
    try to store anything else in this.

    The correct way to use this is with the `with` macro. This ensures both
    thread-safety and that any data edits will be saved when the block has been
    exited.

    If you're not writing any data, then you can just use this object in the
    same way you'd use a dict, but this will not guarantee thread-safety.

    Example:

    with data:
        data["x"]["y"] = "z"
        thing = data["a"]
        # Some other stuff

    This object uses dict-like access methods, including iteration, `keys`
    and `values` methods. Use it how you would a dict. Additionally, the
    following methods are supported..

    Normal dict getters and setters, length, contains, and deletion methods.
    `save()` - Force a flush to file.
    `load()` - Force a reload from file.

    Note: Data access is "jailed" - you can't load a file from outside the data
    directory.

    For sanity's sake, all YAML files should end in .yml - but this is not
    enforced.
    """

    editable = True

    data = {}

    mutex = Lock()
    context_mutex = Lock()
    _context_guarded = False

    def __init__(self, filename):
        self.logger = getLogger("Data")
        filename = filename.strip("..")

        folders = filename.split("/")
        folders.pop()
        folders = "/".join(folders)

        if not os.path.exists(folders):
            os.makedirs(folders)

        self.filename = filename
        self.load()

    def load(self):
        """
        Load or reload data from the filesystem.
        """
        if not self._context_guarded:
            with self.mutex:
                self._load()
        else:
            self._load()

    reload = load

    def _load(self):
        if not os.path.exists(self.filename):
            open(self.filename, "w").close()
        fh = open(self.filename, "r")
        self.data = yaml.load(fh)
        fh.close()
        if not self.data:
            self.data = {}

    def save(self):
        """
        Save data to the filesystem.
        """
        if not self._context_guarded:
            with self.mutex:
                self._save()
        else:
            self._save()

    def _save(self):
        data = yaml.dump(self.data, default_flow_style=False)
        fh = open(self.filename, "w")
        fh.write(data)
        fh.flush()
        fh.close()

    def keys(self):
        return self.data.keys()

    def items(self):
        return self.data.items()

    def values(self):
{'fraction_non_alphanumeric': 0.09017928819909019, 'fraction_numerical': 0.0005351886540005352, 'mean_word_length': 2.2542446669569003, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1321827', 'n_tokens_mistral': 2183, 'n_tokens_neox': 2051, 'n_words': 837}