<commit_before>import pandas as pd
import os
import numpy as np
from warnings import warn
import time

class SciCatalog:
    """
    A class of objects intended to store, modify, and display tabular scientific data with positive and negative errors
    and references.

    Catalogs are stored as a directory containing four (at present) csv files: one each for the values,
    positive errors, negative errors, and references.

    Written by <NAME>, 2015/07/08

    I'm hoping to eventually expand this with methods to quickly make pretty LaTeX tables and such and maybe
    accomodate multiple values or references for a single entry (though that would be difficult).
    """

    keys = ['value', 'errpos', 'errneg', 'ref']
    tableFiles = ['values', 'errors_positive', 'errors_negative', 'references']
    fileSuffix = 'csv'
    nullValues = [np.nan, np.nan, np.nan, 'none']
    refDictFile = 'reference_dictionary'
    refFileSuffix = 'txt'

    def __init__(self, path, values=None, errpos=None, errneg=None, refs=None, refDict={}, index=None, columns=None):
        """
        Creates an empty SciCatalog object by intializing the four pandas DataFrame tables and a reference dictionary
        that are kept synced to the disk as changes are made.

        Parameters
        ----------
        path : string
            Path of the directory in which the table files exist or are to be created.
        values, errpos, errneg, refs :
            Data to be put into DataFrames. Each will be used as the `data` keyword argument to instantiate a
            pandas DataFrame. Can be lots of different data types (array, dictionary, DataFrame,
            ... see pandas.DataFrame docs)
        refDict : dict
            A dictionary defining the various reference keys used. This is intended both for brevity in the reference
            table and possibly later use with BibTex... not sure if that will ever come to fruition in a useful way.

        Other **kwargs will be passed along to DataFrame call. See DataFrame documentation for info, but important
        ones include index and columns for providing lists of indices and column names.

        Returns
        -------
        SciCatalog object.

        """

        if os.path.exists(path) and values is not None:
            raise Exception('A directory named {} already exists at {}. You must use a different name or manually '
                            'delete that directory before you an create the catalog you want at that disk location. '
                            ''.format(*os.path.split(path)))

        # store auxilliary data
        self.path = path
        self.paths  = self._tablepaths(path)
        self.archive = os.path.join(path, 'archive')
        self.refDict = refDict
        self.refDictPath = os.path.join(path, self.refDictFile) + '.' + self.refFileSuffix

        # either load or create the SciCat table as appropriate
        if os.path.exists(path):
            self.tables = [pd.DataFrame.from_csv(p) for p in self.paths]
            self.values, self.errpos, self.errneg, self.refs = self.tables

            refs, defs = [], []
            with open(self.refDictPath) as f:
                lines = f.read().splitlines()
                pairs = [line.split(' : ') for line in lines]
            self.refDict = dict(pairs)

            # make a backup copy
            if 'archive' not in path:
                self.backup()

        else:
            # functions to create DataFrames filled with null or good data as appropriate
            nullDF = lambda val : self._fillDF(val, columns, index)
            goodDF = lambda data: pd.DataFrame(data=data, columns=columns, index=index)
            DF = lambda data, val: nullDF(val) if data is None else goodDF(data)

            # create tables
            self.tables = map(DF, [values, errpos, errneg, refs], self.nullValues)
            self.values, self.errpos, self.errneg, self.refs = self.tables

            # check that all reference keys are defined
            for ref in self.refs.values.ravel():
                self.checkRef(ref)

            # write to disk
            if not os.path.exists(path):
                os.mkdir(path)
            if not os.path.exists(self.archive):
                os.mkdir(self.archive)

            self.save()


    def set(self, index, col, value=None, errpos=None, errneg=None, ref=None):
        """
        Set the value of an item in the catalog in-place, using null values for any keywords with None values and save
        to disk.

        Parameters
        ----------
        index, col: str
            Index and column locating the item in the catalog (e.g. 'alpha cen', 'distance'). Either index or column
            can be an iterable to allow setting values for the multiple rows in the same column or multiple columns
            of the same row at once, but not both!
        value, errpos, errneg : flt
            Value and errors (positive and negative) for the item.
        ref : str
            Reference key for the item.

        Returns
        -------
        None
        """
        data = [value, errpos, errneg, ref]

        iterI, iterC = [hasattr(x, '__iter__') for x in [index, col]]

        if iterI and iterC:
            raise TypeError('Only one of index and col can be iterable.')

        def groomLen(n):
            for i in range(len(data)):
                if data[i] is None:
                    data[i] = [data[i]]*n
                elif len(data[i]) != n:
                    raise ValueError('Given your input for index and col, value, errpos, errneg, and ref must all be '
                                     'iterables of length {} or be None.'.format(n))

        if iterI:
            groomLen(len(index))
            for i, v, ep, en, r in zip(index, *data):
                self._setSingle(i, col, v, ep, en, r)
        elif iterC:
            groomLen(len(col))
            for c, v, ep, en, r in zip(col, *data):
                self._setSingle(index, c, v, ep, en, r)
        else:
            self._setSingle(index, col, *data)

        self.save()

<commit_msg>ahhh! fixed so multiple users can't edit<commit_after>import pandas as pd
import os
import numpy as np
from warnings import warn
import time
import getpass

class SciCatalog:
    """
    A class of objects intended to store, modify, and display tabular scientific data with positive and negative errors
    and references.

    Catalogs are stored as a directory containing four (at present) csv files: one each for the values,
    positive errors, negative errors, and references.

    Written by <NAME>, 2015/07/08

    I'm hoping to eventually expand this with methods to quickly make pretty LaTeX tables and such and maybe
    accomodate multiple values or references for a single entry (though that would be difficult).
    """

    keys = ['value', 'errpos', 'errneg', 'ref']
    tableFiles = ['values', 'errors_positive', 'errors_negative', 'references']
    fileSuffix = 'csv'
    nullValues = [np.nan, np.nan, np.nan, 'none']
    refDictFile = 'reference_dictionary'
    refFileSuffix = 'txt'
    accessFile = 'user_accessing.txt'

    def __init__(self, path, values=None, errpos=None, errneg=None, refs=None, refDict={}, index=None, columns=None):
        """
        Creates an empty SciCatalog object by intializing the four pandas DataFrame tables and a reference dictionary
        that are kept synced to the disk as changes are made.

        Parameters
        ----------
        path : string
            Path of the directory in which the table files exist or are to be created.
        values, errpos, errneg, refs :
            Data to be put into DataFrames. Each will be used as the `data` keyword argument to instantiate a
            pandas DataFrame. Can be lots of different data types (array, dictionary, DataFrame,
            ... see pandas.DataFrame docs)
        refDict : dict
            A dictionary defining the various reference keys used. This is intended both for brevity in the reference
            table and possibly later use with BibTex... not sure if that will ever come to fruition in a useful way.

        Other **kwargs will be passed along to DataFrame call. See DataFrame documentation for info, but important
        ones include index and columns for providing lists of indices and column names.

        Returns
        -------
        SciCatalog object.

        """

        if os.path.exists(path) and values is not None:
            raise Exception('A directory named {} already exists at {}. You must use a different name or manually '
                            'delete that directory before you an create the catalog you want at that disk location. '
                            ''.format(*os.path.split(path)))

        # store auxilliary data
        self.path = path
        self.name = os.path.basename(path)
        self.paths  = self._tablepaths(path)
        self.archive = os.path.join(path, 'archive')
        self.refDict = refDict
        self.refDictPath = os.path.join(path, self.refDictFile) + '.' + self.refFileSuffix

        # either load or create the SciCat table as appropriate
        if os.path.exists(path):
            # try to prevent possible editing by multiple users at the same time by looking for or creating a file
            # that just contains the name of the user, to be later removed with the close() method.
            self._accessPath = os.path.join(path, self.accessFile)
            if os.path.exists(self._accessPath):
                with open(self._accessPath) as f:
                    user = f.readline().strip()
                raise Exception('Cannot load the catalog because it is currently in use by {}'.format(user))
            else:
                print("IMPORTANT: You MUST use execute the command '{c}.close()' or 'del {c}' when you are done "
                      "modifying the {c} catalog or other users will not be able to open and edit it."
                      "".format(c=self.name))
                with open(self._accessPath, 'w') as f:
                    f.write(getpass.getuser())

            # load in the table data
            self.tables = [pd.DataFrame.from_csv(p) for p in self.paths]
            self.values, self.errpos, self.errneg, self.refs = self.tables

            # load in the refernce dictionary
            refs, defs = [], []
            with open(self.refDictPath) as f:
                lines = f.read().splitlines()
                pairs = [line.split(' : ') for line in lines]
            self.refDict = dict(pairs)

            # make a backup copy
            if 'archive' not in path:
                self.backup()

        else:
            # functions to create DataFrames filled with null or good data as appropriate
            nullDF = lambda val : self._fillDF(val, columns, index)
            goodDF = lambda data: pd.DataFrame(data=data, columns=columns, index=index)
            DF = lambda data, val: nullDF(val) if data is None else goodDF(data)

            # create tables
            self.tables = map(DF, [values, errpos, errneg, refs], self.nullValues)
            self.values, self.errpos, self.errneg, self.refs = self.tables

            # check that all reference keys are defined
            for ref in self.refs.values.ravel():
                self.checkRef(ref)

            # write to disk
            if not os.path.exists(path):
                os.mkdir(path)
            if not os.path.exists(self.archive):
                os.mkdir(self.archive)

            self.save()


    def set(self, index, col, value=None, errpos=None, errneg=None, ref=None):
        """
        Set the value of an item in the catalog in-place, using null values for any keywords with None values and save
        to disk.

        Parameters
        ----------
        index, col: str
            Index and column locating the item in the catalog (e.g. 'alpha cen', 'distance'). Either index or column
            can be an iterable to allow setting values for the multiple rows in the same column or multiple columns
            of the same row at once, but not both!
        value, errpos, errneg : flt
            Value and errors (positive and negative) for the item.
        ref : str
            Reference key for the item.

        Returns
        -------
        None
        """
        data = [value, errpos, errneg, ref]

        iterI, iterC = [hasattr(x, '__iter__') for x in [index, col]]

        if iterI and iterC:
            raise TypeError('Only one of index and col can be iterable.')

        def groomLen(n):
            for i in range(len(data)):
                if data[i] is None:
                    data[i] = [data[i]]*n
                elif len(data[i]) != n:
                    raise ValueError('Given your input for index and col, value, errpos, errneg, and ref must all be '
                                     'iterables of length {} or be None.'.format(n))

        if iterI:
            groomLen(len(index))
            for i, v, ep, en, r in zip(index, *data):
                self._setSingle(i, col, v, ep, en, r)
        elif iterC:
            groomLen(len(col))
            for c, v, ep, en, r in zip(col, *data):
                self._setSingle(index, c, v, ep, en, r)
        else:
            self._setSingle(index, col, *data)

        self.save()


    def __del__(self):
        self.close()
        del self


    def __exit__(self):
        self.close()


    def close(self):
        """
        Close the catalog, making it available for other users to open and edit.
        """
        os.remove(self._accessPath)


{'fraction_non_alphanumeric': 0.08728416739830261, 'fraction_numerical': 0.001170617500731636, 'mean_word_length': 1.9427341227125943, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '292651', 'n_tokens_mistral': 3679, 'n_tokens_neox': 3535, 'n_words': 1474}