<commit_before># -*- coding: utf-8 -*-
"""
An incredibly basic, not very flexible database for keeping track of all
the initial conditions that I create.  This is not user friendly, but mostly
for my personal use

EXAMPLE

import icdb

# Create a new database looking at 2 base directories (and all subdirectories)

# Finds all files matching '*_settings.p'

db = icdb.database('*_settings.p', '~/simulations', \
'/home/ibackus/hyak/gscratch/vsm/ibackus/simulations')

# Query the full paths to IC files:

db.data('filename')

# Query Qmin

db.data('sigma.Qmin')

# Access the settings/information for the first simulation

sim_settings = db.data[0]

# Refresh database

db.refresh()

Created on Mon Sep 15 16:06:30 2014

@author: ibackus
"""
import numpy as np
import ICgen_settings
import os
import fnmatch

class database():
    
    """
    IC database class.  Initialize with:
    
    db = icdb.database(filefilter, dir1, dir2, ...)
    
    filefilter is a string used to filter IC settings files, ie '*_settings.p'
    dir1, dir2 are base directories which are recursively searched to find
    settings files
    """
    
    def __init__(self, filefilter='IC_settings.p', *directories, **kwargs):
        
        print 'Building database...'
        if len(directories) == 0:
            
            directories = ['.']
        print 'Number of base directories: ', len(directories)
        self.dirs = directories
        self.filefilter = filefilter
        self._kwargs = kwargs
        
        matches = []
        
        for directory in directories:
            
            print 'Building for ', directory            
            new_matches = self._scan_dir(directory, filefilter)
            print '    Found {} files'.format(len(new_matches))
            matches += new_matches

        filelist = list(set(matches))
        datalist = []
        
        for i, fname in enumerate(filelist):
            
            a = ICgen_settings.settings()
            a.load(fname)
            a_dir = os.path.split(fname)[0]
            ICname = a.filenames.IC_file_name
            ICname = os.path.join(a_dir, ICname)
            ICname = os.path.realpath(ICname)
            a.filename = ICname
            
            if os.path.exists(ICname):
                
                datalist.append(a)
            
        nfiles = len(datalist)
        print 'Found {} files in total'.format(nfiles)
        data = fancy_array(nfiles)
        data[:] = datalist
        self.data = data
        
    def nfiles(self):
        """
        Returns the current number of files in the database
        """<commit_msg>Simple database to keep track of ICs that have been generated<commit_after># -*- coding: utf-8 -*-
"""
An incredibly basic, not very flexible database for keeping track of all
the initial conditions that I create.  This is not user friendly, but mostly
for my personal use

EXAMPLE

import icdb

# Create a new database looking at 2 base directories (and all subdirectories)

# Finds all files matching '*_settings.p'

db = icdb.database('*_settings.p', '~/simulations', \
'/home/ibackus/hyak/gscratch/vsm/ibackus/simulations')

# Query the full paths to IC files:

db.data('filename')

# Query Qmin

db.data('sigma.Qmin')

# Access the settings/information for the first simulation

sim_settings = db.data[0]

# Refresh database

db.refresh()

Created on Mon Sep 15 16:06:30 2014

@author: ibackus
"""
import numpy as np
import ICgen_settings
import os
import fnmatch

class database():
    
    """
    IC database class.  Initialize with:
    
    db = icdb.database(filefilter, dir1, dir2, ...)
    
    filefilter is a string used to filter IC settings files, ie '*_settings.p'
    dir1, dir2 are base directories which are recursively searched to find
    settings files
    """
    
    def __init__(self, filefilter='IC_settings.p', *directories, **kwargs):
        
        print 'Building database...'
        if len(directories) == 0:
            
            directories = ['.']
        print 'Number of base directories: ', len(directories)
        self.dirs = directories
        self.filefilter = filefilter
        self._kwargs = kwargs
        
        matches = []
        
        for directory in directories:
            
            print 'Building for ', directory            
            new_matches = self._scan_dir(directory, filefilter)
            print '    Found {} files'.format(len(new_matches))
            matches += new_matches

        filelist = list(set(matches))
        datalist = []
        
        for i, fname in enumerate(filelist):
            
            a = ICgen_settings.settings()
            a.load(fname)
            a_dir = os.path.split(fname)[0]
            ICname = a.filenames.IC_file_name
            ICname = os.path.join(a_dir, ICname)
            ICname = os.path.realpath(ICname)
            
            if os.path.exists(ICname):
                
                a.dir = os.path.realpath(a_dir)
                datalist.append(a)
            
        nfiles = len(datalist)
        print 'Found {} files in total'.format(nfiles)
        data = fancy_array(nfiles)
        data[:] = datalist
        self.data = data
        
    def nfiles(self):
        """
        Returns the current number of files in the database
        """
{'fraction_non_alphanumeric': 0.0945945945945946, 'fraction_numerical': 0.007882882882882882, 'mean_word_length': 2.0434037692747, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6896045', 'n_tokens_mistral': 1564, 'n_tokens_neox': 1510, 'n_words': 458}