<commit_before>#!/usr/bin/python
from __future__ import print_function,division
import sqlite3 as lite
import os as os
import time as time
import sys

class Camoco(object):
    def __init__(self,basedir="~/.camoco"):
        # Set up our base directory
        self.basedir = os.path.realpath(os.path.expanduser(basedir))
        if not os.path.exists(self.basedir):
            # If it doesn't exists, set up first time directories
            try:    
                os.mkdir(self.basedir)
                os.mkdir(os.path.join(self.basedir,"logs"))
                os.mkdir(os.path.join(self.basedir,"databases"))
                os.mkdir(os.path.join(self.basedir,"analyses"))
            except Exception as e:
                print("[CAMOCO]",time.ctime(), '-', *args,file=sys.stderr)
        self.log_file = open(os.path.join(self.basedir,"logs","logfile.txt"),'w')


    def log(self,msg,*args):
        print("[CAMOCO]",time.ctime(), '-', msg.format(*args),file=sys.stderr)
        print("[CAMOCO]",time.ctime(), '-', msg.format(*args),file=self.log_file)
           
    def _resource(self,type,filename):
        return os.path.join(self.basedir,type,filename)

    def database(self,dbname):
        # return a connection if exists
        return lite.connect(self._resource("databases",str(dbname)+".db"))

    def available_datasets(self):
        cur=self.database("camoco").cursor()
        return cur.execute("SELECT rowid,* FROM datasets;").fetchall()
        
               

    def add_dataset(self,name,description='',type=''):
        con = self.database("camoco")
        cur = con.cursor()
        try:
            # update information table
            cur.execute('''
                INSERT INTO datasets (name,description,type)
                VALUES ('{}','{}','{}')'''.format(name,description,type))
            con.commit()
            # create new sqlite file for dataset
            self.database(".".join([type,name]))
        except lite.IntegrityError as e:
            self.log("{} already added to database.",name)

    def del_dataset(self,name,type):
        con = self.database('camoco')
        con.execute(''' DELETE FROM datasets WHERE name = '{}' and type = '{}';'''.format(name,type))
        con.commit() 
        os.remove(self._resource("databases",".".join([type,name])+".db"))
<commit_msg>added tmp_file method to camoco class<commit_after>#!/usr/bin/python
from __future__ import print_function,division
import sqlite3 as lite
import os as os
import time as time
import sys
import tempfile

class Camoco(object):
    def __init__(self,basedir="~/.camoco"):
        # Set up our base directory
        self.basedir = os.path.realpath(os.path.expanduser(basedir))
        if not os.path.exists(self.basedir):
            # If it doesn't exists, set up first time directories
            try:    
                os.mkdir(self.basedir)
                os.mkdir(os.path.join(self.basedir,"logs"))
                os.mkdir(os.path.join(self.basedir,"databases"))
                os.mkdir(os.path.join(self.basedir,"analyses"))
                os.mkdir(os.path.join(self.basedir,"tmp"))
            except Exception as e:
                print("[CAMOCO]",time.ctime(), '-', *args,file=sys.stderr)
        self.log_file = open(os.path.join(self.basedir,"logs","logfile.txt"),'w')


    def log(self,msg,*args):
        print("[CAMOCO]",time.ctime(), '-', msg.format(*args),file=sys.stderr)
        print("[CAMOCO]",time.ctime(), '-', msg.format(*args),file=self.log_file)
           
    def _resource(self,type,filename):
        return os.path.join(self.basedir,type,filename)

    def database(self,dbname):
        # return a connection if exists
        return lite.connect(self._resource("databases",str(dbname)+".db"))
    def tmp_file(self):
        # returns a handle to a tmp file
        return tempfile.NamedTemporaryFile(dir=os.path.join(self.basedir,"tmp"))
        

    def available_datasets(self):
        cur=self.database("camoco").cursor()
        return cur.execute("SELECT rowid,* FROM datasets;").fetchall()
    

    def add_dataset(self,name,description='',type=''):
        con = self.database("camoco")
        cur = con.cursor()
        try:
            # update information table
            cur.execute('''
                INSERT INTO datasets (name,description,type)
                VALUES ('{}','{}','{}')'''.format(name,description,type))
            con.commit()
            # create new sqlite file for dataset
            self.database(".".join([type,name]))
        except lite.IntegrityError as e:
            self.log("{} already added to database.",name)

    def del_dataset(self,name,type):
        con = self.database('camoco')
        con.execute(''' DELETE FROM datasets WHERE name = '{}' and type = '{}';'''.format(name,type))
        con.commit() 
        os.remove(self._resource("databases",".".join([type,name])+".db"))

{'fraction_non_alphanumeric': 0.1555510370069134, 'fraction_numerical': 0.00040666937779585197, 'mean_word_length': 2.533764367816092, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '174500', 'n_tokens_mistral': 1498, 'n_tokens_neox': 1444, 'n_words': 281}