<commit_before>#!/usr/bin/env python
import logging
from collections import defaultdict
from errno import ENOENT
from stat import S_IFDIR, S_IFLNK, S_IFREG
from sys import argv, exit
from time import time
from time import time
import datetime
from fuse import FUSE, FuseOSError, Operations, LoggingMixIn
from xmlrpclib import Binary
import sys, pickle, xmlrpclib

count = 0

from fuse import FUSE, FuseOSError, Operations, LoggingMixIn
from pymongo import MongoClient

import collections
import timeit

if not hasattr(__builtins__, 'bytes'):
    bytes = str



## DEFINE CACHE SIZE
CACHE_SIZE = 10

###
##LRU CLASS 


class lruCache:
    
    def __init__(self, size):
        self.size = CACHE_SIZE
        # LRU Cache is imlemented using Ordered Dictionaries
        self.cache = collections.OrderedDict()
    
    def get(self,key):
        ## get the f
        print "get from cache -- \n", key
        print "CACHE CONTENTS\n",self.cache.keys()
        try:
          value = self.cache.pop(key)
          self.cache[key] = value
          return value
        
        except KeyError:
            print "couldn't find in cache"
            return False
    
    def put(self, key, value):
        print "put in cache -- \n", key
        #print "\n",self.cache
        try:
            #pop the old file with same key from cache
            self.cache.pop(key)
        
        except KeyError:
            if len(self.cache) < self.size:
                #put the new value
                self.cache[key] = value
            elif len(self.cache) == self.size:
                #pop the least recent used item from the cache dict
                # And insert new incoming value
                self.cache.popitem(last=False)
                self.cache[key] = value
            
    
lru_cache = lruCache(CACHE_SIZE)       
    

class FileNode:
    def __init__(self,name,isFile,path,url):
        
        self.name = name
        self.path = path
        self.url = url
        self.isFile = isFile
        self.data_id = 0
        self.meta_id = 0
        self.list_nodes_id = 0
        key = self.path+"&&"+"meta"
        mongoc = MongoClient(url,27017)
        db = mongoc.fsdb
        fsys = db.fs_cache
         # true if node is a file, false if is a directory.
        if not bool(fsys.find_one({"key":key})): # checks if it is already on the server
          self.put("data","") # used if it is a file
          self.put("meta",{})
          self.put("list_nodes",{})# contains a tuple of <name:FileNode>  used only if it is a dir. 
          
        
    def put(self,key,value):
        ## start time of the put function
        start = timeit.default_timer()

        skey = self.path+"&&"+key
        print "\n \n", skey

        mongoc = MongoClient(url,27017)
        db = mongoc.fsdb
        fsys = db.fs_cache               
        
        fdict = dict()
        # pickle the value of the the dictionary
        pickle_value = pickle.dumps(value)
        fdict = {"key":skey, "value":pickle_value}
                
        #first put the file in the cache                  
        lru_cache.put(skey,fdict)
        
        #update the file in the server
        
        fsys.update({"key":skey},{ "$set": { "value": pickle_value } },upsert =True)
        
        #print time taken for PUT
        stop = timeit.default_timer()
        
        print "PUT - RUNTIME--> ", (stop-start)*(1000000)," us"
        
        
        
    def get(self,key):
        
        #GET fucntion start time
        start = timeit.default_timer()
        
        skey = self.path+"&&"+key
        print "\n \n", skey               

        mongoc = MongoClient(url,27017)
        db = mongoc.fsdb
        fsys = db.fs_cache
        
        
        qkey = {"key":str(skey)}
        
        print "root fs: ", fsys.find_one({"key":'/&&list_nodes'})
        
        # first try getting the file from cache
        cdict = lru_cache.get(skey)
        print "cache get", cdict
        
        # if the doc is present in the cache
        if cdict:
            if "value" in cdict.keys():
                
                #print time taken for GET
                stop = timeit.default_timer()
                print "GET - RUNTIME--> ", (stop-start)*(1000000)," us"
                
                return pickle.loads(cdict["value"])
            else:
                return None
        
        #else get from the db server    
        else:
            fdict = fsys.find_one(qkey)
            print fdict
            if "value" in fdict.keys():
                
                if not lru_cache.put(skey,fdict):print "Cache is Full"
                
                #print time taken for GET<commit_msg>asdas<commit_after>#!/usr/bin/env python
import logging
from collections import defaultdict
from errno import ENOENT
from stat import S_IFDIR, S_IFLNK, S_IFREG
from sys import argv, exit
from time import time
from time import time
import datetime
from fuse import FUSE, FuseOSError, Operations, LoggingMixIn
from xmlrpclib import Binary
import sys, pickle, xmlrpclib

count = 0

from fuse import FUSE, FuseOSError, Operations, LoggingMixIn
from pymongo import MongoClient

import collections
import timeit

if not hasattr(__builtins__, 'bytes'):
    bytes = str



## DEFINE CACHE SIZE
CACHE_SIZE = 10

###
##LRU CLASS 


class lruCache:
    
    def __init__(self, size):
        self.size = CACHE_SIZE
        # LRU Cache is imlemented using Ordered Dictionaries
        self.cache = collections.OrderedDict()
    
    def get(self,key):
        ## get the f
        print "get from cache -- \n", key
        #print "CACHE CONTENTS\n",self.cache.keys()
        try:
          value = self.cache.pop(key)
          self.cache[key] = value
          return value
        
        except KeyError:
            print "couldn't find in cache"
            return False
    
    def put(self, key, value):
        #print "put in cache -- \n", key
        #print "\n",self.cache
        try:
            #pop the old file with same key from cache
            self.cache.pop(key)
        
        except KeyError:
            if len(self.cache) < self.size:
                #put the new value
                self.cache[key] = value
            elif len(self.cache) == self.size:
                #pop the least recent used item from the cache dict
                # And insert new incoming value
                self.cache.popitem(last=False)
                self.cache[key] = value
            
    
lru_cache = lruCache(CACHE_SIZE)       
    

class FileNode:
    def __init__(self,name,isFile,path,url):
        
        self.name = name
        self.path = path
        self.url = url
        self.isFile = isFile
        self.data_id = 0
        self.meta_id = 0
        self.list_nodes_id = 0
        key = self.path+"&&"+"meta"
        mongoc = MongoClient(url,27017)
        db = mongoc.fsdb
        fsys = db.fs_cache
         # true if node is a file, false if is a directory.
        if not bool(fsys.find_one({"key":key})): # checks if it is already on the server
          self.put("data","") # used if it is a file
          self.put("meta",{})
          self.put("list_nodes",{})# contains a tuple of <name:FileNode>  used only if it is a dir. 
          
        
    def put(self,key,value):
        ## start time of the put function
        start = timeit.default_timer()

        skey = self.path+"&&"+key
        print "\n \n", skey

        mongoc = MongoClient(url,27017)
        db = mongoc.fsdb
        fsys = db.fs_cache               
        
        fdict = dict()
        # pickle the value of the the dictionary
        pickle_value = pickle.dumps(value)
        fdict = {"key":skey, "value":pickle_value}
                
        #first put the file in the cache                  
        lru_cache.put(skey,fdict)
        
        #update the file in the server
        
        fsys.update({"key":skey},{ "$set": { "value": pickle_value } },upsert =True)
        
        #print time taken for PUT
        stop = timeit.default_timer()
        
        print "PUT - RUNTIME--> ", (stop-start)*(1000000)," us"
        
        
        
    def get(self,key):
        
        #GET fucntion start time
        start = timeit.default_timer()
        
        skey = self.path+"&&"+key
        print "\n \n", skey               

        mongoc = MongoClient(url,27017)
        db = mongoc.fsdb
        fsys = db.fs_cache
        
        
        qkey = {"key":str(skey)}
        
        #print "root fs: ", fsys.find_one({"key":'/&&list_nodes'})
        
        # first try getting the file from cache
        cdict = lru_cache.get(skey)
        #print "cache get", cdict
        
        # if the doc is present in the cache
        if cdict:
            if "value" in cdict.keys():
                
                #print time taken for GET
                stop = timeit.default_timer()
                print "CACHE GET - RUNTIME--> ", (stop-start)*(1000000)," us"
                
                return pickle.loads(cdict["value"])
            else:
                return None
        
        #else get from the db server    
        else:
            fdict = fsys.find_one(qkey)
            print fdict
            if "value" in fdict.keys():
                
                if not lru_cache.put(skey,fdict):print "Cache is Full"
                
                #print time taken for GET
{'fraction_non_alphanumeric': 0.09400804573364387, 'fraction_numerical': 0.007410544145670125, 'mean_word_length': 1.5587757313109425, 'pattern_counts': {'":': 16, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6386314', 'n_tokens_mistral': 2889, 'n_tokens_neox': 2738, 'n_words': 828}