<commit_before>#!/usr/bin/env python
"""
FUSE-Gett: file system to directly access Ge.tt shares
"""
import logging

from collections import defaultdict
from errno import ENOENT
from stat import S_IFDIR, S_IFLNK, S_IFREG
from sys import argv, exit
from time import time

from fuse import FUSE, FuseOSError, Operations, LoggingMixIn

import requests
import simplejson as json

if not hasattr(__builtins__, 'bytes'):
    bytes = str

class Gett(LoggingMixIn, Operations):
    """ 
    Starting off based on the example memory filesystem.
    """
    apibase = "http://open.ge.tt"


    def __init__(self, apikey, email, password):
        jdata = json.dumps(dict(apikey=apikey, email=email, password=password))
        print "Connecting to Ge.tt"
        apitarget = "%s/1/users/login" %(self.apibase)
        req = requests.post(apitarget, data=jdata)
        if req.ok:
            print "Logged in!"
            res = json.loads(req.content)
            self.atoken = res['accesstoken']
            sharelist = self._getsharelist()
            self.spaceused, self.spacetotal = res['user']['storage']['used'], res['user']['storage']['limit']
        else:
            sharelist = []

        now = time()
        self.files = {}
        self.data = defaultdict(bytes)
        self.fd = 0
        self.files['/'] = dict(st_mode=(S_IFDIR | 0755), st_ctime=now,
                               st_mtime=now, st_atime=now, st_nlink=2)

        # Populate the directory with the share names
        for share in sharelist:
            dirname = share['title'] if "title" in share else share['sharename']
            self.files["/%s" %(dirname)] = dict(st_mode=(S_IFDIR | 0755),
                                      st_ctime=share['created'],
                                      st_mtime=share['created'],
                                      st_atime=now,
                                      st_nlink=2,
                                      )
            for f in share['files']:
                filename = f['filename']
                size = f['size'] if 'size' in f else 0
                self.files["/%s/%s" %(dirname, filename)] = dict(st_mode=(S_IFREG | 0755),
                                                                      st_ctime=f['created'],
                                                                      st_mtime=f['created'],
                                                                      st_atime=now,
                                                                      st_nlink=1,
                                                                      st_size=size,
                                                                      )
                self.files["/%s" %(dirname)]['st_nlink'] += 1
            self.files['/']['st_nlink'] += 1


    def _getsharelist(self):
        """ Get the list of shares for the logged in user """
        apitarget = "%s/1/shares?accesstoken=%s" %(self.apibase, self.atoken)
        req = requests.get(apitarget)
        result = json.loads(req.content) if req.ok else []
        return result

    def chmod(self, path, mode):
        self.files[path]['st_mode'] &= 0770000
        self.files[path]['st_mode'] |= mode
        return 0

    def chown(self, path, uid, gid):
        self.files[path]['st_uid'] = uid
        self.files[path]['st_gid'] = gid

    def create(self, path, mode):
        print("Create: %s" %(path))
        self.files[path] = dict(st_mode=(S_IFREG | mode), st_nlink=1,
                                st_size=0, st_ctime=time(), st_mtime=time(),
                                st_atime=time())

        self.fd += 1
        return self.fd

    def getattr(self, path, fh=None):
        if path not in self.files:
            raise FuseOSError(ENOENT)

        return self.files[path]

    def getxattr(self, path, name, position=0):
        attrs = self.files[path].get('attrs', {})

        try:
            return attrs[name]
        except KeyError:
            return ''       # Should return ENOATTR

    def listxattr(self, path):
        attrs = self.files[path].get('attrs', {})
        return attrs.keys()

    def mkdir(self, path, mode):
        self.files[path] = dict(st_mode=(S_IFDIR | mode), st_nlink=2,
                                st_size=0, st_ctime=time(), st_mtime=time(),
                                st_atime=time())

        self.files['/']['st_nlink'] += 1

    def open(self, path, flags):
        self.fd += 1
        return self.fd

    def read(self, path, size, offset, fh):
        return self.data[path][offset:offset + size]

    def readdir(self, path, fh):
        """ Read a directory """
        # Currently working but low performance because we have to
        # scan the complete list of files
        pathlen = len(path)
        if path != '/':
            pathlen += 1
        result = ['.', '..']
        for x in self.files:
            if x == '/' or not x.startswith(path):
                continue
            print x[pathlen:]
            name = x[pathlen:].split('/')
            if len(name) == 1:
                result += [name[0]]
        return result

    def readlink(self, path):
        return self.data[path]

    def removexattr(self, path, name):<commit_msg>download file data, closes #2<commit_after>#!/usr/bin/env python
"""
FUSE-Gett: file system to directly access Ge.tt shares
"""
import logging

from collections import defaultdict
from errno import ENOENT
from stat import S_IFDIR, S_IFLNK, S_IFREG
from sys import argv, exit
from time import time

from fuse import FUSE, FuseOSError, Operations, LoggingMixIn

import requests
import simplejson as json

if not hasattr(__builtins__, 'bytes'):
    bytes = str

class Gett(LoggingMixIn, Operations):
    """ 
    Starting off based on the example memory filesystem.
    """
    apibase = "http://open.ge.tt"


    def __init__(self, apikey, email, password):
        jdata = json.dumps(dict(apikey=apikey, email=email, password=password))
        print "Connecting to Ge.tt"
        apitarget = "%s/1/users/login" %(self.apibase)
        req = requests.post(apitarget, data=jdata)
        if req.ok:
            print "Logged in!"
            res = json.loads(req.content)
            self.atoken = res['accesstoken']
            sharelist = self._getsharelist()
            self.spaceused, self.spacetotal = res['user']['storage']['used'], res['user']['storage']['limit']
        else:
            sharelist = []

        now = time()
        self.files = {}
        self.data = defaultdict(bytes)
        self.fd = 0
        self.files['/'] = dict(st_mode=(S_IFDIR | 0755), st_ctime=now,
                               st_mtime=now, st_atime=now, st_nlink=2)

        # Populate the directory with the share names
        for share in sharelist:
            sharename = share['sharename']
            dirname = share['title'] if "title" in share else sharename
            self.files["/%s" %(dirname)] = dict(st_mode=(S_IFDIR | 0755),
                                      st_ctime=share['created'],
                                      st_mtime=share['created'],
                                      st_atime=now,
                                      st_nlink=2,
                                      )
            for f in share['files']:
                filename = f['filename']
                size = f['size'] if 'size' in f else 0
                self.files["/%s/%s" %(dirname, filename)] = dict(st_mode=(S_IFREG | 0755),
                                                                 st_ctime=f['created'],
                                                                 st_mtime=f['created'],
                                                                 st_atime=now,
                                                                 st_nlink=1,
                                                                 st_size=size,
                                                                 sharename=sharename,
                                                                 fileid=f['fileid'],
                                                                 )
                self.files["/%s" %(dirname)]['st_nlink'] += 1
            self.files['/']['st_nlink'] += 1


    def _getsharelist(self):
        """ Get the list of shares for the logged in user """
        apitarget = "%s/1/shares?accesstoken=%s" %(self.apibase, self.atoken)
        req = requests.get(apitarget)
        result = json.loads(req.content) if req.ok else []
        return result

    def _getfile(self, sharename, fileid):
        apitarget = "%s/1/files/%s/%s/blob" %(self.apibase, sharename, fileid)
        req = requests.get(apitarget)
        if req.ok:
            res = req.content
            print "Downloaded: ", sharename, fileid
        else:
            res = 0
        return res

    def chmod(self, path, mode):
        self.files[path]['st_mode'] &= 0770000
        self.files[path]['st_mode'] |= mode
        return 0

    def chown(self, path, uid, gid):
        self.files[path]['st_uid'] = uid
        self.files[path]['st_gid'] = gid

    def create(self, path, mode):
        print("Create: %s" %(path))
        self.files[path] = dict(st_mode=(S_IFREG | mode), st_nlink=1,
                                st_size=0, st_ctime=time(), st_mtime=time(),
                                st_atime=time())

        self.fd += 1
        return self.fd

    def getattr(self, path, fh=None):
        if path not in self.files:
            raise FuseOSError(ENOENT)

        return self.files[path]

    def getxattr(self, path, name, position=0):
        attrs = self.files[path].get('attrs', {})

        try:
            return attrs[name]
        except KeyError:
            return ''       # Should return ENOATTR

    def listxattr(self, path):
        attrs = self.files[path].get('attrs', {})
        return attrs.keys()

    def mkdir(self, path, mode):
        self.files[path] = dict(st_mode=(S_IFDIR | mode), st_nlink=2,
                                st_size=0, st_ctime=time(), st_mtime=time(),
                                st_atime=time())

        self.files['/']['st_nlink'] += 1

    def open(self, path, flags):
        self.fd += 1
        return self.fd

    def read(self, path, size, offset, fh):
        """ Called when a file is read """
        # Download data if we haven't done that yet
        if path not in self.data:
            sharename, fileid = self.files[path]['sharename'], self.files[path]['fileid']
            binfile = self._getfile(sharename, fileid)
            self.data[path] = binfile
        return self.data[path][offset:offset + size]

    def readdir(self, path, fh):
        """ Read a directory """
        # Currently working but low performance because we have to
        # scan the complete list of files
        pathlen = len(path)
        if path != '/':
            pathlen += 1
        result = ['.', '..']
        for x in self.files:
            if x == '/' or not x.startswith(path):
                continue
            print x[pathlen:]
            name = x[pathlen:].split('/')
            if len(name) == 1:
                result += [name[0]]
        return result

    def readlink(self, path):
        return self.data[path]

    def removexattr(self, path, name):
{'fraction_non_alphanumeric': 0.12001418188264493, 'fraction_numerical': 0.007356851622052828, 'mean_word_length': 1.5006648936170213, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 46, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2913624', 'n_tokens_mistral': 3266, 'n_tokens_neox': 3127, 'n_words': 783}