<commit_before># -*- coding: utf-8 -*-

import md5
import re
from StringIO import StringIO
import mimetypes
import dateutil.parser
import time
import os
import tempfile

from oauth2client.client import AccessTokenRefreshError

import client as client_module
from utils import *
from apiclient.errors import HttpError
from configuration import Configuration


class User:

    def __init__(self, login, client):
        # check that we have only the login, not the full email address
        if '@' in login:
            login = login.partition('@')[0]
        self._login = login
        self._client = client
        self._documents = None
        self._folders = UserFolders(self)
        self._black_listed_ids = []

    def __repr__(self):
        return self._login

    @property
    def login(self):
        return self._login

    @property
    def folders(self):
        return self._folders

    @property
    def documents(self):
        if self._documents is None:
            self._fetch_docs_list()
        return self._documents

    # just the doc ids
    @property
    def document_ids(self):
        return [doc.id for doc in self.documents]

    @property
    def drive_service(self):
        client = self._client
        client.authorize(self)
        return client.drive_service

    def save_documents(self, backend, owned_only):
        Log.verbose(u'Processing docs for {}'.format(self.login))
        # let's filter folders out
        doc_generator = client_module.UserDocumentsGenerator(
            self, Document.get_folder_query()
        )
        for document in doc_generator:
            if not backend.need_to_fetch_contents(self, document)\
                    or (owned_only and not document.is_owned):
                # mark as done, and get to the next one
                Log.verbose(
                    u'Not necessary to fetch doc id {}'.format(document.id)
                )
                doc_generator.add_processed_id(document.id)
                continue

            Log.verbose(u'Processing {}\'s doc "{}" (id: {})'.format(
                self.login, document.title, document.id
            ))
            try:
                document.fetch_contents(self._client)
                self._save_single_document(backend, document)
            except client_module.ExpiredTokenException as ex:
                if document.id in self._black_listed_ids:
                    # we already got a 403 on that one!
                    explanation = 'Two 403 errors on a row on document id {}'\
                        .format(document.id)
                    ex.brive_explanation = explanation
                    raise
                # otherwise try again
                Log.verbose(
                    '403 response, sleeping one minute and re-trying...'
                )
                time.sleep(60)
                # the re-auth is handled by the the list request
                doc_generator.reset_to_current_page()
                self._black_listed_ids.append(document.id)
                continue
            except Exception as ex:
                explanation = \
                    'Unexpected error when processing ' \
                    + '{}\'s documents '.format(self.login) \
                    + u'(doc id: {})'.format(document.id)
                ex.brive_explanation = explanation
                raise
            # mark as done
            doc_generator.add_processed_id(document.id)
        # let's save some memory
        self._cleanup()
        backend.close_user(self)

    def retrieve_single_document(self, backend, doc_id):
        try:
            document = self.retrieve_single_document_meta(doc_id)
        except Exception as e:
            explanation = 'Error while retrieving single document id ' \
                + u'{} for user {}, '.format(doc_id, self.login) \
                + 'it\'s liklely this user isn\'t allowed to see that doc'
            e.brive_explanation = explanation
            raise
        document.fetch_contents(self._client)
        self._save_single_document(backend, document)

    # NOTE: Google's API doesn't like to get a lot of such calls,
    # so use defensively...
    @Utils.multiple_tries_decorator(None)
    def retrieve_single_document_meta(self, doc_id):
        try:
            meta = self.drive_service.files().get(fileId=doc_id).execute()
            return Document(meta, self.folders)
        except AccessTokenRefreshError:
            # most likely 403
            raise client_module.ExpiredTokenException

    def _save_single_document(self, backend, document):
        try:
            Log.verbose(u'Saving {}\'s doc "{}" (id: {})'.format(
                self.login, document.title, document.id
            ))
            backend.save(self, document)
        except Exception as ex:
            explanation = \
                'Unexpected error when saving ' \
                + '{}\'s documents '.format(self.login) \
                + u'(doc id: {})'.format(document.id)
            ex.brive_explanation = explanation
            raise
        # no need to keep the potentially big document's contents in memory
        document.del_contents()

    def _cleanup(self):
        del self._documents
        del self._folders
        del self._black_listed_ids


# keeps tracks of the user's folders, and caches the paths to them
class UserFolders:

    def __init__(self, user):
        self._user = user
        self._initialized = False

    def get_path(self, folder_id):
        if folder_id is None:
            # the root has ID None by convention
            return ''
        self._do_init()
        folder = self._folders[folder_id]
        parent_path = self.get_path(folder.parent_id)
        parent_path += os.sep if parent_path else ''
        return parent_path + folder.title

    def _do_init(self):
        if self._initialized:
            return
        self._initialized = True
        Log.debug(u'Initializing folders for user {}'.format(self._user.login))
        # dict that maps a folder id to its object
        self._folders = self._build_folders()

    def _build_folders(self):
        folder_generator = client_module.UserDocumentsGenerator(
            self._user, Document.get_folder_query(False), Folder
        )
        return {folder.id: folder for folder in folder_generator}


class DocumentContent(object):

    def __init__(self, client, url, document):
        self._client = client
        self._url = url
        self._document = document
        headers, self._content = self._make_request()
        self.file_name = self._get_file_name(headers)
        try:
            self.size = int(headers['content-length'])
        except KeyError:
            self.size = None

    _CHUNK_SIZE = 1048576  # 1 Mb

    # returns a file-like object
    # if size_requested is set to True, then the self.size attribute
    # will be accurate after this returns
    def get_file_object(self, size_requested=False):
        if self._client.streaming:
            _, self._content = self._make_request()
            if not size_requested or self.size is not None:
                return self._content<commit_msg>never trust google...<commit_after># -*- coding: utf-8 -*-

import md5
import re
from StringIO import StringIO
import mimetypes
import dateutil.parser
import time
import os
import tempfile

from oauth2client.client import AccessTokenRefreshError

import client as client_module
from utils import *
from apiclient.errors import HttpError
from configuration import Configuration


class User:

    def __init__(self, login, client):
        # check that we have only the login, not the full email address
        if '@' in login:
            login = login.partition('@')[0]
        self._login = login
        self._client = client
        self._documents = None
        self._folders = UserFolders(self)
        self._black_listed_ids = []

    def __repr__(self):
        return self._login

    @property
    def login(self):
        return self._login

    @property
    def folders(self):
        return self._folders

    @property
    def documents(self):
        if self._documents is None:
            self._fetch_docs_list()
        return self._documents

    # just the doc ids
    @property
    def document_ids(self):
        return [doc.id for doc in self.documents]

    @property
    def drive_service(self):
        client = self._client
        client.authorize(self)
        return client.drive_service

    def save_documents(self, backend, owned_only):
        Log.verbose(u'Processing docs for {}'.format(self.login))
        # let's filter folders out
        doc_generator = client_module.UserDocumentsGenerator(
            self, Document.get_folder_query()
        )
        for document in doc_generator:
            if not backend.need_to_fetch_contents(self, document)\
                    or (owned_only and not document.is_owned):
                # mark as done, and get to the next one
                Log.verbose(
                    u'Not necessary to fetch doc id {}'.format(document.id)
                )
                doc_generator.add_processed_id(document.id)
                continue

            Log.verbose(u'Processing {}\'s doc "{}" (id: {})'.format(
                self.login, document.title, document.id
            ))
            try:
                document.fetch_contents(self._client)
                self._save_single_document(backend, document)
            except client_module.ExpiredTokenException as ex:
                if document.id in self._black_listed_ids:
                    # we already got a 403 on that one!
                    explanation = 'Two 403 errors on a row on document id {}'\
                        .format(document.id)
                    ex.brive_explanation = explanation
                    raise
                # otherwise try again
                Log.verbose(
                    '403 response, sleeping one minute and re-trying...'
                )
                time.sleep(60)
                # the re-auth is handled by the the list request
                doc_generator.reset_to_current_page()
                self._black_listed_ids.append(document.id)
                continue
            except Exception as ex:
                explanation = \
                    'Unexpected error when processing ' \
                    + '{}\'s documents '.format(self.login) \
                    + u'(doc id: {})'.format(document.id)
                ex.brive_explanation = explanation
                raise
            # mark as done
            doc_generator.add_processed_id(document.id)
        # let's save some memory
        self._cleanup()
        backend.close_user(self)

    def retrieve_single_document(self, backend, doc_id):
        try:
            document = self.retrieve_single_document_meta(doc_id)
        except Exception as e:
            explanation = 'Error while retrieving single document id ' \
                + u'{} for user {}, '.format(doc_id, self.login) \
                + 'it\'s liklely this user isn\'t allowed to see that doc'
            e.brive_explanation = explanation
            raise
        document.fetch_contents(self._client)
        self._save_single_document(backend, document)

    # NOTE: Google's API doesn't like to get a lot of such calls,
    # so use defensively...
    @Utils.multiple_tries_decorator(None)
    def retrieve_single_document_meta(self, doc_id):
        try:
            meta = self.drive_service.files().get(fileId=doc_id).execute()
            return Document(meta, self.folders)
        except AccessTokenRefreshError:
            # most likely 403
            raise client_module.ExpiredTokenException

    def _save_single_document(self, backend, document):
        try:
            Log.verbose(u'Saving {}\'s doc "{}" (id: {})'.format(
                self.login, document.title, document.id
            ))
            backend.save(self, document)
        except Exception as ex:
            explanation = \
                'Unexpected error when saving ' \
                + '{}\'s documents '.format(self.login) \
                + u'(doc id: {})'.format(document.id)
            ex.brive_explanation = explanation
            raise
        # no need to keep the potentially big document's contents in memory
        document.del_contents()

    def _cleanup(self):
        del self._documents
        del self._folders
        del self._black_listed_ids


# keeps tracks of the user's folders, and caches the paths to them
class UserFolders:

    def __init__(self, user):
        self._user = user
        self._initialized = False

    def get_path(self, folder_id):
        if folder_id is None:
            # the root has ID None by convention
            return ''
        self._do_init()
        folder = self._folders[folder_id]
        parent_path = self.get_path(folder.parent_id)
        parent_path += os.sep if parent_path else ''
        return parent_path + folder.title

    def _do_init(self):
        if self._initialized:
            return
        self._initialized = True
        Log.debug(u'Initializing folders for user {}'.format(self._user.login))
        # dict that maps a folder id to its object
        self._folders = self._build_folders()

    def _build_folders(self):
        folder_generator = client_module.UserDocumentsGenerator(
            self._user, Document.get_folder_query(False), Folder
        )
        return {folder.id: folder for folder in folder_generator}


class DocumentContent(object):

    def __init__(self, client, url, document):
        self._client = client
        self._url = url
        self._document = document
        headers, self._content = self._make_request()
        self.file_name = self._get_file_name(headers)
        self.size = None

    _CHUNK_SIZE = 1048576  # 1 Mb

    # returns a file-like object
    # if size_requested is set to True, then the self.size attribute
    # will be accurate after this returns
    def get_file_object(self, size_requested=False):
        if self._client.streaming:
            _, self._content = self._make_request()
            if not size_requested or self.size is not None:
                return self._content
{'fraction_non_alphanumeric': 0.08032522604612041, 'fraction_numerical': 0.003644774654797785, 'mean_word_length': 1.901179341195608, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2662831', 'n_tokens_mistral': 3864, 'n_tokens_neox': 3675, 'n_words': 1066}