<commit_before>"""
Process single file
"""
import leveldb
import tempfile
import shutil
import ntpath
import logging
import os
import sys
from myutils import compute_hash
from myutils import random_name

class FileProcessor(object):
    HASH_SEPARATOR = '@'
    FILE_PATH_MARK = ''
    ENCRYPTED_FILE_MARK = '#'

    def __init__(self, config, db, encryptor):
        self.database = db
        self.dropbox_folder = os.path.join(config.get('DropBox', \
            'drop_box_dir'), '')
        self.password = config.get('Credentials', 'password')
        self.logger = logging.getLogger("mylog")
        self.encryptor = encryptor

    def process(self, file_path):
        """
        Process single file
        """
        self.logger.info('Processing '+file_path)
        file_path_aug = file_path + self.FILE_PATH_MARK
        current_hash_of_unenc_file = ''
        try:
            current_hash_of_unenc_file = compute_hash(file_path)
            existing_combined_hash = self.database.Get(file_path_aug)
            split = existing_combined_hash.split(self.HASH_SEPARATOR)
            previous_hash_of_unenc_file = split[0]
            name = split[2]

            if previous_hash_of_unenc_file != current_hash_of_unenc_file:
                self.logger.info('File needs to be updated. Adding to dropbox' +
                    ' folder')
                self.__encrypt_copy(file_path, current_hash_of_unenc_file, name)
                self.logger.debug('Done')
            else:
                self.logger.info('File is up to date')

        except KeyError:
            self.logger.info('File not know. Adding to dropbox folder')
            self.__encrypt_copy(file_path, current_hash_of_unenc_file)
            self.logger.debug('Done')

    def __encrypt_copy(self, file_path, current_hash_of_unenc_file, name = None):
        """
        Encrypt file and copy to dropbox folder
        """

        self.logger.debug('Calling __encrypt_copy, file_path: ' + file_path +
            ' current_hash_of_unenc_file: ' + current_hash_of_unenc_file +
            ' name: ' + str(name))

        file_name = ntpath.basename(file_path)
        temp_dir = tempfile.gettempdir()
        temp_dir += '/'
        destination_path = temp_dir+file_name
        self.logger.debug('Temp destination: ' + destination_path)<commit_msg>Fix typo<commit_after>"""
Process single file
"""
import leveldb
import tempfile
import shutil
import ntpath
import logging
import os
import sys
from myutils import compute_hash
from myutils import random_name

class FileProcessor(object):
    HASH_SEPARATOR = '@'
    FILE_PATH_MARK = ''
    ENCRYPTED_FILE_MARK = '#'

    def __init__(self, config, db, encryptor):
        self.database = db
        self.dropbox_folder = os.path.join(config.get('DropBox', \
            'drop_box_dir'), '')
        self.password = config.get('Credentials', 'password')
        self.logger = logging.getLogger("mylog")
        self.encryptor = encryptor

    def process(self, file_path):
        """
        Process single file
        """
        self.logger.info('Processing '+file_path)
        file_path_aug = file_path + self.FILE_PATH_MARK
        current_hash_of_unenc_file = ''
        try:
            current_hash_of_unenc_file = compute_hash(file_path)
            existing_combined_hash = self.database.Get(file_path_aug)
            split = existing_combined_hash.split(self.HASH_SEPARATOR)
            previous_hash_of_unenc_file = split[0]
            name = split[2]

            if previous_hash_of_unenc_file != current_hash_of_unenc_file:
                self.logger.info('File needs to be updated. Adding to dropbox' +
                    ' folder')
                self.__encrypt_copy(file_path, current_hash_of_unenc_file, name)
                self.logger.debug('Done')
            else:
                self.logger.info('File is up to date')

        except KeyError:
            self.logger.info('File not known. Adding to dropbox folder')
            self.__encrypt_copy(file_path, current_hash_of_unenc_file)
            self.logger.debug('Done')

    def __encrypt_copy(self, file_path, current_hash_of_unenc_file, name = None):
        """
        Encrypt file and copy to dropbox folder
        """

        self.logger.debug('Calling __encrypt_copy, file_path: ' + file_path +
            ' current_hash_of_unenc_file: ' + current_hash_of_unenc_file +
            ' name: ' + str(name))

        file_name = ntpath.basename(file_path)
        temp_dir = tempfile.gettempdir()
        temp_dir += '/'
        destination_path = temp_dir+file_name
        self.logger.debug('Temp destination: ' + destination_path)
{'fraction_non_alphanumeric': 0.09305317324185249, 'fraction_numerical': 0.0008576329331046312, 'mean_word_length': 2.476154992548435, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '869742', 'n_tokens_mistral': 1429, 'n_tokens_neox': 1389, 'n_words': 272}