<commit_before>#!/usr/bin/env python
# Files.py
#   Copyright (C) 2006 CCLRC, <NAME>
#
#   This code is distributed under the BSD license, a copy of which is 
#   included in the root directory of this package.
#
# A manager for files - this will record temporary and output files from 
# xia2, which can be used for composing a dump of "useful" files at the end
# if processing.
#
# This will also be responsible for migrating the data - that is, when
# the .xinfo file is parsed the directories referred to therein may be
# migrated to a local disk. This will use a directory created by
# tempfile.mkdtemp().

import os
import sys
import exceptions
import shutil
import tempfile

from Environment import Environment
from Handlers.Streams import Chatter

class _FileHandler:
    '''A singleton class to manage files.'''

    def __init__(self):
        self._temporary_files = []
        self._output_files = []

        self._log_files = { }
        self._log_file_keys = []

        # for data migration to local disk, bug # 2274
        
        self._data_migrate = { }

    def migrate(self, directory):
        '''Migrate (or not) data to a local directory.'''

        # import this here to cope with race conditions at startup
        from Handlers.CommandLine import CommandLine

        if not CommandLine.get_migrate_data():
            # we will not migrate this data
            return directory

        if directory in self._data_migrate.keys():
            # we have already migrated this data
            return self._data_migrate[directory]

        # create a directory to move data to...
        self._data_migrate[directory] = tempfile.mkdtemp()

        # copy all files in source directory to new directory
        # retaining timestamps etc.

        migrated = 0
        migrated_dir = 0
        for f in os.listdir(directory):
            # copy over only files....
            if os.path.isfile(os.path.join(directory, f)):
                shutil.copy2(os.path.join(directory, f),
                             self._data_migrate[directory])
                migrated += 1
            elif os.path.isdir(os.path.join(directory, f)):
                shutil.copytree(os.path.join(directory, f),
                                self._data_migrate[directory])
                migrated_dir += 1
                

        Chatter.write('Migrated %d files from %s to %s' % \
                      (migrated, directory, self._data_migrate[directory]))

        if migrated_dir > 0:
            Chatter.write('Migrated %d directories from %s to %s' % \
                          (migrated_dir, directory,
                           self._data_migrate[directory]))


        return self._data_migrate[directory]
        
    def cleanup(self):
        out = open('xia-files.txt', 'w')
        for f in self._temporary_files:
            try:<commit_msg>Fixed bug copying directory - need a full directory name to copy to...<commit_after>#!/usr/bin/env python
# Files.py
#   Copyright (C) 2006 CCLRC, <NAME>
#
#   This code is distributed under the BSD license, a copy of which is 
#   included in the root directory of this package.
#
# A manager for files - this will record temporary and output files from 
# xia2, which can be used for composing a dump of "useful" files at the end
# if processing.
#
# This will also be responsible for migrating the data - that is, when
# the .xinfo file is parsed the directories referred to therein may be
# migrated to a local disk. This will use a directory created by
# tempfile.mkdtemp().

import os
import sys
import exceptions
import shutil
import tempfile

from Environment import Environment
from Handlers.Streams import Chatter

class _FileHandler:
    '''A singleton class to manage files.'''

    def __init__(self):
        self._temporary_files = []
        self._output_files = []

        self._log_files = { }
        self._log_file_keys = []

        # for data migration to local disk, bug # 2274
        
        self._data_migrate = { }

    def migrate(self, directory):
        '''Migrate (or not) data to a local directory.'''

        # import this here to cope with race conditions at startup
        from Handlers.CommandLine import CommandLine

        if not CommandLine.get_migrate_data():
            # we will not migrate this data
            return directory

        if directory in self._data_migrate.keys():
            # we have already migrated this data
            return self._data_migrate[directory]

        # create a directory to move data to...
        self._data_migrate[directory] = tempfile.mkdtemp()

        # copy all files in source directory to new directory
        # retaining timestamps etc.

        migrated = 0
        migrated_dir = 0
        for f in os.listdir(directory):
            # copy over only files....
            if os.path.isfile(os.path.join(directory, f)):
                shutil.copy2(os.path.join(directory, f),
                             self._data_migrate[directory])
                migrated += 1
            elif os.path.isdir(os.path.join(directory, f)):
                shutil.copytree(os.path.join(directory, f),
                                os.path.join(self._data_migrate[directory],
                                             f))
                migrated_dir += 1
                

        Chatter.write('Migrated %d files from %s to %s' % \
                      (migrated, directory, self._data_migrate[directory]))

        if migrated_dir > 0:
            Chatter.write('Migrated %d directories from %s to %s' % \
                          (migrated_dir, directory,
                           self._data_migrate[directory]))


        return self._data_migrate[directory]
        
    def cleanup(self):
        out = open('xia-files.txt', 'w')
        for f in self._temporary_files:
            try:
{'fraction_non_alphanumeric': 0.08178248249957316, 'fraction_numerical': 0.005122076148198737, 'mean_word_length': 1.9964194373401534, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '573949', 'n_tokens_mistral': 1630, 'n_tokens_neox': 1568, 'n_words': 547}