<commit_before>#!/usr/bin/env python
# encoding: utf-8


'''
a simple script to archive data to multiple hard drives

usage:
    archive.py archive <path>
    archive.py unarchive <path>
    archive.py trash

archive configuration is stored in 'archive.cfg'.
contains the list of remotes, such as:
ARCHIVE1 /run/media/user/3a817f8a-9a77-45af-966d/archive/
ARCHIVE2 /run/media/user/9a77-45af-966d/archive/

after a file has been archived, it is replaced by a (broken) symlink of the form:
    file -> REMOTE/SHASUM

the original file is moved to a folder 'trash'.
the content of this folder can be safely removed by using the command
"archive.py trash", which also checks that the content of each file can be read
from the remote.
'''


config = 'archive.cfg'
trash_dir = 'trash'

margin = 10 # do not fill drive to more than this value (in Mb)

from os import statvfs
from sys import argv
from os.path import isdir
import os
import shutil
from os.path import exists, join, dirname
import hashlib
from string import split
from datetime import datetime

def get_first_free_remote(remotes, filename):
    '''
    returns the index of the first free remote
    of -1 if there is none
    '''

    for i in xrange(len(remotes)):

        (rem, rempath) = remotes[i]

        free = get_free_space_mb(rempath)
        if free == None: continue

        fs = file_size(filename)
        if fs + margin > free:
            continue
        else:
            return i

    return -1

def clean_dir(directory):
    '''
    remove the directory if empty
    and its parents if they are also empty
    '''

    assert isdir(directory)
    d = directory
    while isdir(d) and len(os.listdir(d)) == 0:
        os.rmdir(d)
        d = dirname(d)



def get_shasum(filename):
    return hashlib.sha256(open(filename, 'rb').read()).hexdigest()

def get_free_space_mb(path):
    '''
    Return folder/drive free space (in Mb)
    return None if path does not exist
    '''

    if not exists(path): return None

    st = statvfs(path)
    return st.f_bavail * st.f_frsize/(1024*1024)

def file_size(filename):
    '''
    return file size in Mb
    '''
    statinfo = os.stat(filename)

    return statinfo.st_size/(1024*1024)


def loop(paths):
    for path in paths:
        for (dirpath, dirnames, filenames) in os.walk(path):
            dirnames.sort()
            for filename in filenames:
                yield join(dirpath, filename)


def archive(remotes, paths):

    print 'archive', paths
    if paths == []:
        print 'please give a path to archive'<commit_msg>loop not only over directories<commit_after>#!/usr/bin/env python
# encoding: utf-8


'''
a simple script to archive data to multiple hard drives

usage:
    archive.py archive <path>
    archive.py unarchive <path>
    archive.py trash

archive configuration is stored in 'archive.cfg'.
contains the list of remotes, such as:
ARCHIVE1 /run/media/user/3a817f8a-9a77-45af-966d/archive/
ARCHIVE2 /run/media/user/9a77-45af-966d/archive/

after a file has been archived, it is replaced by a (broken) symlink of the form:
    file -> REMOTE/SHASUM

the original file is moved to a folder 'trash'.
the content of this folder can be safely removed by using the command
"archive.py trash", which also checks that the content of each file can be read
from the remote.
'''


config = 'archive.cfg'
trash_dir = 'trash'

margin = 10 # do not fill drive to more than this value (in Mb)

from os import statvfs
from sys import argv
from os.path import isdir
import os
import shutil
from os.path import exists, join, dirname
import hashlib
from string import split
from datetime import datetime
from glob import glob

def get_first_free_remote(remotes, filename):
    '''
    returns the index of the first free remote
    of -1 if there is none
    '''

    for i in xrange(len(remotes)):

        (rem, rempath) = remotes[i]

        free = get_free_space_mb(rempath)
        if free == None: continue

        fs = file_size(filename)
        if fs + margin > free:
            continue
        else:
            return i

    return -1

def clean_dir(directory):
    '''
    remove the directory if empty
    and its parents if they are also empty
    '''

    assert isdir(directory)
    d = directory
    while isdir(d) and len(os.listdir(d)) == 0:
        os.rmdir(d)
        d = dirname(d)



def get_shasum(filename):
    return hashlib.sha256(open(filename, 'rb').read()).hexdigest()

def get_free_space_mb(path):
    '''
    Return folder/drive free space (in Mb)
    return None if path does not exist
    '''

    if not exists(path): return None

    st = statvfs(path)
    return st.f_bavail * st.f_frsize/(1024*1024)

def file_size(filename):
    '''
    return file size in Mb
    '''
    statinfo = os.stat(filename)

    return statinfo.st_size/(1024*1024)


def loop(paths):
    for path in paths:

        if isdir(path):
            for (dirpath, dirnames, filenames) in os.walk(path):
                dirnames.sort()
                for filename in filenames:
                    yield join(dirpath, filename)

        else:
            for filename in glob(path): yield filename


def archive(remotes, paths):

    print 'archive', paths
    if paths == []:
        print 'please give a path to archive'
{'fraction_non_alphanumeric': 0.09044368600682594, 'fraction_numerical': 0.01820250284414107, 'mean_word_length': 2.720028208744711, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3887950', 'n_tokens_mistral': 1724, 'n_tokens_neox': 1616, 'n_words': 598}