<commit_before>#!/usr/bin/python2.7

import click

from datetime import datetime
from glob import glob
import os
from subprocess import Popen
from os.path import basename, dirname, exists
import re
import shutil


# Set by command line arguments, defaults below Click.command()
root = None
docker_container_id = None
neo4j_shell = None

debug = True

backup_template = 'backup-%s.dump'
backup_glob = backup_template % '*'
backup_re = re.compile(backup_template % '(\d\d\d\d)(\d\d)(\d\d)')

DAILY_KEPT = 7  # should be >= 7
WEEKLY_KEPT = 5  # should be >= 5

def unlink(filename):
    if debug:
        print("unlinking %s")
    os.unlink(filename)

def makedirs(path):
    if exists(path):
        return
    if debug:
        print("makedirs %s" % path)
    os.makedirs(path)

def copy(src, dst):
    if debug:
        print("copy %s %s" % (src, dst))
    if not exists(src):
        print("bug: %s doesn't exist") % src
        raise SystemExit
    dst_path = dirname(dst)
    if not exists(dst_path):
        makedirs(dst_path)
    shutil.copy(src, dst)

def neo4jshell(command, stdout):
    args = [neo4j_shell, '-c', command]
    if docker_container_id is not None:
        args = ['docker', 'exec', docker_container_id] + args
    if debug:
        print("executing %s (%r)" % (' '.join(args), args))
    Popen(args=args, stdout=open(stdout, 'w+')).wait()

def next_daily_filename():
    filename = os.path.join(daily_root(), backup_template % datetime.now().strftime('%Y%m%d'))
    return filename

def daily_root():
    return os.path.join(root, 'daily')

def weekly_root():
    return os.path.join(root, 'weekly')

def monthly_root():
    return os.path.join(root, 'monthly')

def backup_year_month_day(filename):
    return map(int, backup_re.match(basename(filename)).groups())

def to_weekly(filename):
    year, month, day = backup_year_month_day(filename)
    _, (wkly_year, wkly_month, wkly_day) = latest_weekly()
    # import pdb; pdb.set_trace()
    delta = datetime(year, month, day) - datetime(wkly_year, wkly_month, wkly_day)
    return delta.days >= 7

def to_monthly(filename):
    """ return True if filename is newer by a month from the newest monthly """
    year, month, day = backup_year_month_day(filename)
    _, (monthly_year, monthly_month, monthly_day) = latest_monthly()
    if monthly_year < year:
        month += 12
    return month > monthly_month and day >= monthly_day

def latest_weekly():
    return latest(weekly_root())

def latest_monthly():
    return latest(monthly_root())

def latest(subdir):
    """
    returns filename, (year, month, day)
    """
    filenames = glob(os.path.join(subdir, backup_glob))
    if len(filenames) == 0:
        return 'no_such_file', (1848, 1, 1)
    return max((fname, backup_year_month_day(fname)) for fname in filenames)

def sorted_backups(subdir):
    def cmp_files(fa, fb):
        ta = os.stat(fa).mtime
        tb = os.stat(fb).mtime<commit_msg>rz_backup: fix s/mtime/st_mtime/<commit_after>#!/usr/bin/python2.7

import click

from datetime import datetime
from glob import glob
import os
from subprocess import Popen
from os.path import basename, dirname, exists
import re
import shutil


# Set by command line arguments, defaults below Click.command()
root = None
docker_container_id = None
neo4j_shell = None

debug = True

backup_template = 'backup-%s.dump'
backup_glob = backup_template % '*'
backup_re = re.compile(backup_template % '(\d\d\d\d)(\d\d)(\d\d)')

DAILY_KEPT = 7  # should be >= 7
WEEKLY_KEPT = 5  # should be >= 5

def unlink(filename):
    if debug:
        print("unlinking %s")
    os.unlink(filename)

def makedirs(path):
    if exists(path):
        return
    if debug:
        print("makedirs %s" % path)
    os.makedirs(path)

def copy(src, dst):
    if debug:
        print("copy %s %s" % (src, dst))
    if not exists(src):
        print("bug: %s doesn't exist") % src
        raise SystemExit
    dst_path = dirname(dst)
    if not exists(dst_path):
        makedirs(dst_path)
    shutil.copy(src, dst)

def neo4jshell(command, stdout):
    args = [neo4j_shell, '-c', command]
    if docker_container_id is not None:
        args = ['docker', 'exec', docker_container_id] + args
    if debug:
        print("executing %s (%r)" % (' '.join(args), args))
    Popen(args=args, stdout=open(stdout, 'w+')).wait()

def next_daily_filename():
    filename = os.path.join(daily_root(), backup_template % datetime.now().strftime('%Y%m%d'))
    return filename

def daily_root():
    return os.path.join(root, 'daily')

def weekly_root():
    return os.path.join(root, 'weekly')

def monthly_root():
    return os.path.join(root, 'monthly')

def backup_year_month_day(filename):
    return map(int, backup_re.match(basename(filename)).groups())

def to_weekly(filename):
    year, month, day = backup_year_month_day(filename)
    _, (wkly_year, wkly_month, wkly_day) = latest_weekly()
    # import pdb; pdb.set_trace()
    delta = datetime(year, month, day) - datetime(wkly_year, wkly_month, wkly_day)
    return delta.days >= 7

def to_monthly(filename):
    """ return True if filename is newer by a month from the newest monthly """
    year, month, day = backup_year_month_day(filename)
    _, (monthly_year, monthly_month, monthly_day) = latest_monthly()
    if monthly_year < year:
        month += 12
    return month > monthly_month and day >= monthly_day

def latest_weekly():
    return latest(weekly_root())

def latest_monthly():
    return latest(monthly_root())

def latest(subdir):
    """
    returns filename, (year, month, day)
    """
    filenames = glob(os.path.join(subdir, backup_glob))
    if len(filenames) == 0:
        return 'no_such_file', (1848, 1, 1)
    return max((fname, backup_year_month_day(fname)) for fname in filenames)

def sorted_backups(subdir):
    def cmp_files(fa, fb):
        ta = os.stat(fa).st_mtime
        tb = os.stat(fb).st_mtime
{'fraction_non_alphanumeric': 0.13109186619120394, 'fraction_numerical': 0.006452708439463406, 'mean_word_length': 3.6015625, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '230631', 'n_tokens_mistral': 2099, 'n_tokens_neox': 2026, 'n_words': 521}