<commit_before>import argparse
import csv
import json
import logging
import os
import requests_unixsocket
from requests.exceptions import ConnectionError
import sys
import urllib
from bottle import app
from buttervolume.plugin import jsonloads, SCHEDULE
from buttervolume.plugin import SCHEDULE_LOG, SNAPSHOTS_PATH
from datetime import datetime, timedelta
from threading import Timer
from waitress import serve
from webtest import TestApp
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()
SOCKET = '/run/docker/plugins/btrfs.sock'
TIMER = 60
app = app()


class Session(object):
    """wrapper for requests_unixsocket.Session
    """
    def __init__(self):
        self.session = requests_unixsocket.Session()

    def post(self, *a, **kw):
        try:
            return self.session.post(*a, **kw)
        except ConnectionError:
            logger.error('Failed to connect to Buttervolume. '
                         'You can start it with: buttervolume run')
            sys.exit(1)

    def get(self, *a, **kw):
        try:
            return self.session.get(*a, **kw)
        except ConnectionError:
            logger.error('Failed to connect to Buttervolume. '
                         'You can start it with: buttervolume run')
            sys.exit(1)


def get_from(resp, key):
    """get specified key from plugin response output
    """
    try:  # bottle
        content = resp.content
    except:  # TestApp
        content = resp.body
    if resp.status_code != 200:
        logger.error('%s: %s', resp.status_code, resp.reason)
        sys.exit(1)
    else:
        error = jsonloads(content)['Err']
        if error:
            logger.error(error)
            sys.exit(1)
    return jsonloads(content).get(key)


def snapshot(args, test=False):
    urlpath = '/VolumeDriver.Snapshot'
    param = json.dumps({'Name': args.name[0]})
    if test:
        resp = TestApp(app).post(urlpath, param)
    else:
        resp = Session().post(
            'http+unix://{}{}'
            .format(urllib.parse.quote_plus(SOCKET), urlpath),
            param)
    res = get_from(resp, 'Snapshot') or sys.exit(1)
    if res:
        print(res)
        return res


def schedule(args):
    urlpath = '/VolumeDriver.Schedule'
    param = json.dumps({
        'Name': args.name[0],
        'Action': args.action[0],
        'Timer': args.timer[0]})
    Session().post(
        'http+unix://{}{}'
        .format(urllib.parse.quote_plus(SOCKET), urlpath), param)


def scheduled(args):
    urlpath = '/VolumeDriver.Schedule.List'
    resp = Session().get(
        'http+unix://{}{}'
        .format(urllib.parse.quote_plus(SOCKET), urlpath))
    scheduled = get_from(resp, 'Schedule')
    print('\n'.join(["{Action} {Timer} {Name}".format(**job)
                     for job in scheduled]))


def snapshots(args):
    resp = Session().post(
        'http+unix://{}/VolumeDriver.Snapshot.List'
        .format(urllib.parse.quote_plus(SOCKET)),
        json.dumps({'Name': args.name}))
    snapshots = get_from(resp, 'Snapshots')
    if snapshots is None:
        sys.exit(1)
    elif snapshots:
        print('\n'.join(snapshots))


def restore(args):
    resp = Session().post(
        'http+unix://{}/VolumeDriver.Snapshot.Restore'
        .format(urllib.parse.quote_plus(SOCKET)),
        json.dumps({'Name': args.name[0]}))
    res = get_from(resp, 'VolumeBackup')
    if res:
        print(res)
        return res


def send(args, test=False):
    urlpath = '/VolumeDriver.Snapshot.Send'
    param = {'Name': args.snapshot[0], 'Host': args.host[0]}
    if test:
        param['Test'] = True
        resp = TestApp(app).post(urlpath, json.dumps(param))
    else:
        resp = Session().post(
            'http+unix://{}{}'
            .format(urllib.parse.quote_plus(SOCKET), urlpath),
            json.dumps(param))
    res = get_from(resp, '')
    if res:
        print(res)


def remove(args):
    urlpath = '/VolumeDriver.Snapshot.Remove'
    param = json.dumps({'Name': args.name[0]})
    resp = Session().post(
        ('http+unix://{}{}')
        .format(urllib.parse.quote_plus(SOCKET), urlpath),
        param)
    res = get_from(resp, '')
    if res:
        print(res)


class Arg():
    def __init__(self, *a, **kw):
        for k, v in kw.items():
            setattr(self, k, v)


def scheduler(config=SCHEDULE, test=False):
    """Read the scheduler config and apply it, then scheduler again.
    WARNING: this should be guaranteed against runtime errors
    otherwise the next scheduler won't run
    """
    # open the config and launch the tasks
    if not os.path.exists(config):
        logger.warn('No config file %s', config)
        if not test:
            Timer(TIMER, scheduler).start()
        return
    name = action = timer = ''
    # run each action in the schedule if time is elapsed since the last one
    with open(config) as f:
        for line in csv.reader(f):
            try:
                name, action, timer = line
                now = datetime.now()
                # just starting, we consider beeing late on snapshots
                SCHEDULE_LOG.setdefault(action, {})
                SCHEDULE_LOG[action].setdefault(name, now - timedelta(1))
                last = SCHEDULE_LOG[action][name]
                if now < last + timedelta(minutes=int(timer)):<commit_msg>added a log for debug<commit_after>import argparse
import csv
import json
import logging
import os
import requests_unixsocket
from requests.exceptions import ConnectionError
import sys
import urllib
from bottle import app
from buttervolume.plugin import jsonloads, SCHEDULE
from buttervolume.plugin import SCHEDULE_LOG, SNAPSHOTS_PATH
from datetime import datetime, timedelta
from threading import Timer
from waitress import serve
from webtest import TestApp
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()
SOCKET = '/run/docker/plugins/btrfs.sock'
TIMER = 60
app = app()


class Session(object):
    """wrapper for requests_unixsocket.Session
    """
    def __init__(self):
        self.session = requests_unixsocket.Session()

    def post(self, *a, **kw):
        try:
            return self.session.post(*a, **kw)
        except ConnectionError:
            logger.error('Failed to connect to Buttervolume. '
                         'You can start it with: buttervolume run')
            sys.exit(1)

    def get(self, *a, **kw):
        try:
            return self.session.get(*a, **kw)
        except ConnectionError:
            logger.error('Failed to connect to Buttervolume. '
                         'You can start it with: buttervolume run')
            sys.exit(1)


def get_from(resp, key):
    """get specified key from plugin response output
    """
    try:  # bottle
        content = resp.content
    except:  # TestApp
        content = resp.body
    if resp.status_code != 200:
        logger.error('%s: %s', resp.status_code, resp.reason)
        sys.exit(1)
    else:
        error = jsonloads(content)['Err']
        if error:
            logger.error(error)
            sys.exit(1)
    return jsonloads(content).get(key)


def snapshot(args, test=False):
    urlpath = '/VolumeDriver.Snapshot'
    param = json.dumps({'Name': args.name[0]})
    if test:
        resp = TestApp(app).post(urlpath, param)
    else:
        resp = Session().post(
            'http+unix://{}{}'
            .format(urllib.parse.quote_plus(SOCKET), urlpath),
            param)
    res = get_from(resp, 'Snapshot') or sys.exit(1)
    if res:
        print(res)
        return res


def schedule(args):
    urlpath = '/VolumeDriver.Schedule'
    param = json.dumps({
        'Name': args.name[0],
        'Action': args.action[0],
        'Timer': args.timer[0]})
    Session().post(
        'http+unix://{}{}'
        .format(urllib.parse.quote_plus(SOCKET), urlpath), param)


def scheduled(args):
    urlpath = '/VolumeDriver.Schedule.List'
    resp = Session().get(
        'http+unix://{}{}'
        .format(urllib.parse.quote_plus(SOCKET), urlpath))
    scheduled = get_from(resp, 'Schedule')
    print('\n'.join(["{Action} {Timer} {Name}".format(**job)
                     for job in scheduled]))


def snapshots(args):
    resp = Session().post(
        'http+unix://{}/VolumeDriver.Snapshot.List'
        .format(urllib.parse.quote_plus(SOCKET)),
        json.dumps({'Name': args.name}))
    snapshots = get_from(resp, 'Snapshots')
    if snapshots is None:
        sys.exit(1)
    elif snapshots:
        print('\n'.join(snapshots))


def restore(args):
    resp = Session().post(
        'http+unix://{}/VolumeDriver.Snapshot.Restore'
        .format(urllib.parse.quote_plus(SOCKET)),
        json.dumps({'Name': args.name[0]}))
    res = get_from(resp, 'VolumeBackup')
    if res:
        print(res)
        return res


def send(args, test=False):
    urlpath = '/VolumeDriver.Snapshot.Send'
    param = {'Name': args.snapshot[0], 'Host': args.host[0]}
    if test:
        param['Test'] = True
        resp = TestApp(app).post(urlpath, json.dumps(param))
    else:
        resp = Session().post(
            'http+unix://{}{}'
            .format(urllib.parse.quote_plus(SOCKET), urlpath),
            json.dumps(param))
    res = get_from(resp, '')
    if res:
        print(res)


def remove(args):
    urlpath = '/VolumeDriver.Snapshot.Remove'
    param = json.dumps({'Name': args.name[0]})
    resp = Session().post(
        ('http+unix://{}{}')
        .format(urllib.parse.quote_plus(SOCKET), urlpath),
        param)
    res = get_from(resp, '')
    if res:
        print(res)


class Arg():
    def __init__(self, *a, **kw):
        for k, v in kw.items():
            setattr(self, k, v)


def scheduler(config=SCHEDULE, test=False):
    """Read the scheduler config and apply it, then scheduler again.
    WARNING: this should be guaranteed against runtime errors
    otherwise the next scheduler won't run
    """
    # open the config and launch the tasks
    if not os.path.exists(config):
        logger.warn('No config file %s', config)
        if not test:
            Timer(TIMER, scheduler).start()
        return
    name = action = timer = ''
    # run each action in the schedule if time is elapsed since the last one
    with open(config) as f:
        for line in csv.reader(f):
            try:
                name, action, timer = line
                now = datetime.now()
                # just starting, we consider beeing late on snapshots
                SCHEDULE_LOG.setdefault(action, {})
                SCHEDULE_LOG[action].setdefault(name, now - timedelta(1))
                last = SCHEDULE_LOG[action][name]
                logger.info("SCHEDULE_LOG {}".format(SCHEDULE_LOG))
                if now < last + timedelta(minutes=int(timer)):
{'fraction_non_alphanumeric': 0.13138821780338228, 'fraction_numerical': 0.003716781267422412, 'mean_word_length': 2.377157201129589, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6916623', 'n_tokens_mistral': 3325, 'n_tokens_neox': 3277, 'n_words': 825}