<commit_before>import os
import time
import sys

from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

from vmmaster.core.utils.init import home_dir
from vmmaster.core.config import setup_config, config
setup_config('%s/config.py' % home_dir())
from vmmaster.core.db import Session, SessionLogStep
from vmmaster.core.utils.utils import change_user_vmmaster

from logging import getLogger, Formatter, StreamHandler, DEBUG


engine = create_engine(config.DATABASE)
session_factory = sessionmaker(bind=engine)


def setup_logging():
    l = getLogger('cleanup')
    ch = StreamHandler(sys.stdout)

    formatter = Formatter('%(asctime)s - %(levelname)s :: %(message)s')
    ch.setFormatter(formatter)

    l.addHandler(ch)
    l.setLevel(DEBUG)
    return l

log = setup_logging()


def transaction(func):
    def wrapper(*args, **kwargs):
        db_session = session_factory()
        try:
            return func(db_session=db_session, *args, **kwargs)
        except:
            db_session.rollback()
            raise
        finally:
            db_session.close()
    return wrapper


def old():
    d = time.time() - 60 * 60 * 24 * config.SCREENSHOTS_DAYS
    return d


@transaction
def old_sessions(db_session=None):
    return db_session.query(Session).filter(Session.time < old()).all()


def delete_files(sessions=None):
    from shutil import rmtree
    from errno import ENOENT

    for session in sessions:
        session_dir = os.path.join(config.SCREENSHOTS_DIR, str(session.id))
        try:
            rmtree(session_dir)
        except OSError as os_error:
            if os_error.errno != ENOENT:  # Ignore 'No such file or directory' error
                log.info('Unable to delete %s (%s)' % (str(session_dir), os_error.strerror))


@transaction
def delete_session_data(sessions=None, db_session=None):
    from datetime import datetime, timedelta
    sessions_count = len(sessions)

    log.info("Got %s sessions. " % str(sessions_count))
    if sessions_count:
        first_id = sessions[0].id
        last_id = sessions[-1].id
        checkpoint = datetime.now()
        time_step = timedelta(days=0, seconds=10)

        log.info("Done: %s%% (0 / %d)" % ('0.0'.rjust(5), sessions_count))
        for num, session in enumerate(sessions):
            delta = datetime.now() - checkpoint
            if delta > time_step or num == sessions_count - 1:  # Show deletion progress each 10 seconds
                percentage = str(round((num + 1)/float(sessions_count) * 100, 1))
                log.info("Done: %s%% (%d / %d)" % (percentage.rjust(5), num + 1, sessions_count))
                checkpoint = datetime.now()
            db_session.delete(session)
        db_session.commit()
        delete_files(sessions)<commit_msg>Separate commit and screenshot delete for each session<commit_after>import os
import time
import sys

from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

from vmmaster.core.utils.init import home_dir
from vmmaster.core.config import setup_config, config
setup_config('%s/config.py' % home_dir())
from vmmaster.core.db import Session, SessionLogStep
from vmmaster.core.utils.utils import change_user_vmmaster

from logging import getLogger, Formatter, StreamHandler, DEBUG


engine = create_engine(config.DATABASE)
session_factory = sessionmaker(bind=engine)


def setup_logging():
    l = getLogger('cleanup')
    ch = StreamHandler(sys.stdout)

    formatter = Formatter('%(asctime)s - %(levelname)s :: %(message)s')
    ch.setFormatter(formatter)

    l.addHandler(ch)
    l.setLevel(DEBUG)
    return l

log = setup_logging()


def transaction(func):
    def wrapper(*args, **kwargs):
        db_session = session_factory()
        try:
            return func(db_session=db_session, *args, **kwargs)
        except:
            db_session.rollback()
            raise
        finally:
            db_session.close()
    return wrapper


def old():
    d = time.time() - 60 * 60 * 24 * config.SCREENSHOTS_DAYS
    return d


@transaction
def old_sessions(db_session=None):
    return db_session.query(Session).filter(Session.time < old()).all()


def delete_files(session=None):
    from shutil import rmtree
    from errno import ENOENT

    if session:
        session_dir = os.path.join(config.SCREENSHOTS_DIR, str(session.id))
        try:
            rmtree(session_dir)
        except OSError as os_error:
            if os_error.errno != ENOENT:  # Ignore 'No such file or directory' error
                log.info('Unable to delete %s (%s)' % (str(session_dir), os_error.strerror))


@transaction
def delete_session_data(sessions=None, db_session=None):
    from datetime import datetime, timedelta
    sessions_count = len(sessions)

    log.info("Got %s sessions. " % str(sessions_count))
    if sessions_count:
        first_id = sessions[0].id
        last_id = sessions[-1].id
        checkpoint = datetime.now()
        time_step = timedelta(days=0, seconds=10)

        log.info("Done: %s%% (0 / %d)" % ('0.0'.rjust(5), sessions_count))
        for num, session in enumerate(sessions):
            delta = datetime.now() - checkpoint
            if delta > time_step or num == sessions_count - 1:  # Show deletion progress each 10 seconds
                percentage = str(round((num + 1)/float(sessions_count) * 100, 1))
                log.info("Done: %s%% (%d / %d)" % (percentage.rjust(5), num + 1, sessions_count))
                checkpoint = datetime.now()
            delete_files(session)
            db_session.delete(session)
            db_session.commit()
{'fraction_non_alphanumeric': 0.10987831066571224, 'fraction_numerical': 0.00894774516821761, 'mean_word_length': 2.969460227272727, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5175174', 'n_tokens_mistral': 1779, 'n_tokens_neox': 1693, 'n_words': 420}