<commit_before>from __future__ import print_function
from __future__ import division

import os
import errno
import pickle
import shutil
import random
import time

from contextlib import contextmanager
from datetime import datetime

import numpy as np
from pyquaternion import Quaternion
import cv2
from tqdm import tqdm
from scipy import spatial

import consts


def get_timestamp():
    return datetime.now().strftime("%Y_%m_%d-%H_%M_%S")


class SessionInfo:
    def __init__(self, title, out_dir=None, suffix=None):
        self.title = title

        self.out_dir = (self.title + '_' + get_timestamp()) if out_dir is None else out_dir
        if suffix is not None:
            self.out_dir += suffix


def get_meshNet_session_info(mesh_name, model_type, roi, epochs, grid_step, test_only, load_weights, x_type, y_type,
                             mess):
    title = "meshNet"

    suffix = "_%s%s_%s_%sEpochs_%s_ROI_%s_%s_%s_%s_GridStep%s_%s_%s%s" % \
             ('Test' if test_only else 'Train', 'Resume' if load_weights else '', model_type, epochs, mesh_name,
              roi[0], roi[1], roi[2], roi[3], grid_step, y_type, x_type, '_MESS' if mess else '')

    sess_info = SessionInfo(title, suffix=suffix)
    return sess_info


def mkdirs(full_path):
    """Create all directories in the given path if they does not exist"""

    if not os.path.exists(full_path):
        try:
            os.makedirs(full_path)
        except OSError as exc:  # Guard against race condition
            if exc.errno != errno.EEXIST:
                raise


@contextmanager
def cd(new_dir):
    prev_dir = os.getcwd()
    os.chdir(os.path.expanduser(new_dir))
    try:
        yield
    finally:
        os.chdir(prev_dir)
<commit_msg>Add fine-tune<commit_after>from __future__ import print_function
from __future__ import division

import os
import errno
import pickle
import shutil
import random
import time

from contextlib import contextmanager
from datetime import datetime

import numpy as np
from pyquaternion import Quaternion
import cv2
from tqdm import tqdm
from scipy import spatial

import consts


def get_timestamp():
    return datetime.now().strftime("%Y_%m_%d-%H_%M_%S")


class SessionInfo:
    def __init__(self, title, out_dir=None, suffix=None):
        self.title = title

        self.out_dir = (self.title + '_' + get_timestamp()) if out_dir is None else out_dir
        if suffix is not None:
            self.out_dir += suffix


def get_meshNet_session_info(mesh_name, model_type, roi, epochs, grid_step, test_only, load_weights, x_type, y_type,
                             mess, fine_tune):
    title = "meshNet"

    if fine_tune and not load_weights:
        raise Exception("Fine-tuning initial weight does not make much sense")
    resume_and_fine_tune = 'Finetune' if load_weights and fine_tune else 'Resume' if load_weights else ''

    suffix = "_%s%s_%s_%sEpochs_%s_ROI_%s_%s_%s_%s_GridStep%s_%s_%s%s" % \
             ('Test' if test_only else 'Train', resume_and_fine_tune, model_type, epochs, mesh_name,
              roi[0], roi[1], roi[2], roi[3], grid_step, y_type, x_type, '_MESS' if mess else '')

    sess_info = SessionInfo(title, suffix=suffix)
    return sess_info


def mkdirs(full_path):
    """Create all directories in the given path if they does not exist"""

    if not os.path.exists(full_path):
        try:
            os.makedirs(full_path)
        except OSError as exc:  # Guard against race condition
            if exc.errno != errno.EEXIST:
                raise


@contextmanager
def cd(new_dir):
    prev_dir = os.getcwd()
    os.chdir(os.path.expanduser(new_dir))
    try:
        yield
    finally:
        os.chdir(prev_dir)

{'fraction_non_alphanumeric': 0.09182015167930661, 'fraction_numerical': 0.0027085590465872156, 'mean_word_length': 3.117056856187291, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3344276', 'n_tokens_mistral': 1251, 'n_tokens_neox': 1219, 'n_words': 346}