<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Created by <NAME>
This file is part of the CernVM File System auxiliary tools.
"""

import ctypes
import tempfile
import zlib
import sqlite3
import subprocess
import shutil


_REPO_CONFIG_PATH      = "/etc/cvmfs/repositories.d"
_SERVER_CONFIG_NAME    = "server.conf"

_MANIFEST_NAME         = ".cvmfspublished"
_WHITELIST_NAME        = ".cvmfswhitelist"
_LAST_REPLICATION_NAME = ".cvmfs_last_snapshot"
_REPLICATING_NAME      = ".cvmfs_is_snapshotting"


class CvmfsNotInstalled(Exception):
    def __init__(self):
        Exception.__init__(self, "It seems that cvmfs is not installed on this machine!")


class CompressedObject:
    file_            = None
    compressed_file_ = None

    def __init__(self, compressed_file):
        self.compressed_file_ = compressed_file
        self._decompress()

    def get_compressed_file(self):
        return self.compressed_file_

    def get_uncompressed_file(self):
        return self.file_

    def save_to(self, path):
        shutil.copyfile(self.get_compressed_file().name, path)

    def save_uncompressed_to(self, path):
        shutil.copyfile(self.get_uncompressed_file().name, path)

    def _decompress(self):
        """ Unzip a file to a temporary referenced by self.file_ """
        self.file_ = tempfile.NamedTemporaryFile('w+b')
        self.compressed_file_.seek(0)
        self.file_.write(zlib.decompress(self.compressed_file_.read()))
        self.file_.flush()
        self.file_.seek(0)
        self.compressed_file_.seek(0)

    def _close(self):
        if self.file_:
            self.file_.close()
        if self.compressed_file_:
            self.compressed_file_.close()


class FileObject(CompressedObject):
    def __init__(self, compressed_file):
        CompressedObject.__init__(self, compressed_file)

    def file(self):
        return self.get_uncompressed_file()<commit_msg>add _logistic_function() closure to _common<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Created by <NAME>
This file is part of the CernVM File System auxiliary tools.
"""

import ctypes
import tempfile
import zlib
import sqlite3
import subprocess
import shutil
import math


_REPO_CONFIG_PATH      = "/etc/cvmfs/repositories.d"
_SERVER_CONFIG_NAME    = "server.conf"

_MANIFEST_NAME         = ".cvmfspublished"
_WHITELIST_NAME        = ".cvmfswhitelist"
_LAST_REPLICATION_NAME = ".cvmfs_last_snapshot"
_REPLICATING_NAME      = ".cvmfs_is_snapshotting"


class CvmfsNotInstalled(Exception):
    def __init__(self):
        Exception.__init__(self, "It seems that cvmfs is not installed on this machine!")


class CompressedObject:
    file_            = None
    compressed_file_ = None

    def __init__(self, compressed_file):
        self.compressed_file_ = compressed_file
        self._decompress()

    def get_compressed_file(self):
        return self.compressed_file_

    def get_uncompressed_file(self):
        return self.file_

    def save_to(self, path):
        shutil.copyfile(self.get_compressed_file().name, path)

    def save_uncompressed_to(self, path):
        shutil.copyfile(self.get_uncompressed_file().name, path)

    def _decompress(self):
        """ Unzip a file to a temporary referenced by self.file_ """
        self.file_ = tempfile.NamedTemporaryFile('w+b')
        self.compressed_file_.seek(0)
        self.file_.write(zlib.decompress(self.compressed_file_.read()))
        self.file_.flush()
        self.file_.seek(0)
        self.compressed_file_.seek(0)

    def _close(self):
        if self.file_:
            self.file_.close()
        if self.compressed_file_:
            self.compressed_file_.close()


class FileObject(CompressedObject):
    def __init__(self, compressed_file):
        CompressedObject.__init__(self, compressed_file)

    def file(self):
        return self.get_uncompressed_file()


def _logistic_function(a):
    return lambda x: round(1 - (1/(1 + math.exp(-5.5 * ((float(x)/float(a)) - 1)))), 2)
{'fraction_non_alphanumeric': 0.1012216404886562, 'fraction_numerical': 0.004238344552480678, 'mean_word_length': 3.1489141675284387, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2723894', 'n_tokens_mistral': 1330, 'n_tokens_neox': 1278, 'n_words': 264}