<commit_before>#!/usr/bin/env python

#   This file is part of Rogue Vision.
#
#   Copyright (C) 2016 <NAME>, <NAME>, <NAME>,
#       <NAME>, <NAME>, <NAME>,
#       <NAME>, <NAME>
#
#   Rogue Vision is free software: you can redistribute it and/or modify
#   it under the terms of the GNU Affero General Public License as published by
#   the Free Software Foundation, either version 3 of the License, or
#   (at your option) any later version.
#
#   Rogue Vision is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU Affero General Public License for more details.
#
#   You should have received a copy of the GNU Affero General Public License
#   along with Rogue Vision.  If not, see <http://www.gnu.org/licenses/>.

# Imports os
import os
# Imports logging
import logging
# Imports glob
import glob
# Import Configparser
import ConfigParser
# Imports Sys
import sys
# Imports SqLite
import sqlite3
# Imports ConfigParser
import ConfigParser

# Creates a TextFile "Running.txt" on Start to let writeCarrierDataToDataBase.py know that the script is still running
def createRunningFile():
    with open("Running.txt", "w") as text_file:
        text_file.write("Running")
        logging.info("compressInitialData.py now running. 'Running.txt' created.")


# Removes the status.txt file after the end of the simulation and writes its status to log file
def deleteRunningFile():
    os.remove("Running.txt")
    logging.info("Terminating compressInitialData.py. 'Running.txt removed.")

# Checks for .csv Files in particular folder and returns a list
def checkForCSVFilesInFolder(folder):
    dataFileNames = []
    for files in glob.glob(folder+"/*.csv"):
        dataFileNames.append(files)
    logging.info("Found " + str(len(dataFileNames)) + " new files.")
    return dataFileNames

# Function that checks if new csv files are in the folder
def check_folder(dataFileNames):

    # Returns False or True depending on whether or not files are stored in dataFileNames
    if not dataFileNames:
        logging.info("No new files found")
        return False
    else:
        logging.info(str(len(dataFileNames)) + " Files found")
        return True

# Updates the Config File
# Input: Section of ConfigFile, sectionValue, value
def updated_config(section, sectionValue, value):
    # Reads ConfigFile
    config = ConfigParser.ConfigParser()
    config.read('settings.cfg')

    # Sets the value of a particular section, sectionValue combination
    config.set(section, sectionValue, value)

    # Wirtes the new ConfigFile
    with open('settings.cfg', 'wb') as configfile:
        config.write(configfile)
<commit_msg>REFACTORING: Changed Path of config File to absoulute File<commit_after>#!/usr/bin/env python

#   This file is part of Rogue Vision.
#
#   Copyright (C) 2016 <NAME>, <NAME>, <NAME>,
#       <NAME>, <NAME>, <NAME>,
#       <NAME>, <NAME>
#
#   Rogue Vision is free software: you can redistribute it and/or modify
#   it under the terms of the GNU Affero General Public License as published by
#   the Free Software Foundation, either version 3 of the License, or
#   (at your option) any later version.
#
#   Rogue Vision is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU Affero General Public License for more details.
#
#   You should have received a copy of the GNU Affero General Public License
#   along with Rogue Vision.  If not, see <http://www.gnu.org/licenses/>.

# Imports os
import os
# Imports logging
import logging
# Imports glob
import glob
# Import Configparser
import ConfigParser
# Imports Sys
import sys
# Imports SqLite
import sqlite3
# Imports ConfigParser
import ConfigParser

# Creates a TextFile "Running.txt" on Start to let writeCarrierDataToDataBase.py know that the script is still running
def createRunningFile():
    with open("Running.txt", "w") as text_file:
        text_file.write("Running")
        logging.info("compressInitialData.py now running. 'Running.txt' created.")


# Removes the status.txt file after the end of the simulation and writes its status to log file
def deleteRunningFile():
    os.remove("Running.txt")
    logging.info("Terminating compressInitialData.py. 'Running.txt removed.")

# Checks for .csv Files in particular folder and returns a list
def checkForCSVFilesInFolder(folder):
    dataFileNames = []
    for files in glob.glob(folder+"/*.csv"):
        dataFileNames.append(files)
    logging.info("Found " + str(len(dataFileNames)) + " new files.")
    return dataFileNames

# Function that checks if new csv files are in the folder
def check_folder(dataFileNames):

    # Returns False or True depending on whether or not files are stored in dataFileNames
    if not dataFileNames:
        logging.info("No new files found")
        return False
    else:
        logging.info(str(len(dataFileNames)) + " Files found")
        return True

# Updates the Config File
# Input: Section of ConfigFile, sectionValue, value
def updated_config(section, sectionValue, value):
    # Reads ConfigFile
    config = ConfigParser.ConfigParser()
    config.read('settings.cfg')

    # Sets the value of a particular section, sectionValue combination
    config.set(section, sectionValue, value)

    # Wirtes the new ConfigFile
    with open('/srv/DataProcessing/settings.cfg', 'wb') as configfile:
        config.write(configfile)

{'fraction_non_alphanumeric': 0.08442139152208906, 'fraction_numerical': 0.0021463065641209086, 'mean_word_length': 3.7389830508474575, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2753240', 'n_tokens_mistral': 1554, 'n_tokens_neox': 1531, 'n_words': 630}