<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#########################
# CONFIGURATION GOES HERE

# In production the filename is /etc/hosts.deny
FILENAME = '/etc/hosts.deny'

# Use this separator in file. Really no use in changing
SEPARATOR = '## AUTOGENERATED LIST'

# URL to blacklist
BLACKLIST_URL = 'http://www.openbl.org/lists/hosts.deny'


#################
# CODE GOES HERE!

import sys
import urllib2

def add_separator():
    """
    Append the string SEPARATOR to the end of hosts.deny-file
    """

    with open(FILENAME, 'a') as f:
        try:
            f.write('\n\n' + SEPARATOR)
        except Exception, e:
            print str(e)
            sys.exit(1)
        finally:
            print 'Added str "%s" to end of file %s' % (SEPARATOR, FILENAME)


def get_file():
    """
    Read content of hosts.deny file and return code that is generated
    static and dynamic. I.e. What is seperated by the string SEPARATOR
    """

    with open(FILENAME, 'r') as f:
        try:
            content = f.read()
        except Exception, e:
            print str(e)
            sys.exit(1)

        if SEPARATOR in content:
            static, dynamic = content.split(SEPARATOR)
            return static.strip(), dynamic.strip()
        else:
            add_separator()
            print '\nPlease run command again!'
            sys.exit(2)


def update_file():
    """
    Update hosts.deny-file with dynamic content
    This is the main function to run.
    """

    static, _ = get_file()

    try:
        dynamic = urllib2.urlopen(BLACKLIST_URL).read()
    except Exception, e:
        print str(e)
        print '\nCould not fetch url:', BLACKLIST_URL
        sys.exit(1)

    with open(FILENAME, 'w') as f:
        f.write(static)
        f.write('\n\n')
        f.write(SEPARATOR)
        f.write('\n\n')
        f.write(dynamic)
        f.write('\n')

    added_lines = dynamic.count('\n')
    print 'Number of lines added to %s: %s' % (FILENAME, added_lines)


if __name__ == '__main__':
    update_file()<commit_msg>Simplified and removed redundant code<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#########################
# CONFIGURATION GOES HERE

# In production the filename is /etc/hosts.deny
FILENAME = '/etc/hosts.deny'

# Use this separator in file. Really no use in changing
SEPARATOR = '## AUTOGENERATED LIST'

# URL to blacklist
BLACKLIST_URL = 'http://www.openbl.org/lists/hosts.deny'


#################
# CODE GOES HERE!

import sys
import urllib2


def get_file():
    """
    Read content of hosts.deny file and return code that is generated
    static and dynamic. I.e. What is seperated by the string SEPARATOR.

    Returns a tuple: (static_part, generated_part)
    """

    with open(FILENAME, 'r') as f:
        try:
            content = f.read()
        except Exception, e:
            print str(e)
            sys.exit(1)

        if SEPARATOR in content:
            static, dynamic = content.split(SEPARATOR)
            return static.strip(), dynamic.strip()
        else:
            return content.strip(), ''


def update_file():
    """
    Update hosts.deny-file with dynamic content
    This is the main function to run.
    """

    static, _ = get_file()

    try:
        dynamic = urllib2.urlopen(BLACKLIST_URL).read()
    except Exception, e:
        print str(e)
        print '\nCould not fetch url:', BLACKLIST_URL
        sys.exit(1)

    with open(FILENAME, 'w') as f:
        f.write(static + '\n\n')
        f.write(SEPARATOR + '\n\n')
        f.write(dynamic + '\n')

    added_lines = dynamic.count('\n')
    print 'Number of lines added to %s: %s' % (FILENAME, added_lines)


if __name__ == '__main__':
    update_file()
{'fraction_non_alphanumeric': 0.13879957127545553, 'fraction_numerical': 0.003215434083601286, 'mean_word_length': 2.257417102966841, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2057608', 'n_tokens_mistral': 1228, 'n_tokens_neox': 1181, 'n_words': 348}