<commit_before>"""
Removes passwords from config files

"""


import unittest
import json
import re
from urlparse import urlparse, urlunparse
import traceback

def is_valid_json(invalid_json):
    """
    Is the given string valid JSON?
    """
    got_exception = True
    try:
        json.loads(invalid_json)
        got_exception = False
    except Exception as e:
        pass

    return got_exception == False

def remove_passwords_from_db_config(database):
    """
    Given a dictionary with database level config, remove all passwords and sensitive info
    """

    if not isinstance(database, dict):
        return

    if "server" in database:
        database["server"] = strip_password_from_url(database["server"])
    if "password" in database:
        database["password"] = "******"

    # # bucket shadowing
    # if "shadow" in database:
    #     shadow_settings = database["shadow"]
    #     if "server" in shadow_settings:
    #         shadow_settings["server"] = strip_password_from_url(shadow_settings["server"])
    #     if "password" in shadow_settings:
    #         shadow_settings["password"] = "******"
    #
    # # distributed index / sg accell
    # if "channel_index" in database:
    #     channel_index_settings = database["channel_index"]
    #     if "server" in channel_index_settings:
    #         channel_index_settings["server"] = strip_password_from_url(channel_index_settings["server"])
    #     if "password" in channel_index_settings:
    #         channel_index_settings["password"] = "******"

    for key, item in database.items():
        print "key: {0} item: {1}".format(key, item)
        if isinstance(item, dict):
            print "is a dictionary"
            remove_passwords_from_db_config(item)
            # walk(item)


def walk(node):

    for key, item in node.items():
        print "key: {0} item: {1}".format(key, item)
        if isinstance(item, dict):
            print "is a dictionary"
            remove_passwords_from_db_config(item)
            walk(item)


    # for key, item in node.items():
    #     if isinstance(item, dict):
    #         walk(item)
    #     else:
    #         remove_passwords_from_db_config(item)

def remove_passwords(json_text, log_json_parsing_exceptions=True):
    """
    Content postprocessor that strips out all of the sensitive passwords
    """
    try:

        # lower case everything so that "databases" works as a key even if the JSON has "Databases"
        # as a key.  seems like there has to be a better way!
        json_text = json_text.lower()

        valid_json = convert_to_valid_json(json_text)

        parsed_json = json.loads(valid_json)

        remove_passwords_from_db_config(parsed_json)

        # walk(parsed_json)

        # # if only a database config fragment was passed rather than a full SG config
        # # there will be top level "server" and "password" elements that need to be
        # # patches
        # remove_passwords_from_db_config(parsed_json)
        #
        # if "databases" in parsed_json:
        #     databases = parsed_json["databases"]
        #     for key, database in databases.iteritems():
        #         remove_passwords_from_db_config(database)

        formatted_json_string = json.dumps(parsed_json, indent=4)

        return formatted_json_string

    except Exception as e:
        msg = "Exception trying to remove passwords from {0}.  Exception: {1}".format(json_text, e)
        if log_json_parsing_exceptions:
            print(msg)
            traceback.print_exc()
        return '{"Error":"Error in sgcollect_info password_remover.py trying to remove passwords.  See logs for details"}'

def strip_password_from_url(url_string):
    """
    Given a URL string like:

    http://bucket-1:foobar@localhost:8091

    Strip out the password and return:

    http://bucket-1:@localhost:8091

    """<commit_msg>Code cleanup, tests still pass<commit_after>"""
Removes passwords from config files

"""


import unittest
import json
import re
from urlparse import urlparse, urlunparse
import traceback

def is_valid_json(invalid_json):
    """
    Is the given string valid JSON?
    """
    got_exception = True
    try:
        json.loads(invalid_json)
        got_exception = False
    except Exception as e:
        pass

    return got_exception == False

def remove_passwords_from_config(config_fragment):
    """
    Given a dictionary that contains configuration values, recursively walk the dictionary and:

    - Replace any fields w/ key "password" with "*****"
    - Replace any fields w/ key "server" with the result of running it through strip_password_from_url()
    """

    if not isinstance(config_fragment, dict):
        return

    if "server" in config_fragment:
        config_fragment["server"] = strip_password_from_url(config_fragment["server"])
    if "password" in config_fragment:
        config_fragment["password"] = "******"

    for key, item in config_fragment.items():
        if isinstance(item, dict):
            remove_passwords_from_config(item)


def remove_passwords(json_text, log_json_parsing_exceptions=True):
    """
    Content postprocessor that strips out all of the sensitive passwords
    """
    try:

        # lower case everything so that "databases" works as a key even if the JSON has "Databases"
        # as a key.  seems like there has to be a better way!
        json_text = json_text.lower()

        valid_json = convert_to_valid_json(json_text)

        parsed_json = json.loads(valid_json)

        remove_passwords_from_config(parsed_json)

        formatted_json_string = json.dumps(parsed_json, indent=4)

        return formatted_json_string

    except Exception as e:
        msg = "Exception trying to remove passwords from {0}.  Exception: {1}".format(json_text, e)
        if log_json_parsing_exceptions:
            print(msg)
            traceback.print_exc()
        return '{"Error":"Error in sgcollect_info password_remover.py trying to remove passwords.  See logs for details"}'

def strip_password_from_url(url_string):
    """
    Given a URL string like:

    http://bucket-1:foobar@localhost:8091

    Strip out the password and return:

    http://bucket-1:@localhost:8091

    """
{'fraction_non_alphanumeric': 0.09169467114738358, 'fraction_numerical': 0.004800768122899664, 'mean_word_length': 2.8532675709001234, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3797127', 'n_tokens_mistral': 1830, 'n_tokens_neox': 1772, 'n_words': 514}