<commit_before>def clean(old_logfile, new_logfile, target_labels):
    old = open(old_logfile, 'r')
    new = open(new_logfile, 'w')

    normal_count = 0
    ddos_count = 0
    others_count = 0

    while True:
        log_string = old.readline()
        if not log_string:
            break
        features = log_string.split(',')
        label = features[-1]
        if label[:-2] in target_labels:
            if label[:-2] == "normal":
                normal_count += 1
                new.writelines([log_string])
            else:
                ddos_count += 1
                features[-1] = "ddos.\n"
                log_string = ','.join(features)
                new.writelines([log_string])
        else:
            others_count += 1
            continue
    print("normal log count: ", normal_count)
    print("ddos log count: ", ddos_count)
    print("other log count: ", others_count)
    old.close()
    new.close()


def main():
    target_labels = ["normal", "apache2", "back", "land", "mailbomb", "neptune", "pod", "processtable", "smurf",
                    "teardrop", "udpstorm"]
    clean("../data/kddcup.data_10_percent_corrected.log", "../data/cleandata.log", target_labels)

if __name__ == '__main__':
    main()<commit_msg>add a module to clean dataset from KDD<commit_after>def clean(old_logfile, new_logfile, target_labels):
    old = open(old_logfile, 'r')
    new = open(new_logfile, 'w')

    normal_count = 0
    ddos_count = 0
    others_count = 0

    while True:
        log_string = old.readline()
        if not log_string:
            break
        features = log_string.split(',')
        label = features[-1]
        if label[:-2] in target_labels:
            if label[:-2] == "normal":
                normal_count += 1
                new.writelines([log_string])
            else:
                ddos_count += 1
                features[-1] = "ddos.\n"
                log_string = ','.join(features)
                new.writelines([log_string])
        else:
            others_count += 1
            continue
    print("normal log count: ", normal_count)
    print("ddos log count: ", ddos_count)
    print("other log count: ", others_count)
    old.close()
    new.close()


def main():
    target_labels = ["normal", "apache2", "back", "land", "mailbomb", "neptune", "pod", "processtable", "smurf",
                    "teardrop", "udpstorm"]
    clean("../data/kddcup.data_10_percent_corrected.log", "../data/labeled_ddos_logs.log", target_labels)

if __name__ == '__main__':
    main()
{'fraction_non_alphanumeric': 0.13774104683195593, 'fraction_numerical': 0.01023219205037387, 'mean_word_length': 2.0552884615384617, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5752501', 'n_tokens_mistral': 837, 'n_tokens_neox': 791, 'n_words': 176}