<commit_before>import argparse
import os
import unicodecsv
import random
from six.moves import urllib
import tarfile
import csv

import nltk
from nltk.stem import SnowballStemmer, WordNetLemmatizer

from pugnlp.futil import find_files
__author__ = 'rkadlec'

"""
Put all Ubuntu Dialog sessions into a single TSV.gz file (0.5 GB compressed)

Adapted from https://github.com/rkadlec/ubuntu-ranking-dataset-creator/blob/master/src/create_ubuntu_dataset.py
Script for generation of train, test and valid datasets from Ubuntu Corpus 1 on 1 dialogs.
Copyright IBM Corporation 2016
LICENSE: Apache License 2.0  URL: ttp://www.apache.org/licenses/LICENSE-2.0
Contact: <NAME> (<EMAIL>)
"""

dialog_end_symbol = "__dialog_end__"
end_of_utterance_symbol = "__eou__"
end_of_turn_symbol = "__eot__"


def translate_dialog_to_lists(dialog_filename):
    """
    Translates the dialog to a list of lists of utterances. In the first
    list each item holds subsequent utterances from the same user. The second level
    list holds the individual utterances.
    :param dialog_filename:
    :return:
    """

    dialog_file = open(dialog_filename, 'r')
    dialog_reader = unicodecsv.reader(dialog_file, delimiter='\t', quoting=csv.QUOTE_NONE)

    # go through the dialog
    first_turn = True
    dialog = []
    same_user_utterances = []
    #last_user = None
    dialog.append(same_user_utterances)

    for dialog_line in dialog_reader:

        if first_turn:
            last_user = dialog_line[1]
            first_turn = False

        if last_user != dialog_line[1]:
            # user has changed
            same_user_utterances = []
            dialog.append(same_user_utterances)

        same_user_utterances.append(dialog_line[3])

        last_user = dialog_line[1]

    dialog.append([dialog_end_symbol])

    return dialog


def get_random_utterances_from_corpus(candidate_dialog_paths, rng, utterances_num=9, min_turn=3, max_turn=20):
    """
    Sample multiple random utterances from the whole corpus.
    :param candidate_dialog_paths:
    :param rng:
    :param utterances_num: number of utterances to generate
    :param min_turn: minimal index of turn that the utterance is selected from
    :return:
    """
    utterances = []
    dialogs_num = len(candidate_dialog_paths)

    for i in xrange(0, utterances_num):
        # sample random dialog
        dialog_path = candidate_dialog_paths[rng.randint(0, dialogs_num - 1)]
        # load the dialog
        dialog = translate_dialog_to_lists(dialog_path)

        # we do not count the last  _dialog_end__ urn
        dialog_len = len(dialog) - 1
        if(dialog_len < min_turn):
            print "Dialog {} was shorter than the minimum required lenght {}".format(dialog_path, dialog_len)
            exit()
        # sample utterance, exclude the last round that is always "dialog end"
        max_ix = min(max_turn, dialog_len) - 1

        # this code deals with corner cases when dialogs are too short
        if min_turn - 1 == max_ix:
            turn_index = max_ix
        else:
            turn_index = rng.randint(min_turn, max_ix)

        utterance = singe_user_utterances_to_string(dialog[turn_index])
        utterances.append(utterance)
    return utterances


def singe_user_utterances_to_string(utterances_list):
    """
    Concatenates multiple user's utterances into a single string.<commit_msg>fixed syntax error on Line 90<commit_after>import argparse
import os
import unicodecsv
import random
from six.moves import urllib
import tarfile
import csv

import nltk
from nltk.stem import SnowballStemmer, WordNetLemmatizer

from pugnlp.futil import find_files
__author__ = 'rkadlec'

"""
Put all Ubuntu Dialog sessions into a single TSV.gz file (0.5 GB compressed)

Adapted from https://github.com/rkadlec/ubuntu-ranking-dataset-creator/blob/master/src/create_ubuntu_dataset.py
Script for generation of train, test and valid datasets from Ubuntu Corpus 1 on 1 dialogs.
Copyright IBM Corporation 2016
LICENSE: Apache License 2.0  URL: ttp://www.apache.org/licenses/LICENSE-2.0
Contact: <NAME> (<EMAIL>)
"""

dialog_end_symbol = "__dialog_end__"
end_of_utterance_symbol = "__eou__"
end_of_turn_symbol = "__eot__"


def translate_dialog_to_lists(dialog_filename):
    """
    Translates the dialog to a list of lists of utterances. In the first
    list each item holds subsequent utterances from the same user. The second level
    list holds the individual utterances.
    :param dialog_filename:
    :return:
    """

    dialog_file = open(dialog_filename, 'r')
    dialog_reader = unicodecsv.reader(dialog_file, delimiter='\t', quoting=csv.QUOTE_NONE)

    # go through the dialog
    first_turn = True
    dialog = []
    same_user_utterances = []
    #last_user = None
    dialog.append(same_user_utterances)

    for dialog_line in dialog_reader:

        if first_turn:
            last_user = dialog_line[1]
            first_turn = False

        if last_user != dialog_line[1]:
            # user has changed
            same_user_utterances = []
            dialog.append(same_user_utterances)

        same_user_utterances.append(dialog_line[3])

        last_user = dialog_line[1]

    dialog.append([dialog_end_symbol])

    return dialog


def get_random_utterances_from_corpus(candidate_dialog_paths, rng, utterances_num=9, min_turn=3, max_turn=20):
    """
    Sample multiple random utterances from the whole corpus.
    :param candidate_dialog_paths:
    :param rng:
    :param utterances_num: number of utterances to generate
    :param min_turn: minimal index of turn that the utterance is selected from
    :return:
    """
    utterances = []
    dialogs_num = len(candidate_dialog_paths)

    for i in xrange(0, utterances_num):
        # sample random dialog
        dialog_path = candidate_dialog_paths[rng.randint(0, dialogs_num - 1)]
        # load the dialog
        dialog = translate_dialog_to_lists(dialog_path)

        # we do not count the last  _dialog_end__ urn
        dialog_len = len(dialog) - 1
        if(dialog_len < min_turn):
            print ("Dialog {} was shorter than the minimum required length {}".format(dialog_path, dialog_len))
            exit()
        # sample utterance, exclude the last round that is always "dialog end"
        max_ix = min(max_turn, dialog_len) - 1

        # this code deals with corner cases when dialogs are too short
        if min_turn - 1 == max_ix:
            turn_index = max_ix
        else:
            turn_index = rng.randint(min_turn, max_ix)

        utterance = singe_user_utterances_to_string(dialog[turn_index])
        utterances.append(utterance)
    return utterances


def singe_user_utterances_to_string(utterances_list):
    """
    Concatenates multiple user's utterances into a single string.
{'fraction_non_alphanumeric': 0.07404129793510325, 'fraction_numerical': 0.007964601769911504, 'mean_word_length': 3.254077791718946, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4887011', 'n_tokens_mistral': 2135, 'n_tokens_neox': 2033, 'n_words': 588}