<commit_before># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2015, Numenta, Inc.  Unless you have purchased from
# Numenta, Inc. a separate commercial license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------
"""
This file contains CSV utility functions to use with nupic.fluent experiments.
"""

import argparse
import os
import random
import string

from fluent.utils.csv_helper import readCSV, writeCSV



def cleanTokens(tokens):
    """ Traverses list of tokens and generates new list of tokens
    with [identifier deleted] as one token

    @param  (str)               List of tokens generated by splitting samples
                                by white space.

    @return (list)              List of tokens with '[identifier deleted]' as
                                one token instead of two.
    """
    cleanTokens = []
    for token in tokens:
      if token == "[identifier":
        cleanTokens.append("[identifier deleted]")
      elif token != "deleted]":
        cleanTokens.append(token)
    return cleanTokens


def generateDataFile(inputData, outputDataDir, type):
    """
    Generates a samples data file with all of the words in the sample
    reversed.

    @param  (str)               Path to input original samples data file
    @param  (TextPreprocess)    Processor to perform some text cleanup.

    """
    if not os.path.exists(outputDataDir):
        os.makedirs(outputDataDir)

    fileName = string.join(inputData.split(".")[:-1], ".") + "_" + type + ".csv"
    dataDict = readCSV(inputData, 2, 3)
    headers = ["QID", "QuestionText", "Response", "Classification1", "Classification2",
               "Classification3"]
    data = []
    for sample in dataDict.items():
      response = sample[1][0]
      tokens = response.split(" ")
      tokens = cleanTokens(tokens)

      response = None
      if type == "scrambled":
        random.shuffle(tokens)
        response = " ".join(tokens)
      elif type == "reversed":
        response = " ".join(tokens[::-1])

      dataToWrite = [sample[0], "", response]
      dataToWrite.extend(sample[1][1])
      data.append(dataToWrite)

    writeCSV(data, headers, os.path.join(outputDataDir, fileName))


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--inputData",
                        type=str,
                        required=True,
                        help="Path to input data file")
    parser.add_argument("--outputDataDir",
                        type=str,
                        default='data/sample_reviews_unit',
                        help="Desired directory for output data file")
    args = parser.parse_args()

    generateDataFile(args.inputData, args.outputDataDir, "scrambled")
    generateDataFile(args.inputData, args.outputDataDir, "reversed")
<commit_msg>fixup unit test data generation script<commit_after># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2015, Numenta, Inc.  Unless you have purchased from
# Numenta, Inc. a separate commercial license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------
"""
This file contains functions to modify the order of text sequences. Running the
script will create a CSV file where the words in each sequence have been
randomized, and another where their order has been reversed.
"""

import argparse
import os
import random
import string

from fluent.utils.csv_helper import readCSV, writeCSV



def cleanTokens(tokens):
    """
    Traverses list of tokens and generates new list of tokens
    with [identifier deleted] as one token

    @param  (str)               List of tokens generated by splitting samples
                                by white space.

    @return (list)              List of tokens with '[identifier deleted]' as
                                one token instead of two.
    """
    cleanTokens = []
    for token in tokens:
      if token == "[identifier":
        cleanTokens.append("[identifier deleted]")
      elif token != "deleted]":
        cleanTokens.append(token)
    return cleanTokens


def generateDataFile(inputData, outputDataDir, type):
    """
    Generates a samples data file with all of the words in the sample
    reversed.

    @param  (str)               Path to input original samples data file
    @param  (TextPreprocess)    Processor to perform some text cleanup.

    """
    if not os.path.exists(outputDataDir):
        os.makedirs(outputDataDir)

    fileName = string.join(inputData.split(".")[:-1], ".") + "_" + type + ".csv"
    dataDict = readCSV(inputData, 2, 3)
    headers = ["QID", "QuestionText", "Response", "Classification1",
               "Classification2", "Classification3"]
    data = []
    for sample in dataDict.items():
      response = sample[1][0]
      tokens = response.split(" ")
      tokens = cleanTokens(tokens)

      response = None
      if type == "scrambled":
        random.shuffle(tokens)
        response = " ".join(tokens)
      elif type == "reversed":
        response = " ".join(tokens[::-1])

      dataToWrite = [sample[0], "", response]
      dataToWrite.extend(sample[1][1])
      data.append(dataToWrite)

    writeCSV(data, headers, os.path.join(outputDataDir, fileName))


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("-fin", "--inputData",
                        type=str,
                        required=True,
                        help="Path to input data file")
    parser.add_argument("-fout", "--outputDataDir",
                        type=str,
                        default="generated_data',
                        help="Desired directory for output data file")
    args = parser.parse_args()

    generateDataFile(args.inputData, args.outputDataDir, "scrambled")
    generateDataFile(args.inputData, args.outputDataDir, "reversed")

{'fraction_non_alphanumeric': 0.130204801302048, 'fraction_numerical': 0.004611420046114201, 'mean_word_length': 2.3186318631863188, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '231661', 'n_tokens_mistral': 1923, 'n_tokens_neox': 1869, 'n_words': 678}