<commit_before>import csv
import MySQLdb
import sys
import logging
import os
from appscale_user_client import AppscaleUserClient

import time


def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs):
    csv_reader = csv.reader(utf8_data, dialect=dialect, **kwargs)
    for row in csv_reader:
        yield [unicode(cell, 'utf-8') for cell in row]

start_time = time.time()

user_client = AppscaleUserClient()

logging.basicConfig(filename='app.log', level=logging.DEBUG)

if len(sys.argv) < 2:
    print('Specify file full path containing data')
    sys.exit(1)

file_path = sys.argv[1]

mydb = MySQLdb.connect(host='localhost',
                       user='root',
                       passwd='<PASSWORD>',
                       db='iupds_db')
cursor = mydb.cursor()

# csv_data = csv.reader(open(file_path, 'r'), delimiter=';')
csv_data = unicode_csv_reader(open(file_path))

for row in csv_data:
    print row
    exit(1)
    logging.debug(row)

    # data in row:
    # ['user_id', 'name', 'username', 'email','pwd', 'user_type', '0', '0', '18','created_at', 'last_login', '', '\n']

    user_id_old = str(row[0])
    fullname = str(row[1])
    username = str(row[2])
    email = str(row[3])
    created_at = str(row[9])

    # check for duplicate
    query = ("SELECT * FROM iupdsmanager_profile WHERE email = '" + email + "' OR username = '" + username + "'")
    cursor.execute(query)
    data = cursor.fetchall()

    sql = ""
    if len(data) == 0:

        try:
            if user_client.does_user_exist(email):
                user_client.disable_user(email)
                user_client.delete_user(email)
            else:
                print "User does not exist, proceed "
                logging.info("User does not exist, proceed ")

            # call uaserver to create user
            response = user_client.create_user(email, os.urandom(7))
            print "Credentials "+str(email) + " -> " + str(os.urandom(7)) + "\n"
            if response:
                user_data = user_client.get_user_data(email)

                sql = "INSERT INTO iupdsmanager_profile(uid, user_id_old, email, username, full_name," \
                      " created_at, is_active, admin_type,first_name, last_name, is_admin,updated_at," \<commit_msg>final fix for encoding issue<commit_after>import csv
import MySQLdb
import sys
import logging
import os
from appscale_user_client import AppscaleUserClient

import time

start_time = time.time()

user_client = AppscaleUserClient()

logging.basicConfig(filename='app.log', level=logging.DEBUG)

if len(sys.argv) < 2:
    print('Specify file full path containing data')
    sys.exit(1)

file_path = sys.argv[1]

mydb = MySQLdb.connect(host='localhost',
                       user='root',
                       passwd='<PASSWORD>',
                       db='iupds_db')
cursor = mydb.cursor()

csv_data = csv.reader(open(file_path, 'r'), delimiter=';')

for row in csv_data:
    print row
    logging.debug(row)

    # data in row:
    # ['user_id', 'name', 'username', 'email','pwd', 'user_type', '0', '0', '18','created_at', 'last_login', '', '\n']

    user_id_old = str(row[0])
    fullname = str(row[1]).upper().decode('utf-8').upper().encode('utf-8')
    username = str(row[2])
    email = str(row[3])
    created_at = str(row[9])

    # check for duplicate
    query = ("SELECT * FROM iupdsmanager_profile WHERE email = '" + email + "' OR username = '" + username + "'")
    cursor.execute(query)
    data = cursor.fetchall()

    sql = ""
    if len(data) == 0:

        try:
            if user_client.does_user_exist(email):
                user_client.disable_user(email)
                user_client.delete_user(email)
            else:
                print "User does not exist, proceed "
                logging.info("User does not exist, proceed ")

            # call uaserver to create user
            response = user_client.create_user(email, os.urandom(7))
            print "Credentials "+str(email) + " -> " + str(os.urandom(7)) + "\n"
            if response:
                user_data = user_client.get_user_data(email)

                sql = "INSERT INTO iupdsmanager_profile(uid, user_id_old, email, username, full_name," \
                      " created_at, is_active, admin_type,first_name, last_name, is_admin,updated_at," \
{'fraction_non_alphanumeric': 0.13223714682723484, 'fraction_numerical': 0.008337193144974525, 'mean_word_length': 2.6263643996641477, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1347292', 'n_tokens_mistral': 1404, 'n_tokens_neox': 1345, 'n_words': 310}