<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import json
import riak
import memcache

from utils import pandas_to_dict

from pandas import DataFrame
from sqlalchemy import create_engine


mc = memcache.Client(['127.0.0.1:11211'], debug=0)
mc.delete('testando')
mc.delete('testando-columns')

myClient = riak.RiakClient(protocol='http', http_port=8098, host='127.0.0.1')
myBucket = myClient.bucket('openmining')

c = 'mysql://root:123mudar@192.168.12.4/upessencia_dev1'
e = create_engine('mysql://root:123mudar@192.168.12.4/upessencia_dev1')
connection = e.connect()

sql = """SELECT 
cliente.id_cliente,
cliente.nome,
pedido.id_pedido,
pedido.criacao_ts as 'pedido_data',
cliente.criacao_ts as 'cliente_data' 
FROM pedido 
inner join cliente on cliente.id_cliente = pedido.id_cliente 
WHERE pedido.id_pedido_status NOT IN (4,8,9) 
AND pedido.entrega_forma<>'franquia' limit 100"""

resoverall = connection.execute(sql)

df = DataFrame(resoverall.fetchall())
df.columns = resoverall.keys()
df.head()

convert = pandas_to_dict(df)

join = json.dumps(convert)
b1 = myBucket.new('testando', data=join)
b1.store()

b2 = myBucket.new('testando-columns', data=json.dumps([c for c in df.columns]))
b2.store()

b3 = myBucket.new('testando-connect', data=c)
b3.store()

b4 = myBucket.new('testando-sql', data=sql)
b4.store()<commit_msg>Fix pep8 on bin file<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import json
import riak
import memcache

from pandas import DataFrame
from sqlalchemy import create_engine

from utils import pandas_to_dict

mc = memcache.Client(['127.0.0.1:11211'], debug=0)
mc.delete('testando')
mc.delete('testando-columns')

myClient = riak.RiakClient(protocol='http', http_port=8098, host='127.0.0.1')
myBucket = myClient.bucket('openmining')

c = 'mysql://root:123mudar@192.168.12.4/upessencia_dev1'
e = create_engine('mysql://root:123mudar@192.168.12.4/upessencia_dev1')
connection = e.connect()

sql = """SELECT
cliente.id_cliente,
cliente.nome,
pedido.id_pedido,
pedido.criacao_ts as 'pedido_data',
cliente.criacao_ts as 'cliente_data'
FROM pedido
inner join cliente on cliente.id_cliente = pedido.id_cliente
WHERE pedido.id_pedido_status NOT IN (4,8,9)
AND pedido.entrega_forma<>'franquia' limit 100"""

resoverall = connection.execute(sql)

df = DataFrame(resoverall.fetchall())
df.columns = resoverall.keys()
df.head()

convert = pandas_to_dict(df)

join = json.dumps(convert)
b1 = myBucket.new('testando', data=join)
b1.store()

b2 = myBucket.new('testando-columns', data=json.dumps([c for c in df.columns]))
b2.store()

b3 = myBucket.new('testando-connect', data=c)
b3.store()

b4 = myBucket.new('testando-sql', data=sql)
b4.store()
{'fraction_non_alphanumeric': 0.1527001862197393, 'fraction_numerical': 0.047299813780260706, 'mean_word_length': 5.8346055979643765, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 8, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5050231', 'n_tokens_mistral': 1115, 'n_tokens_neox': 1051, 'n_words': 199}