<commit_before># -*- coding: utf-8 -*-

import boto3
import json
import time
from collections import OrderedDict
import re
import csv
from datetime import datetime, date, timedelta
import requests
from flask import Flask, render_template, url_for, request, redirect, g, jsonify, flash, session, Markup
from run import app
from pymongo import *
from bs4 import BeautifulSoup
from settings import *

client = MongoClient(MONGODB_DATABASE['uri'])
db = client[MONGODB_DATABASE['database_name']]

in_progress = False

def get_country_codes():
    payload = dict(
        api_key=API_KEY,
        )

    endpoint_string = 'http://' + ADMIN_DOMAIN_URL + '/api/1/get.asmx/Currencies'
    soup = requests.post(endpoint_string,json=payload)
    r = soup.json()

    country_codes = {}

    for x in r["d"]["currencies"]:
        abbreviation = x["currency_abbr"]
        currency_id = str(x["currency_id"])
        country_codes[currency_id] = abbreviation

    return country_codes

country_codes = get_country_codes()

def return_currency_name(country_id, country_codes):
    country_id = str(country_id)
    country_name = ''.join({value for key, value in country_codes.items() if country_id == key})
    return country_name


def receive_message():

    in_progress = True
    client = boto3.client('sqs')
    queue_size_response = client.get_queue_attributes(QueueUrl= SQS_QUEUE['url'],
                                                AttributeNames=['ApproximateNumberOfMessages'])
    queue_size = queue_size_response["Attributes"]["ApproximateNumberOfMessages"]
    if queue_size != "0":
        response = OrderedDict(client.receive_message(QueueUrl = SQS_QUEUE['url'],
                                            AttributeNames=['Body'],
                                            MaxNumberOfMessages=1))
        return response
    elif queue_size == "0":
        response = "No Messages in Queue"
        return response

def delete_message(receipt_handle):
    client = boto3.client('sqs')
    response = client.delete_message(QueueUrl= SQS_QUEUE['url'],
                                     ReceiptHandle=receipt_handle)
    return response

def enumerate_dates(start, end):
    start_date = datetime(start.year, start.month, start.day)
    end_date = datetime(end.year, end.month, end.day)
    delta = end_date - start_date

    for i in range(delta.days + 1):
        next_start_date = start_date + timedelta(days=1)
        for i in range(48):
            next_start_hour = start_date + timedelta(minutes=30)
            yield start_date, next_start_hour
            start_date = next_start_hour
        start_date=next_start_date

def date_convert_for_csv(date):
    extract_integers = re.findall('\d+', date)
    date_string = ''.join(extract_integers)
    if len(date_string) > 10:
        date_string = date_string[:10] + '.' + date_string[10:]
        date_result = (datetime.utcfromtimestamp(float(date_string)) + timedelta(hours=1)).strftime("%d-%m-%YT%H:%M:%S.%f")
        return date_result
    else:
        timestamp_parsed = (datetime.utcfromtimestamp(int(date_string))+ timedelta(hours=1)) + '.000000'
        date_result = timestamp_parsed.strftime("%d-%m-%YT%H:%M:%S.%f")
        return date_result

def conversion_time_delta(conversion_date, click_date):
    conversion_date = datetime.strptime(conversion_date, "%d-%m-%YT%H:%M:%S.%f")
    click_date = datetime.strptime(click_date, "%d-%m-%YT%H:%M:%S.%f")

    if click_date > conversion_date:
        time_delta = "1"
        return time_delta

    elif conversion_date > click_date:
        time_delta = conversion_date - click_date
        if time_delta.seconds < 1:
            time_delta = "1"
            return time_delta
        else:
            return time_delta

def s3_job(filename):
# expire 86400 seconds is 24 hours
    s3 = boto3.resource('s3')
    s3.meta.client.upload_file('temp.csv', S3_BUCKET['name'], '%s.csv' % filename)

    client = boto3.client('s3')
    url = client.generate_presigned_url('get_object',
                                        Params={'Bucket': S3_BUCKET['name'],'Key': '%s.csv' % filename},
                                        ExpiresIn=86400)
    return url


def execute_call(response):

    body = (response["Messages"][0]["Body"]).replace("'", "\"")
    load_body = json.loads(body)

    start_date = load_body['start_date']
    end_date = load_body['end_date']
    job_id = load_body['job_id']
    created_date = load_body['created_date']
    receipt_handle = response["Messages"][0]["ReceiptHandle"]
    delete_message(receipt_handle)

    start_datetime = datetime.strptime(start_date, "%m/%d/%y")
    end_datetime = datetime.strptime(end_date, "%m/%d/%y")

    collection_name = db[MONGODB_DATABASE['collection_name']]
    collection_name.update_one({"created_date": created_date}, {"$set": {"status": "In Progress"}})


    try:
        with open('temp.csv', 'wb') as text_file:
            writer = csv.writer(text_file)
            writer.writerow(('Conversion ID',
                            'Last Updated',
                            'Conversion Date',
                            'Click Date',
                            'Click to Conversion Time',
                            'Affiliate ID',
                            'Affiliate Name',
                            'Advertiser ID',
                            'Advertiser Name',
                            'Offer ID',
                            'Offer Name',
                            'Creative',
                            'Sub ID',
                            'Sub ID 2',
                            'Sub ID 3',
                            'Sub ID 4',
                            'Sub ID 5',
                            'Type',
                            'Price Paid Currency',
                            'Paid',
                            'Price Received Currency',
                            'Received',
                            'Pixel',
                            'Transaction ID',
                            'IP Address',
                            'Click IP Address',
                            'Country',
                            'Conversion Referrer',
                            'Referrer',
                            'Conversion User Agent',
                            'Click User Agent',
                            'Disposition',
                            'Region',
                            'Language',
                            'Provider Name',
                            'Device',
                            'Operating System',
                            'OS(Major Version)',
                            'OS(Minor Version)',
                            'Browser',
                            'Browser(Major Version)',
                            'Browser(Minor Versions)',
                            'Conversion Score',
                            'Paid Unbilled',
                            'Received Unbilled',
                            'Click Request Session ID',
                            'Event Name',
                            'Price Format',
                            'Tracking ID',
                            'UDID'))


            date_generator = enumerate_dates(start_datetime, end_datetime)

            for start, end in date_generator:
                start = datetime.strftime(start, '%m/%d/%Y %H:%M')
                end = datetime.strftime(end, '%m/%d/%Y %H:%M')
                print(start, end)
                payload = dict(
                    api_key=API_KEY,
                    start_date=start,
                    end_date=end,
                    conversion_type='all',
                    event_type='macro_event_conversions',
                    event_id=0,
                    channel_id=0,
                    source_affiliate_id=0,
                    brand_advertiser_id=0,
                    site_offer_id=0,
                    site_offer_contract_id=0,
                    source_affiliate_tag_id=0,
                    brand_advertiser_tag_id=0,
                    site_offer_tag_id=0,
                    campaign_id=0,
                    creative_id=0,
                    price_format_id=0,
                    disposition_type='all',
                    disposition_id=0,
                    source_affiliate_billing_status='all',
                    brand_advertiser_billing_status='all',
                    test_filter='non_tests',
                    start_at_row=0,
                    row_limit=100000,
                    sort_field='event_conversion_date',
                    sort_descending='false')

                endpoint_string = 'http://' + ADMIN_DOMAIN_URL + '/api/15/reports.asmx/EventConversions'
                soup = requests.post(endpoint_string,json=payload)
                print('processing API response')
                soup_text = soup.text
                response = json.loads(soup_text)

                for c in response["d"]["event_conversions"]:
                    c["event_conversion_date"] = date_convert_for_csv(c["event_conversion_date"])
                    c["last_updated"] = date_convert_for_csv(c["last_updated"])
                    c["paid"]["currency_id"] = return_currency_name(c["paid"]["currency_id"], country_codes)
                    c["received"]["currency_id"] = return_currency_name(c["received"]["currency_id"], country_codes)

                    if c["click_date"]:
                        c["click_date"] = date_convert_for_csv(c["click_date"])

                    if c["event_conversion_date"] and c["click_date"]:
                        time_delta = conversion_time_delta(c["event_conversion_date"], c["click_date"])
                    else:
                        time_delta = ""


                    # encodings
                    conversion_id = c["event_conversion_id"]
                    last_updated = c["last_updated"]
                    conversion_date = c["event_conversion_date"]
                    click_date = c["click_date"]
                    affiliate_id = c["source_affiliate"]["source_affiliate_id"]<commit_msg>changed conversion api to include macro and micro events<commit_after># -*- coding: utf-8 -*-

import boto3
import json
import time
from collections import OrderedDict
import re
import csv
from datetime import datetime, date, timedelta
import requests
from flask import Flask, render_template, url_for, request, redirect, g, jsonify, flash, session, Markup
from run import app
from pymongo import *
from bs4 import BeautifulSoup
from settings import *

client = MongoClient(MONGODB_DATABASE['uri'])
db = client[MONGODB_DATABASE['database_name']]

in_progress = False

def get_country_codes():
    payload = dict(
        api_key=API_KEY,
        )

    endpoint_string = 'http://' + ADMIN_DOMAIN_URL + '/api/1/get.asmx/Currencies'
    soup = requests.post(endpoint_string,json=payload)
    r = soup.json()

    country_codes = {}

    for x in r["d"]["currencies"]:
        abbreviation = x["currency_abbr"]
        currency_id = str(x["currency_id"])
        country_codes[currency_id] = abbreviation

    return country_codes

country_codes = get_country_codes()

def return_currency_name(country_id, country_codes):
    country_id = str(country_id)
    country_name = ''.join({value for key, value in country_codes.items() if country_id == key})
    return country_name


def receive_message():

    in_progress = True
    client = boto3.client('sqs')
    queue_size_response = client.get_queue_attributes(QueueUrl= SQS_QUEUE['url'],
                                                AttributeNames=['ApproximateNumberOfMessages'])
    queue_size = queue_size_response["Attributes"]["ApproximateNumberOfMessages"]
    if queue_size != "0":
        response = OrderedDict(client.receive_message(QueueUrl = SQS_QUEUE['url'],
                                            AttributeNames=['Body'],
                                            MaxNumberOfMessages=1))
        return response
    elif queue_size == "0":
        response = "No Messages in Queue"
        return response

def delete_message(receipt_handle):
    client = boto3.client('sqs')
    response = client.delete_message(QueueUrl= SQS_QUEUE['url'],
                                     ReceiptHandle=receipt_handle)
    return response

def enumerate_dates(start, end):
    start_date = datetime(start.year, start.month, start.day)
    end_date = datetime(end.year, end.month, end.day)
    delta = end_date - start_date

    for i in range(delta.days + 1):
        next_start_date = start_date + timedelta(days=1)
        for i in range(48):
            next_start_hour = start_date + timedelta(minutes=30)
            yield start_date, next_start_hour
            start_date = next_start_hour
        start_date=next_start_date

def date_convert_for_csv(date):
    extract_integers = re.findall('\d+', date)
    date_string = ''.join(extract_integers)
    if len(date_string) > 10:
        date_string = date_string[:10] + '.' + date_string[10:]
        date_result = (datetime.utcfromtimestamp(float(date_string)) + timedelta(hours=1)).strftime("%d-%m-%YT%H:%M:%S.%f")
        return date_result
    else:
        timestamp_parsed = (datetime.utcfromtimestamp(int(date_string))+ timedelta(hours=1)) + '.000000'
        date_result = timestamp_parsed.strftime("%d-%m-%YT%H:%M:%S.%f")
        return date_result

def conversion_time_delta(conversion_date, click_date):
    conversion_date = datetime.strptime(conversion_date, "%d-%m-%YT%H:%M:%S.%f")
    click_date = datetime.strptime(click_date, "%d-%m-%YT%H:%M:%S.%f")

    if click_date > conversion_date:
        time_delta = "1"
        return time_delta

    elif conversion_date > click_date:
        time_delta = conversion_date - click_date
        if time_delta.seconds < 1:
            time_delta = "1"
            return time_delta
        else:
            return time_delta

def s3_job(filename):
# expire 86400 seconds is 24 hours
    s3 = boto3.resource('s3')
    s3.meta.client.upload_file('temp.csv', S3_BUCKET['name'], '%s.csv' % filename)

    client = boto3.client('s3')
    url = client.generate_presigned_url('get_object',
                                        Params={'Bucket': S3_BUCKET['name'],'Key': '%s.csv' % filename},
                                        ExpiresIn=86400)
    return url


def execute_call(response):

    body = (response["Messages"][0]["Body"]).replace("'", "\"")
    load_body = json.loads(body)

    start_date = load_body['start_date']
    end_date = load_body['end_date']
    job_id = load_body['job_id']
    created_date = load_body['created_date']
    receipt_handle = response["Messages"][0]["ReceiptHandle"]
    delete_message(receipt_handle)

    start_datetime = datetime.strptime(start_date, "%m/%d/%y")
    end_datetime = datetime.strptime(end_date, "%m/%d/%y")

    collection_name = db[MONGODB_DATABASE['collection_name']]
    collection_name.update_one({"created_date": created_date}, {"$set": {"status": "In Progress"}})


    try:
        with open('temp.csv', 'wb') as text_file:
            writer = csv.writer(text_file)
            writer.writerow(('Conversion ID',
                            'Last Updated',
                            'Conversion Date',
                            'Click Date',
                            'Click to Conversion Time',
                            'Affiliate ID',
                            'Affiliate Name',
                            'Advertiser ID',
                            'Advertiser Name',
                            'Offer ID',
                            'Offer Name',
                            'Creative',
                            'Sub ID',
                            'Sub ID 2',
                            'Sub ID 3',
                            'Sub ID 4',
                            'Sub ID 5',
                            'Type',
                            'Price Paid Currency',
                            'Paid',
                            'Price Received Currency',
                            'Received',
                            'Pixel',
                            'Transaction ID',
                            'IP Address',
                            'Click IP Address',
                            'Country',
                            'Conversion Referrer',
                            'Referrer',
                            'Conversion User Agent',
                            'Click User Agent',
                            'Disposition',
                            'Region',
                            'Language',
                            'Provider Name',
                            'Device',
                            'Operating System',
                            'OS(Major Version)',
                            'OS(Minor Version)',
                            'Browser',
                            'Browser(Major Version)',
                            'Browser(Minor Versions)',
                            'Conversion Score',
                            'Paid Unbilled',
                            'Received Unbilled',
                            'Click Request Session ID',
                            'Event Name',
                            'Price Format',
                            'Tracking ID',
                            'UDID'))


            date_generator = enumerate_dates(start_datetime, end_datetime)

            for start, end in date_generator:
                start = datetime.strftime(start, '%m/%d/%Y %H:%M')
                end = datetime.strftime(end, '%m/%d/%Y %H:%M')
                print(start, end)
                payload = dict(
                    api_key=API_KEY,
                    start_date=start,
                    end_date=end,
                    conversion_type='all',
                    event_type='all',
                    event_id=0,
                    channel_id=0,
                    source_affiliate_id=0,
                    brand_advertiser_id=0,
                    site_offer_id=0,
                    site_offer_contract_id=0,
                    source_affiliate_tag_id=0,
                    brand_advertiser_tag_id=0,
                    site_offer_tag_id=0,
                    campaign_id=0,
                    creative_id=0,
                    price_format_id=0,
                    disposition_type='all',
                    disposition_id=0,
                    source_affiliate_billing_status='all',
                    brand_advertiser_billing_status='all',
                    test_filter='non_tests',
                    start_at_row=0,
                    row_limit=100000,
                    sort_field='event_conversion_date',
                    sort_descending='false')

                endpoint_string = 'http://' + ADMIN_DOMAIN_URL + '/api/15/reports.asmx/EventConversions'
                soup = requests.post(endpoint_string,json=payload, stream=True)
                print('processing API response')
                soup_text = soup.text
                response = json.loads(soup_text)

                for c in response["d"]["event_conversions"]:
                    c["event_conversion_date"] = date_convert_for_csv(c["event_conversion_date"])
                    c["last_updated"] = date_convert_for_csv(c["last_updated"])
                    c["paid"]["currency_id"] = return_currency_name(c["paid"]["currency_id"], country_codes)
                    c["received"]["currency_id"] = return_currency_name(c["received"]["currency_id"], country_codes)

                    if c["click_date"]:
                        c["click_date"] = date_convert_for_csv(c["click_date"])

                    if c["event_conversion_date"] and c["click_date"]:
                        time_delta = conversion_time_delta(c["event_conversion_date"], c["click_date"])
                    else:
                        time_delta = ""


                    # encodings
                    conversion_id = c["event_conversion_id"]
                    last_updated = c["last_updated"]
                    conversion_date = c["event_conversion_date"]
                    click_date = c["click_date"]
                    affiliate_id = c["source_affiliate"]["source_affiliate_id"]
{'fraction_non_alphanumeric': 0.10361874629226814, 'fraction_numerical': 0.008008700810757366, 'mean_word_length': 1.6021353228711088, 'pattern_counts': {'":': 10, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '295679', 'n_tokens_mistral': 5657, 'n_tokens_neox': 5379, 'n_words': 1008}