<commit_before>import os, sys
import pika
import json
import time
import traceback

from peewee import *

# DEBUG = int(os.environ.get('DEBUG'))
MQTT_HOST = os.environ.get('MQTT_HOST')
MQTT_USER = os.environ.get('MQTT_USER')
MQTT_PASSWORD = os.environ.get('MQTT_PASSWORD')

DB_HOST = os.environ.get('DB_HOST')
DB_PASSWORD = <PASSWORD>('DB_PASSWORD')
DB_USER = os.environ.get('DB_USER')
DB_NAME = os.environ.get('DB_NAME')


"""
PSQL ORM courtesy of PeeWee

No need for schema.sql since PeeWee can take care of this for us!
"""

psql_db = PostgresqlDatabase(DB_NAME, user=DB_USER, password=<PASSWORD>, host=DB_HOST)

class BaseModel(Model):
    class Meta:
        database = psql_db


class FacebookContact(BaseModel):
    facebook_resource_locator = TextField(primary_key=True)
    org_name = CharField(null = True)
    description = TextField(null = True)
    address = TextField(null = True)
    country = CharField(null = True)
    state = CharField(null = True)
    postal_code = IntegerField(null = True)
    contact_no = CharField(null = True)
    industry = CharField(null = True)
    fan_count = IntegerField(null = True)
    hours = TextField(null = True)
    link = TextField(null = True)
    longitude = TextField(null = True)
    latitude = TextField(null = True)
    intl_number_with_plus = TextField(null = True)

class LinkedInContact(BaseModel):
    linkedin_resource_locator = TextField(primary_key = True)
    org_name = CharField(null = True)
    org_type = CharField(null = True)
    description = TextField(null = True)
    address = TextField(null = True)
    city = CharField(null = True)
    state = CharField(null = True)
    postal_code = IntegerField(null = True)
    website = TextField(null = True)
    industry = CharField(null = True)
    specialities = TextField(null = True)
    follower_count = IntegerField(null = True)
    year_founded = TextField(null = True)
    size = TextField(null = True)

class FourSquareContact(BaseModel):
    foursquare_resource_locator = TextField(primary_key=True)
    org_name = CharField(null = True)
    description = TextField(null = True)
    address = TextField(null = True)
    country = CharField(null = True)
    #state = CharField(null = True)
    postal_code = CharField(null = True)
    contact_no = CharField(null = True)
    #industry = CharField(null = True)
    fan_count = IntegerField(null = True)
    hours = TextField(null = True)
    link = TextField(null = True)
    longitude = TextField(null = True)
    latitude = TextField(null = True)  

class GoogleContact(BaseModel):
    google_resource_locator = TextField(primary_key=True)
    org_name = CharField(null = True)
    address = CharField(null = True)
    country = CharField(null = True)
    postal_code = CharField(null = True)
    contact_no = CharField(null = True)
    industry = CharField(null = True)
    rating = FloatField(null = True)
    link = TextField(null = True)
    longitude = TextField(null = True)
    latitude = TextField(null = True)
    intl_number_with_plus = TextField(null = True)


while True:
    try:
        psql_db.connect()
        break
    except Exception:
        time.sleep(5)

if not FacebookContact.table_exists():
    FacebookContact.create_table()

if not LinkedInContact.table_exists():
    LinkedInContact.create_table()

if not FourSquareContact.table_exists():
    FourSquareContact.create_table()    

if not GoogleContact.table_exists():
    GoogleContact.create_table()
"""
RabbitMQ support courtesy of Pika

MQTT tutorial from 
https://cuongba.com/install-rabbitmq-and-send-json-data-with-python-on-ubuntu/
"""

while True:
    try:
        _credentials = pika.PlainCredentials(MQTT_USER, MQTT_PASSWORD)
        mqtt_connection = pika.BlockingConnection(pika.ConnectionParameters(host=MQTT_HOST, credentials=_credentials))
        break
    except Exception:
        time.sleep(5)

pqdata = dict()
pqdata['x-max-priority'] = 5

ingress_channel = mqtt_connection.channel()
ingress_channel.queue_declare(queue='store', durable=True, arguments=pqdata)


"""
Updating function

- Currently it just overwrites, also makes multiple requests per update, not cool
- follower count is different for different sites
"""

def updateFacebookContact(data):
    if data.has_key("org_name") and data["org_name"] != None:
        FacebookContact.update(org_name = data["org_name"]).where(FacebookContact.facebook_resource_locator == data["facebook_resource_locator"]).execute()
    if data.has_key("description") and data["description"] != None:
        FacebookContact.update(description = data["description"]).where(FacebookContact.facebook_resource_locator == data["facebook_resource_locator"]).execute()
    if data.has_key("address") and data["address"] != None:
        FacebookContact.update(address = data["address"]).where(FacebookContact.facebook_resource_locator == data["facebook_resource_locator"]).execute()
    if data.has_key("country") and data["country"] != None:
        FacebookContact.update(country = data["country"]).where(FacebookContact.facebook_resource_locator == data["facebook_resource_locator"]).execute()
    if data.has_key("state") and data["state"] != None:
        FacebookContact.update(state = data["state"]).where(FacebookContact.facebook_resource_locator == data["facebook_resource_locator"]).execute()
    if data.has_key("postal_code") and isinstance(data["postal_code"], int) and data["postal_code"] != None:
        FacebookContact.update(postal_code = data["postal_code"]).where(FacebookContact.facebook_resource_locator == data["facebook_resource_locator"]).execute()
    if data.has_key("contact_no") and data["contact_no"] != None:
        FacebookContact.update(contact_no = data["contact_no"]).where(FacebookContact.facebook_resource_locator == data["facebook_resource_locator"]).execute()
    if data.has_key("industry") and data["industry"] != None:
        FacebookContact.update(industry = data["industry"]).where(FacebookContact.facebook_resource_locator == data["facebook_resource_locator"]).execute()
    if data.has_key("fan_count") and isinstance(data["fan_count"], int) and data["fan_count"] != None:
        FacebookContact.update(fan_count = data["fan_count"]).where(FacebookContact.facebook_resource_locator == data["facebook_resource_locator"]).execute()
    if data.has_key("hours") and data["hours"] != None:
        FacebookContact.update(hours = data["hours"]).where(FacebookContact.facebook_resource_locator == data["facebook_resource_locator"]).execute()
    if data.has_key("link") and data["link"] != None:
        FacebookContact.update(link = data["link"]).where(FacebookContact.facebook_resource_locator == data["facebook_resource_locator"]).execute()
    if data.has_key("longitude") and data["longitude"] != None:
        FacebookContact.update(longitude = data["longitude"]).where(FacebookContact.facebook_resource_locator == data["facebook_resource_locator"]).execute()
    if data.has_key("latitude") and data["latitude"] != None:
        FacebookContact.update(latitude = data["latitude"]).where(FacebookContact.facebook_resource_locator == data["facebook_resource_locator"]).execute()
    if data.has_key("intl_number_with_plus") and data["intl_number_with_plus"] != None:
        FacebookContact.update(intl_number_with_plus = data["intl_number_with_plus"]).where(FacebookContact.facebook_resource_locator == data["facebook_resource_locator"]).execute()
    return

def updateLinkedInContact(data):
    if data.has_key("org_name") and data["org_name"] != None:
        LinkedInContact.update(org_name = data["org_name"]).where(LinkedInContact.linkedin_resource_locator == data["linkedin_resource_locator"]).execute()
    if data.has_key("org_type") and data["org_type"] != None:
        LinkedInContact.update(org_type = data["org_type"]).where(LinkedInContact.linkedin_resource_locator == data["linkedin_resource_locator"]).execute()
    if data.has_key("description") and data["description"] != None:
        LinkedInContact.update(description = data["description"]).where(LinkedInContact.linkedin_resource_locator == data["linkedin_resource_locator"]).execute()<commit_msg>updated facebook schema (postal code -> string)<commit_after>import os, sys
import pika
import json
import time
import traceback

from peewee import *

# DEBUG = int(os.environ.get('DEBUG'))
MQTT_HOST = os.environ.get('MQTT_HOST')
MQTT_USER = os.environ.get('MQTT_USER')
MQTT_PASSWORD = os.environ.get('MQTT_PASSWORD')

DB_HOST = os.environ.get('DB_HOST')
DB_PASSWORD = os.environ.get('DB_PASSWORD')
DB_USER = os.environ.get('DB_USER')
DB_NAME = os.environ.get('DB_NAME')


"""
PSQL ORM courtesy of PeeWee

No need for schema.sql since PeeWee can take care of this for us!
"""

psql_db = PostgresqlDatabase(DB_NAME, user=DB_USER, password=DB_PASSWORD, host=DB_HOST)

class BaseModel(Model):
    class Meta:
        database = psql_db


class FacebookContact(BaseModel):
    facebook_resource_locator = TextField(primary_key=True)
    org_name = CharField(null = True)
    description = TextField(null = True)
    address = TextField(null = True)
    country = CharField(null = True)
    state = CharField(null = True)
    postal_code = CharField(null = True)
    contact_no = CharField(null = True)
    industry = CharField(null = True)
    fan_count = IntegerField(null = True)
    hours = TextField(null = True)
    link = TextField(null = True)
    longitude = TextField(null = True)
    latitude = TextField(null = True)
    intl_number_with_plus = TextField(null = True)

class LinkedInContact(BaseModel):
    linkedin_resource_locator = TextField(primary_key = True)
    org_name = CharField(null = True)
    org_type = CharField(null = True)
    description = TextField(null = True)
    address = TextField(null = True)
    city = CharField(null = True)
    state = CharField(null = True)
    postal_code = IntegerField(null = True)
    website = TextField(null = True)
    industry = CharField(null = True)
    specialities = TextField(null = True)
    follower_count = IntegerField(null = True)
    year_founded = TextField(null = True)
    size = TextField(null = True)

class FourSquareContact(BaseModel):
    foursquare_resource_locator = TextField(primary_key=True)
    org_name = CharField(null = True)
    description = TextField(null = True)
    address = TextField(null = True)
    country = CharField(null = True)
    #state = CharField(null = True)
    postal_code = CharField(null = True)
    contact_no = CharField(null = True)
    #industry = CharField(null = True)
    fan_count = IntegerField(null = True)
    hours = TextField(null = True)
    link = TextField(null = True)
    longitude = TextField(null = True)
    latitude = TextField(null = True)  

class GoogleContact(BaseModel):
    google_resource_locator = TextField(primary_key=True)
    org_name = CharField(null = True)
    address = CharField(null = True)
    country = CharField(null = True)
    postal_code = CharField(null = True)
    contact_no = CharField(null = True)
    industry = CharField(null = True)
    rating = FloatField(null = True)
    link = TextField(null = True)
    longitude = TextField(null = True)
    latitude = TextField(null = True)
    intl_number_with_plus = TextField(null = True)


while True:
    try:
        psql_db.connect()
        break
    except Exception:
        time.sleep(5)

if not FacebookContact.table_exists():
    FacebookContact.create_table()

if not LinkedInContact.table_exists():
    LinkedInContact.create_table()

if not FourSquareContact.table_exists():
    FourSquareContact.create_table()    

if not GoogleContact.table_exists():
    GoogleContact.create_table()
"""
RabbitMQ support courtesy of Pika

MQTT tutorial from 
https://cuongba.com/install-rabbitmq-and-send-json-data-with-python-on-ubuntu/
"""

while True:
    try:
        _credentials = pika.PlainCredentials(MQTT_USER, MQTT_PASSWORD)
        mqtt_connection = pika.BlockingConnection(pika.ConnectionParameters(host=MQTT_HOST, credentials=_credentials))
        break
    except Exception:
        time.sleep(5)

pqdata = dict()
pqdata['x-max-priority'] = 5

ingress_channel = mqtt_connection.channel()
ingress_channel.queue_declare(queue='store', durable=True, arguments=pqdata)


"""
Updating function

- Currently it just overwrites, also makes multiple requests per update, not cool
- follower count is different for different sites
"""

def updateFacebookContact(data):
    if data.has_key("org_name") and data["org_name"] != None:
        FacebookContact.update(org_name = data["org_name"]).where(FacebookContact.facebook_resource_locator == data["facebook_resource_locator"]).execute()
    if data.has_key("description") and data["description"] != None:
        FacebookContact.update(description = data["description"]).where(FacebookContact.facebook_resource_locator == data["facebook_resource_locator"]).execute()
    if data.has_key("address") and data["address"] != None:
        FacebookContact.update(address = data["address"]).where(FacebookContact.facebook_resource_locator == data["facebook_resource_locator"]).execute()
    if data.has_key("country") and data["country"] != None:
        FacebookContact.update(country = data["country"]).where(FacebookContact.facebook_resource_locator == data["facebook_resource_locator"]).execute()
    if data.has_key("state") and data["state"] != None:
        FacebookContact.update(state = data["state"]).where(FacebookContact.facebook_resource_locator == data["facebook_resource_locator"]).execute()
    if data.has_key("postal_code") and data["postal_code"] != None:
        FacebookContact.update(postal_code = data["postal_code"]).where(FacebookContact.facebook_resource_locator == data["facebook_resource_locator"]).execute()
    if data.has_key("contact_no") and data["contact_no"] != None:
        FacebookContact.update(contact_no = data["contact_no"]).where(FacebookContact.facebook_resource_locator == data["facebook_resource_locator"]).execute()
    if data.has_key("industry") and data["industry"] != None:
        FacebookContact.update(industry = data["industry"]).where(FacebookContact.facebook_resource_locator == data["facebook_resource_locator"]).execute()
    if data.has_key("fan_count") and isinstance(data["fan_count"], int) and data["fan_count"] != None:
        FacebookContact.update(fan_count = data["fan_count"]).where(FacebookContact.facebook_resource_locator == data["facebook_resource_locator"]).execute()
    if data.has_key("hours") and data["hours"] != None:
        FacebookContact.update(hours = data["hours"]).where(FacebookContact.facebook_resource_locator == data["facebook_resource_locator"]).execute()
    if data.has_key("link") and data["link"] != None:
        FacebookContact.update(link = data["link"]).where(FacebookContact.facebook_resource_locator == data["facebook_resource_locator"]).execute()
    if data.has_key("longitude") and data["longitude"] != None:
        FacebookContact.update(longitude = data["longitude"]).where(FacebookContact.facebook_resource_locator == data["facebook_resource_locator"]).execute()
    if data.has_key("latitude") and data["latitude"] != None:
        FacebookContact.update(latitude = data["latitude"]).where(FacebookContact.facebook_resource_locator == data["facebook_resource_locator"]).execute()
    if data.has_key("intl_number_with_plus") and data["intl_number_with_plus"] != None:
        FacebookContact.update(intl_number_with_plus = data["intl_number_with_plus"]).where(FacebookContact.facebook_resource_locator == data["facebook_resource_locator"]).execute()
    return

def updateLinkedInContact(data):
    if data.has_key("org_name") and data["org_name"] != None:
        LinkedInContact.update(org_name = data["org_name"]).where(LinkedInContact.linkedin_resource_locator == data["linkedin_resource_locator"]).execute()
    if data.has_key("org_type") and data["org_type"] != None:
        LinkedInContact.update(org_type = data["org_type"]).where(LinkedInContact.linkedin_resource_locator == data["linkedin_resource_locator"]).execute()
    if data.has_key("description") and data["description"] != None:
        LinkedInContact.update(description = data["description"]).where(LinkedInContact.linkedin_resource_locator == data["linkedin_resource_locator"]).execute()
{'fraction_non_alphanumeric': 0.12462342453120197, 'fraction_numerical': 0.0003688902551490931, 'mean_word_length': 5.0944173847883105, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 64, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7269965', 'n_tokens_mistral': 5015, 'n_tokens_neox': 4937, 'n_words': 905}