<commit_before>import os
from urlparse import urlparse
from plenario.celery_app import celery_app
from plenario.models import MetaTable, MasterTable
from plenario.database import task_session as session, task_engine as engine, \
    Base
from plenario.utils.etl import PlenarioETL
from plenario.utils.weather import WeatherETL
from raven.handlers.logging import SentryHandler
from raven.conf import setup_logging
from plenario.settings import CELERY_SENTRY_URL
from sqlalchemy import Table
from sqlalchemy.exc import NoSuchTableError, InternalError
from datetime import datetime

if CELERY_SENTRY_URL:
    handler = SentryHandler(CELERY_SENTRY_URL)
    setup_logging(handler)

@celery_app.task
def delete_dataset(source_url_hash):
    md = session.query(MetaTable).get(source_url_hash)
    try:
        dat_table = Table('dat_%s' % md.dataset_name, Base.metadata, 
            autoload=True, autoload_with=engine, keep_existing=True)
        dat_table.drop(engine, checkfirst=True)
    except NoSuchTableError:
        pass
    master_table = MasterTable.__table__
    delete = master_table.delete()\
        .where(master_table.c.dataset_name == md.dataset_name)
    conn = engine.contextual_connect()
    try:
        conn.execute(delete)
        session.delete(md)
        session.commit()
    except InternalError, e:
        raise delete_dataset.retry(exc=e)
    conn.close()
    return 'Deleted %s' % md.human_name

@celery_app.task
def add_dataset(source_url_hash, s3_path=None, data_types=None):
    md = session.query(MetaTable).get(source_url_hash)
    etl = PlenarioETL(md.as_dict(), data_types=data_types)
    etl.add(s3_path=s3_path)
    return 'Finished adding %s' % md.human_name

@celery_app.task
def monthly_update():
    md = session.query(MetaTable)\
        .filter(MetaTable.update_freq == 'monthly').all()
    for m in md:
        update_dataset.delay(m.source_url_hash)
        print 'Updating %s' % m.human_name
    return 'Weekly update complere'

@celery_app.task
def weekly_update():
    md = session.query(MetaTable)\
        .filter(MetaTable.update_freq == 'weekly').all()
    for m in md:
        update_dataset.delay(m.source_url_hash)
        print 'Updating %s' % m.human_name
    return 'Weekly update complere'

@celery_app.task
def daily_update():
    md = session.query(MetaTable)\
        .filter(MetaTable.update_freq == 'daily').all()
    for m in md:
        update_dataset.delay(m.source_url_hash)
        print 'Updating %s' % m.human_name
    return 'Daily update complete'

@celery_app.task
def hourly_update():
    md = session.query(MetaTable)\
        .filter(MetaTable.update_freq == 'hourly').all()
    for m in md:
        update_dataset.delay(m.source_url_hash)
        print 'Updating %s' % m.human_name
    return 'yay'

@celery_app.task
def update_dataset(source_url_hash, s3_path=None):
    
    md = session.query(MetaTable).get(source_url_hash)
    etl = PlenarioETL(md.as_dict())
    etl.update(s3_path=s3_path)
    return 'Finished updating %s' % md.human_name

@celery_app.task
def update_weather():
    month, year = datetime.now().month, datetime.now().year
    stations = ['94846', '14855', '04807', '14819', '94866', '04831', '04838']
    w = WeatherETL()
    w.initialize_month(year, month, weather_station_list=stations)
    return 'Added weather for %s %s' % (month, year)<commit_msg>Uh, man.<commit_after>import os
from urlparse import urlparse
from plenario.celery_app import celery_app
from plenario.models import MetaTable, MasterTable
from plenario.database import task_session as session, task_engine as engine, \
    Base
from plenario.utils.etl import PlenarioETL
from plenario.utils.weather import WeatherETL
from raven.handlers.logging import SentryHandler
from raven.conf import setup_logging
from plenario.settings import CELERY_SENTRY_URL
from sqlalchemy import Table
from sqlalchemy.exc import NoSuchTableError, InternalError
from datetime import datetime

if CELERY_SENTRY_URL:
    handler = SentryHandler(CELERY_SENTRY_URL)
    setup_logging(handler)

@celery_app.task
def delete_dataset(source_url_hash):
    md = session.query(MetaTable).get(source_url_hash)
    try:
        dat_table = Table('dat_%s' % md.dataset_name, Base.metadata, 
            autoload=True, autoload_with=engine, keep_existing=True)
        dat_table.drop(engine, checkfirst=True)
    except NoSuchTableError:
        pass
    master_table = MasterTable.__table__
    delete = master_table.delete()\
        .where(master_table.c.dataset_name == md.dataset_name)
    conn = engine.contextual_connect()
    try:
        conn.execute(delete)
        session.delete(md)
        session.commit()
    except InternalError, e:
        raise delete_dataset.retry(exc=e)
    conn.close()
    return 'Deleted %s' % md.human_name

@celery_app.task
def add_dataset(source_url_hash, s3_path=None, data_types=None):
    md = session.query(MetaTable).get(source_url_hash)
    etl = PlenarioETL(md.as_dict(), data_types=data_types)
    etl.add(s3_path=s3_path)
    return 'Finished adding %s' % md.human_name

@celery_app.task
def monthly_update():
    md = session.query(MetaTable)\
        .filter(MetaTable.update_freq == 'monthly').all()
    for m in md:
        update_dataset.delay(m.source_url_hash)
        print 'Updating %s' % m.human_name
    return 'Weekly update complere'

@celery_app.task
def weekly_update():
    md = session.query(MetaTable)\
        .filter(MetaTable.update_freq == 'weekly').all()
    for m in md:
        update_dataset.delay(m.source_url_hash)
        print 'Updating %s' % m.human_name
    return 'Weekly update complere'

@celery_app.task
def daily_update():
    md = session.query(MetaTable)\
        .filter(MetaTable.update_freq == 'daily').all()
    for m in md:
        update_dataset.delay(m.source_url_hash)
        print 'Updating %s' % m.human_name
    return 'Daily update complete'

@celery_app.task
def hourly_update():
    md = session.query(MetaTable)\
        .filter(MetaTable.update_freq == 'hourly').all()
    for m in md:
        update_dataset.delay(m.source_url_hash)
        print 'Updating %s' % m.human_name
    return 'yay'

@celery_app.task
def update_dataset(source_url_hash, s3_path=None):
    
    md = session.query(MetaTable).get(source_url_hash)
    etl = PlenarioETL(md.as_dict())
    etl.update(s3_path=s3_path)
    return 'Finished updating %s' % md.human_name

@celery_app.task
def update_weather():
    month, year = datetime.now().month, datetime.now().year
    stations = ['94846', '14855', '04807', '14819', '94866', '04831', '04838']
    w = WeatherETL()
    w.initialize_month(year, month, weather_stations_list=stations)
    return 'Added weather for %s %s' % (month, year)
{'fraction_non_alphanumeric': 0.10593852581319009, 'fraction_numerical': 0.012235153685467026, 'mean_word_length': 3.8050179211469533, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5477871', 'n_tokens_mistral': 2291, 'n_tokens_neox': 2238, 'n_words': 478}