<commit_before>#!/usr/bin/env python

"""
Commands that update or process the application data.
"""
import csv
from datetime import datetime
import json

from fabric.api import task, local
from facebook import GraphAPI
from twitter import Twitter, OAuth

import app_config
import copytext


@task(default=True)
def update():
    """
    Stub function for updating app-specific data.
    """
    #update_featured_social()
    update_songs()

@task
def update_songs():
    local('curl -o data/songs.csv https://docs.google.com/spreadsheets/d/19J4Fi2bpeEicoM5475lu1L345fF8hiBn4owbiCKdqIU/export?format=csv&id=19J4Fi2bpeEicoM5475lu1L345fF8hiBn4owbiCKdqIU&gid=0')

    output = []

    with open('data/songs.csv') as f:
        rows = csv.DictReader(f)

        for row in rows:
            stripped_row = {}

            for name, value in row.items():
                stripped_row[name] = value.strip()

            row = stripped_row

            row['genre_tags'] = []
            for i in range(1,4):
                key = 'genre%i' % i

                if row[key]:
                    row['genre_tags'].append(row[key])

                del row[key]

            row['curator_tags'] = []
            for i in range(1,4):
                key = 'curator%i' % i

                if row[key]:
                    row['curator_tags'].append(row[key])

                del row[key]

            output.append(row)


    with open('data/songs.json', 'w') as f:
        json.dump(output, f)

@task
def update_featured_social():
    """
    Update featured tweets
    """
    COPY = copytext.Copy(app_config.COPY_PATH)
    secrets = app_config.get_secrets()

    # Twitter<commit_msg>scraper to verify song art and aduio links in data spreadsheet. closes #200<commit_after>#!/usr/bin/env python

"""
Commands that update or process the application data.
"""
import csv
from datetime import datetime
import json

from fabric.api import task, local
from facebook import GraphAPI
from twitter import Twitter, OAuth
import requests

import app_config
import copytext


@task(default=True)
def update():
    """
    Stub function for updating app-specific data.
    """
    #update_featured_social()
    update_songs()

@task
def update_songs():
    local('curl -s -o data/songs.csv https://docs.google.com/spreadsheets/d/19J4Fi2bpeEicoM5475lu1L345fF8hiBn4owbiCKdqIU/export?format=csv&id=19J4Fi2bpeEicoM5475lu1L345fF8hiBn4owbiCKdqIU&gid=0')

    output = []

    with open('data/songs.csv') as f:
        rows = csv.DictReader(f)

        for row in rows:
            stripped_row = {}

            for name, value in row.items():
                stripped_row[name] = value.strip()

            row = stripped_row
            check_song_links(row)

            row['genre_tags'] = []
            for i in range(1,4):
                key = 'genre%i' % i

                if row[key]:
                    row['genre_tags'].append(row[key])

                del row[key]

            row['curator_tags'] = []
            for i in range(1,4):
                key = 'curator%i' % i

                if row[key]:
                    row['curator_tags'].append(row[key])

                del row[key]

            output.append(row)


    with open('data/songs.json', 'w') as f:
        json.dump(output, f)

@task
def check_song_links(row):
    """
    Verify all links in the song data are working
    """
    audio_link = 'http://pd.npr.org/anon.npr-mp3%s.mp3' % row['media_url']
    song_art_link = 'http://npr.org%s' % row['song_art']

    audio_request = requests.get(audio_link)
    if audio_request.status_code != 200:
        print '%s - The audio URL for %s - %s is invalid: %s' % (audio_request.status_code, row['artist'], row['title'], audio_link)

    song_art_request = requests.get(song_art_link)
    if song_art_request.status_code != 200:
        print '%s - The song art URL for %s - %s is invalid: %s' % (song_art_request, row['artist'], row['title'], song_art_link)

@task
def update_featured_social():
    """
    Update featured tweets
    """
    COPY = copytext.Copy(app_config.COPY_PATH)
    secrets = app_config.get_secrets()

    # Twitter
{'fraction_non_alphanumeric': 0.11839884253677357, 'fraction_numerical': 0.01856763925729443, 'mean_word_length': 2.347861178369653, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '603862', 'n_tokens_mistral': 1448, 'n_tokens_neox': 1339, 'n_words': 314}