<commit_before>from sqlalchemy import text
from lyrics import get_lyrics, get_sentiment
from audio import get_audio_analysis
import string
import json


def update_songs_table(engine):
    # Query songs that don't have sentiment or lyrics or audio analysis
    sql = text('SELECT id, artist, title, preview_url '
               'FROM songs '
               'WHERE (lyrics IS NULL) AND (pos IS NULL) AND (tempo IS NULL);')

    con = engine.connect()
    result = con.execute(sql).fetchall()

    for res in result:
        song_id = res['id']
        artist = res['artist']
        song = res['title']
        preview_url = res['preview_url']

        print "Doing analysis for '{}/{}'".format(
            artist.encode('utf-8'), song.encode('utf-8'))

        printable = set(string.printable)
        lyrics = get_lyrics(artist, song)
        if(lyrics is None):
            print("Could not find lyrics for " + song)
        else:
            lyrics = ''.join(filter(lambda x: x in printable, lyrics))
        sentiment = get_sentiment(lyrics)
        tempo, pitch, harm, perc, wave = get_audio_analysis(preview_url)

        sql = text('UPDATE songs '
                   'SET lyrics=:lyrics, pos=:pos, neg=:neg, neu=:neu, '
                   '    compound=:compound, tempo=:tempo, '
                   '    pitch=:pitch, harmonic=:harmonic, '
                   '    percussive=:percussive, wave_info=:wave '
                   'WHERE id=:song_id;', autocommit=True)
        con.execute(sql,
                    lyrics=str(lyrics) if lyrics else None,
                    pos=sentiment['pos'],
                    neg=sentiment['neg'],
                    neu=sentiment['neu'],
                    compound=sentiment['compound'],
                    tempo=tempo,
                    pitch=pitch,
                    harmonic=harm,
                    percussive=perc,
                    song_id=song_id,
                    wave=json.dumps(wave))
<commit_msg>Revert wave saving<commit_after>from sqlalchemy import text
from lyrics import get_lyrics, get_sentiment
from audio import get_audio_analysis
import string
import json


def update_songs_table(engine):
    # Query songs that don't have sentiment or lyrics or audio analysis
    sql = text('SELECT id, artist, title, preview_url '
               'FROM songs '
               'WHERE (lyrics IS NULL) AND (pos IS NULL) AND (tempo IS NULL);')

    con = engine.connect()
    result = con.execute(sql).fetchall()

    for res in result:
        song_id = res['id']
        artist = res['artist']
        song = res['title']
        preview_url = res['preview_url']

        print "Doing analysis for '{}/{}'".format(
            artist.encode('utf-8'), song.encode('utf-8'))

        printable = set(string.printable)
        lyrics = get_lyrics(artist, song)
        if(lyrics is None):
            print("Could not find lyrics for " + song)
        else:
            lyrics = ''.join(filter(lambda x: x in printable, lyrics))
        sentiment = get_sentiment(lyrics)
        tempo, pitch, harm, perc, wave = get_audio_analysis(preview_url)

        sql = text('UPDATE songs '
                   'SET lyrics=:lyrics, pos=:pos, neg=:neg, neu=:neu, '
                   '    compound=:compound, tempo=:tempo, '
                   '    pitch=:pitch, harmonic=:harmonic, '
                   '    percussive=:percussive, wave_info=:wave '
                   'WHERE id=:song_id;', autocommit=True)
        con.execute(sql,
                    lyrics=str(lyrics) if lyrics else None,
                    pos=sentiment['pos'],
                    neg=sentiment['neg'],
                    neu=sentiment['neu'],
                    compound=sentiment['compound'],
                    tempo=tempo,
                    pitch=pitch,
                    harmonic=harm,
                    percussive=perc,
                    song_id=song_id,
                    wave=None)

{'fraction_non_alphanumeric': 0.10980891719745223, 'fraction_numerical': 0.001019108280254777, 'mean_word_length': 1.8163558106169297, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4782333', 'n_tokens_mistral': 1144, 'n_tokens_neox': 1103, 'n_words': 274}