<commit_before>#!/usr/bin/env python
import os
import sys
import optparse
import inspect

#import pybossa.model as model
from pybossa.core import db, create_app
from pybossa.model.app import App
from pybossa.model.user import User
from pybossa.model.category import Category

from alembic.config import Config
from alembic import command
from html2text import html2text
from sqlalchemy.sql import text

app = create_app(run_as_server=False)

def setup_alembic_config():
    if "DATABASE_URL" not in os.environ:
        alembic_cfg = Config("alembic.ini")
    else:
        dynamic_filename = "alembic-heroku.ini"
        with file("alembic.ini.template") as f:
            with file(dynamic_filename, "w") as conf:
                for line in f.readlines():
                    if line.startswith("sqlalchemy.url"):
                        conf.write("sqlalchemy.url = %s\n" %
                                   os.environ['DATABASE_URL'])
                    else:
                        conf.write(line)
        alembic_cfg = Config(dynamic_filename)

    command.stamp(alembic_cfg, "head")

def db_create():
    '''Create the db'''
    with app.app_context():
        db.create_all()
        # then, load the Alembic configuration and generate the
        # version table, "stamping" it with the most recent rev:
        setup_alembic_config()
        # finally, add a minimum set of categories: Volunteer Thinking, Volunteer Sensing, Published and Draft
        categories = []
        categories.append(Category(name="Thinking",
                          short_name='thinking',
                          description='Volunteer Thinking projects'))
        categories.append(Category(name="Volunteer Sensing",
                          short_name='sensing',
                          description='Volunteer Sensing projects'))
        db.session.add_all(categories)
        db.session.commit()

def db_rebuild():
    '''Rebuild the db'''
    with app.app_context():
        db.drop_all()
        db.create_all()
        # then, load the Alembic configuration and generate the
        # version table, "stamping" it with the most recent rev:
        setup_alembic_config()

def fixtures():
    '''Create some fixtures!'''
    with app.app_context():
        user = User(
            name=u'tester',
            email_addr=u'<EMAIL>',
            api_key='tester'
            )
        user.set_password(u'<PASSWORD>')
        db.session.add(user)
        db.session.commit()

def markdown_db_migrate():
    '''Perform a migration of the app long descriptions from HTML to
    Markdown for existing database records'''
    with app.app_context():
        query = 'SELECT id, long_description FROM "app";'
        query_result = db.engine.execute(query)
        old_descriptions = query_result.fetchall()
        for old_desc in old_descriptions:
            if old_desc.long_description:
                new_description = html2text(old_desc.long_description)
                query = text('''
                           UPDATE app SET long_description=:long_description
                           WHERE id=:id''')
                db.engine.execute(query, long_description = new_description, id = old_desc.id)


def delete_hard_bounces():
    '''Delete fake accounts from hard bounces.'''
    del_users = 0
    fake_emails = 0
    with app.app_context():
        with open('email.csv', 'r') as f:
            emails = f.readlines()
            print "Number of users: %s" % len(emails)
            for email in emails:
                usr = db.session.query(User).filter_by(email_addr=email.rstrip()).first()
                if usr and len(usr.apps) == 0 and len(usr.task_runs) == 0:
                    print "Deleting user: %s" % usr.email_addr
                    del_users +=1
                    db.session.delete(usr)
                    db.session.commit()
                else:
                    if usr:
                        if len(usr.apps) > 0:
                            print "Invalid email (user owns app): %s" % usr.email_addr
                        if len(usr.task_runs) > 0:
                            print "Invalid email (user has contributed): %s" % usr.email_addr
                        fake_emails +=1
                        usr.valid_email = False
                        db.session.commit()
        print "%s users were deleted" % del_users
        print "%s users have fake emails" % fake_emails


def bootstrap_avatars():
    """Download current links from user avatar and projects to real images hosted in the
    PyBossa server."""
    import requests
    import os
    import time
    from urlparse import urlparse
    from PIL import Image

    def get_gravatar_url(email, size):
        # import code for encoding urls and generating md5 hashes
        import urllib, hashlib

        # construct the url
        gravatar_url = "http://www.gravatar.com/avatar/" + hashlib.md5(email.lower()).hexdigest() + "?"
        gravatar_url += urllib.urlencode({'d':404, 's':str(size)})
        return gravatar_url

    with app.app_context():
        if app.config['UPLOAD_METHOD'] == 'local':
            users = User.query.order_by('id').all()
            print "Downloading avatars for %s users" % len(users)
            for u in users[0:10]:
                print "Downloading avatar for %s ..." % u.name
                container = "user_%s" % u.id
                path = os.path.join(app.config.get('UPLOAD_FOLDER'), container)
                try:
                    print get_gravatar_url(u.email_addr, 100)
                    r = requests.get(get_gravatar_url(u.email_addr, 100), stream=True)
                    if r.status_code == 200:
                        if not os.path.isdir(path):
                            os.makedirs(path)
                        prefix = time.time()
                        filename = "%s_avatar.png" % prefix
                        with open(os.path.join(path, filename), 'wb') as f:
                            for chunk in r.iter_content(1024):
                                f.write(chunk)
                        u.info['avatar'] = filename
                        u.info['container'] = container
                        db.session.commit()
                        print "Done!"
                    else:
                        print "No Gravatar, this user will use the placeholder."
                except:
                    raise
                    print "No gravatar, this user will use the placehoder."


            apps = App.query.all()
            print "Downloading avatars for %s projects" % len(apps)
            for a in apps[0:1]:
                if a.info.get('thumbnail') and not a.info.get('container'):
                    print "Working on project: %s ..." % a.short_name
                    print "Saving avatar: %s ..." % a.info.get('thumbnail')<commit_msg>Loop through all users and projects.<commit_after>#!/usr/bin/env python
import os
import sys
import optparse
import inspect

#import pybossa.model as model
from pybossa.core import db, create_app
from pybossa.model.app import App
from pybossa.model.user import User
from pybossa.model.category import Category

from alembic.config import Config
from alembic import command
from html2text import html2text
from sqlalchemy.sql import text

app = create_app(run_as_server=False)

def setup_alembic_config():
    if "DATABASE_URL" not in os.environ:
        alembic_cfg = Config("alembic.ini")
    else:
        dynamic_filename = "alembic-heroku.ini"
        with file("alembic.ini.template") as f:
            with file(dynamic_filename, "w") as conf:
                for line in f.readlines():
                    if line.startswith("sqlalchemy.url"):
                        conf.write("sqlalchemy.url = %s\n" %
                                   os.environ['DATABASE_URL'])
                    else:
                        conf.write(line)
        alembic_cfg = Config(dynamic_filename)

    command.stamp(alembic_cfg, "head")

def db_create():
    '''Create the db'''
    with app.app_context():
        db.create_all()
        # then, load the Alembic configuration and generate the
        # version table, "stamping" it with the most recent rev:
        setup_alembic_config()
        # finally, add a minimum set of categories: Volunteer Thinking, Volunteer Sensing, Published and Draft
        categories = []
        categories.append(Category(name="Thinking",
                          short_name='thinking',
                          description='Volunteer Thinking projects'))
        categories.append(Category(name="Volunteer Sensing",
                          short_name='sensing',
                          description='Volunteer Sensing projects'))
        db.session.add_all(categories)
        db.session.commit()

def db_rebuild():
    '''Rebuild the db'''
    with app.app_context():
        db.drop_all()
        db.create_all()
        # then, load the Alembic configuration and generate the
        # version table, "stamping" it with the most recent rev:
        setup_alembic_config()

def fixtures():
    '''Create some fixtures!'''
    with app.app_context():
        user = User(
            name=u'tester',
            email_addr=u'<EMAIL>',
            api_key='tester'
            )
        user.set_password(u'<PASSWORD>')
        db.session.add(user)
        db.session.commit()

def markdown_db_migrate():
    '''Perform a migration of the app long descriptions from HTML to
    Markdown for existing database records'''
    with app.app_context():
        query = 'SELECT id, long_description FROM "app";'
        query_result = db.engine.execute(query)
        old_descriptions = query_result.fetchall()
        for old_desc in old_descriptions:
            if old_desc.long_description:
                new_description = html2text(old_desc.long_description)
                query = text('''
                           UPDATE app SET long_description=:long_description
                           WHERE id=:id''')
                db.engine.execute(query, long_description = new_description, id = old_desc.id)


def delete_hard_bounces():
    '''Delete fake accounts from hard bounces.'''
    del_users = 0
    fake_emails = 0
    with app.app_context():
        with open('email.csv', 'r') as f:
            emails = f.readlines()
            print "Number of users: %s" % len(emails)
            for email in emails:
                usr = db.session.query(User).filter_by(email_addr=email.rstrip()).first()
                if usr and len(usr.apps) == 0 and len(usr.task_runs) == 0:
                    print "Deleting user: %s" % usr.email_addr
                    del_users +=1
                    db.session.delete(usr)
                    db.session.commit()
                else:
                    if usr:
                        if len(usr.apps) > 0:
                            print "Invalid email (user owns app): %s" % usr.email_addr
                        if len(usr.task_runs) > 0:
                            print "Invalid email (user has contributed): %s" % usr.email_addr
                        fake_emails +=1
                        usr.valid_email = False
                        db.session.commit()
        print "%s users were deleted" % del_users
        print "%s users have fake emails" % fake_emails


def bootstrap_avatars():
    """Download current links from user avatar and projects to real images hosted in the
    PyBossa server."""
    import requests
    import os
    import time
    from urlparse import urlparse
    from PIL import Image

    def get_gravatar_url(email, size):
        # import code for encoding urls and generating md5 hashes
        import urllib, hashlib

        # construct the url
        gravatar_url = "http://www.gravatar.com/avatar/" + hashlib.md5(email.lower()).hexdigest() + "?"
        gravatar_url += urllib.urlencode({'d':404, 's':str(size)})
        return gravatar_url

    with app.app_context():
        if app.config['UPLOAD_METHOD'] == 'local':
            users = User.query.order_by('id').all()
            print "Downloading avatars for %s users" % len(users)
            for u in users:
                print "Downloading avatar for %s ..." % u.name
                container = "user_%s" % u.id
                path = os.path.join(app.config.get('UPLOAD_FOLDER'), container)
                try:
                    print get_gravatar_url(u.email_addr, 100)
                    r = requests.get(get_gravatar_url(u.email_addr, 100), stream=True)
                    if r.status_code == 200:
                        if not os.path.isdir(path):
                            os.makedirs(path)
                        prefix = time.time()
                        filename = "%s_avatar.png" % prefix
                        with open(os.path.join(path, filename), 'wb') as f:
                            for chunk in r.iter_content(1024):
                                f.write(chunk)
                        u.info['avatar'] = filename
                        u.info['container'] = container
                        db.session.commit()
                        print "Done!"
                    else:
                        print "No Gravatar, this user will use the placeholder."
                except:
                    raise
                    print "No gravatar, this user will use the placehoder."


            apps = App.query.all()
            print "Downloading avatars for %s projects" % len(apps)
            for a in apps:
                if a.info.get('thumbnail') and not a.info.get('container'):
                    print "Working on project: %s ..." % a.short_name
                    print "Saving avatar: %s ..." % a.info.get('thumbnail')
{'fraction_non_alphanumeric': 0.09777227722772278, 'fraction_numerical': 0.004586488060570763, 'mean_word_length': 1.7175074183976262, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6750327', 'n_tokens_mistral': 3893, 'n_tokens_neox': 3685, 'n_words': 1006}