<commit_before>#!/usr/bin/env python
# encoding: utf-8

import logging
import requests
from modularodm import Q

from framework.celery_tasks import app as celery_app

from website import settings
from website.app import init_app
from framework.transactions.context import TokuTransaction
from website.models import Tag, Conference, Node
from website.files.models import StoredFileNode
from scripts import utils as scripts_utils

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)


def main(dry_run=True):
    init_app(routes=False)
    for conf in Conference.find():
        count = 0
        for tag in Tag.find(Q('lower', 'eq', conf.endpoint.lower())):
            for node in Node.find(Q('is_public', 'eq', True) & Q('is_deleted', 'eq', False) & Q('tags', 'eq', tag)):
                record = next(
                    x for x in
                    StoredFileNode.find(
                        Q('node', 'eq', node) &
                        Q('is_file', 'eq', True)
                    ).limit(1)
                ).wrapped()

                if not dry_run:
                    url = 'https://api.keen.io/3.0/projects/{}/queries/count' \
                          '?api_key={}&event_collection=pageviews&timezone=UTC&timeframe' \
                          '=this_14_days'.format(
                            settings.KEEN['public']['project_id'],
                            settings.KEEN['public']['read_key'])

                    query = {
                        'filters': '%5B%7B%22property_name%22%3A%22page.info.path%22%2'
                                   'C%22operator%22%3A%22eq%22%2C%22property_value%22%3'
                                   'A%22{}%22%7D%5D'.format(record._id)}

                    resp = requests.get(url, query)
                    record.visit = resp.json()['result']
                    record.save()
                    count += 1

        logger.info('Get visit counts for {} projects in Conference {}'.format(count, conf.name))
    

@celery_app.task(name='scripts.meeting_visit_count')
def run_main(dry_run=True):
    scripts_utils.add_file_logger(logger, __file__)
    with TokuTransaction():
        main(dry_run=dry_run)<commit_msg>change the filter to be a json object and url encode it<commit_after>#!/usr/bin/env python
# encoding: utf-8

import logging
import json
import requests
import urllib

from modularodm import Q

from framework.celery_tasks import app as celery_app

from website import settings
from website.app import init_app
from framework.transactions.context import TokuTransaction
from website.models import Tag, Conference, Node
from website.files.models import StoredFileNode
from scripts import utils as scripts_utils

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)


def main(dry_run=True):
    init_app(routes=False)
    for conf in Conference.find():
        count = 0
        for tag in Tag.find(Q('lower', 'eq', conf.endpoint.lower())):
            for node in Node.find(Q('is_public', 'eq', True) & Q('is_deleted', 'eq', False) & Q('tags', 'eq', tag)):
                record = next(
                    x for x in
                    StoredFileNode.find(
                        Q('node', 'eq', node) &
                        Q('is_file', 'eq', True)
                    ).limit(1)
                ).wrapped()

                if not dry_run:
                    query = [{"property_name":"page.info.path",
                              "operator":"eq",
                              "property_value": record._id }]
                    query = urllib.quote(json.dumps(query))

                    url = 'https://api.keen.io/3.0/projects/{}/queries/count' \
                          '?api_key={}&event_collection=pageviews&timezone=UTC&timeframe' \
                          '=this_14_days&filters={}'.format(
                            settings.KEEN['public']['project_id'],
                            settings.KEEN['public']['read_key'],
                            query
                    )

                    resp = requests.get(url)
                    record.visit = resp.json()['result']
                    record.save()
                    count += 1

        logger.info('Get visit counts for {} projects in Conference {}'.format(count, conf.name))
    

@celery_app.task(name='scripts.meeting_visit_count')
def run_main(dry_run=True):
    scripts_utils.add_file_logger(logger, __file__)
    with TokuTransaction():
        main(dry_run=dry_run)
{'fraction_non_alphanumeric': 0.11716318363717208, 'fraction_numerical': 0.010893730546909737, 'mean_word_length': 1.891388174807198, 'pattern_counts': {'":': 3, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1338925', 'n_tokens_mistral': 1303, 'n_tokens_neox': 1228, 'n_words': 276}