<commit_before>import pytz

import logging
import requests
from dateutil.parser import parse
from datetime import datetime, timedelta

from django.db.models import Q
from website.app import init_app
from scripts.analytics.base import SummaryAnalytics

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)

LOG_THRESHOLD = 11


class PreprintSummary(SummaryAnalytics):

    @property
    def collection_name(self):
        return 'preprint_summary'

    def get_events(self, date):
        super(PreprintSummary, self).get_events(date)
        from osf.models import PreprintService, PreprintProvider

        # Convert to a datetime at midnight for queries and the timestamp
        timestamp_datetime = datetime(date.year, date.month, date.day).replace(tzinfo=pytz.UTC)
        query_datetime = timestamp_datetime + timedelta(1)

        elastic_query = {
            "query": {
                "bool": {
                    "must": {
                        "match": {
                            "type": "preprints"
                        },
                        "match": {
                            "sources": None
                        }
                    },
                    'filter': [
                        {
                            "range": {
                                "date": {
                                    "lte": "{}||/d".format(query_datetime.strftime('%Y-%m-%d'))
                                }
                            }
                        }
                    ]
                }
            }
        }

        counts = []
        for preprint_provider in PreprintProvider.objects.all():
            name = preprint_provider.name if preprint_provider.name != 'Open Science Framework' else 'OSF'
            if name == '???':
                continue
            elastic_query['query']['bool']['must']['match']['sources'] = name
            resp = requests.post('https://share.osf.io/api/v2/search/creativeworks/_search', json=elastic_query).json()
            counts.append({
                'keen': {
                    'timestamp': timestamp_datetime.isoformat()
                },
                'provider': {
                    'name': preprint_provider.name,
                    'total': resp['hits']['total'],
                },
            })


        return counts


def get_class():
    return PreprintSummary


if __name__ == '__main__':
    init_app()

    preprint_summary = PreprintSummary()
    args = preprint_summary.parse_args()
    yesterday = args.yesterday
    if yesterday:
        date = (datetime.today() - timedelta(1)).date()
    else:
        date = parse(args.date).date() if args.date else None<commit_msg>Remove old test statement<commit_after>import pytz

import logging
import requests
from dateutil.parser import parse
from datetime import datetime, timedelta

from django.db.models import Q
from website.app import init_app
from scripts.analytics.base import SummaryAnalytics

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)

LOG_THRESHOLD = 11


class PreprintSummary(SummaryAnalytics):

    @property
    def collection_name(self):
        return 'preprint_summary'

    def get_events(self, date):
        super(PreprintSummary, self).get_events(date)
        from osf.models import PreprintService, PreprintProvider

        # Convert to a datetime at midnight for queries and the timestamp
        timestamp_datetime = datetime(date.year, date.month, date.day).replace(tzinfo=pytz.UTC)
        query_datetime = timestamp_datetime + timedelta(1)

        elastic_query = {
            "query": {
                "bool": {
                    "must": {
                        "match": {
                            "type": "preprints"
                        },
                        "match": {
                            "sources": None
                        }
                    },
                    'filter': [
                        {
                            "range": {
                                "date": {
                                    "lte": "{}||/d".format(query_datetime.strftime('%Y-%m-%d'))
                                }
                            }
                        }
                    ]
                }
            }
        }

        counts = []
        for preprint_provider in PreprintProvider.objects.all():
            name = preprint_provider.name if preprint_provider.name != 'Open Science Framework' else 'OSF'
            elastic_query['query']['bool']['must']['match']['sources'] = name
            resp = requests.post('https://share.osf.io/api/v2/search/creativeworks/_search', json=elastic_query).json()
            counts.append({
                'keen': {
                    'timestamp': timestamp_datetime.isoformat()
                },
                'provider': {
                    'name': preprint_provider.name,
                    'total': resp['hits']['total'],
                },
            })


        return counts


def get_class():
    return PreprintSummary


if __name__ == '__main__':
    init_app()

    preprint_summary = PreprintSummary()
    args = preprint_summary.parse_args()
    yesterday = args.yesterday
    if yesterday:
        date = (datetime.today() - timedelta(1)).date()
    else:
        date = parse(args.date).date() if args.date else None
{'fraction_non_alphanumeric': 0.10401628724782529, 'fraction_numerical': 0.0018508236165093468, 'mean_word_length': 1.4891754951635192, 'pattern_counts': {'":': 20, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5379756', 'n_tokens_mistral': 1407, 'n_tokens_neox': 1330, 'n_words': 281}