<commit_before>from time import gmtime
from calendar import timegm
from datetime import datetime

from dateutil.relativedelta import relativedelta

from elasticsearch import Elasticsearch

from website import settings

share_es = Elasticsearch(
    settings.SHARE_ELASTIC_URI,
    request_timeout=settings.ELASTIC_TIMEOUT
)

def search(query):
    # Run the real query and get the results
    raw_results = share_es.search(index='share', doc_type=None, body=query)

    results = [hit['_source'] for hit in raw_results['hits']['hits']]
    return {
        'results': results,
        'count': raw_results['hits']['total'],
    }

def count(query):
    if query.get('from') is not None:
        del query['from']
    if query.get('size') is not None:
        del query['size']

    count = share_es.count(index='share', body=query)

    return {
        'results': [],
        'count': count['count']
    }


def stats(query=dict()):
    query['aggs'] = {
        "sources": {
            "terms": {
                "field": "_type",
                "size": 0,
                "exclude": "of|and|or"
            }
        },
        "doisMissing": {
            "filter": {
                "missing": {
                    "field": "id.doi"
                }
            },
            "aggs": {
                "sources": {
                    "terms": {
                        "field": "_type",
                        "size": 0
                    }
                }
            }
        },
        "dois": {
            "filter": {
                "exists": {
                    "field": "id.doi"
                }
            },
            "aggs": {
                "sources": {
                    "terms": {
                        "field": "_type",
                        "size": 0
                    }
                }
            }
        },
        "date_chunks": {
            "terms": {
                "field": "_type",
                "size": 0,
                "exclude": "of|and|or"
            },
            "aggs": {
                "articles_over_time": {
                    "date_histogram": {
                        "field": "dateUpdated",
                        "interval": "month",
                        "min_doc_count": 0,
                        "extended_bounds": {
                            "min": timegm((datetime.now() + relativedelta(months=-3)).timetuple()) * 1000,
                            "max": timegm(gmtime()) * 1000
                        }
                    }
                }
            }
        }
    }

    results = share_es.search(index='share', body=query)

    chart_results = data_for_charts(results)

    return chart_results


def data_for_charts(elastic_results):
    source_data = elastic_results['aggregations']['sources']['buckets']
    for_charts = {}

    ## for the donut graph list of many lists, source and count
    source_and_counts = [[item['key'], item['doc_count']] for item in source_data]
    for_charts['donut_chart'] = source_and_counts

    all_date_data = []
    # for the date aggregations
    chunk_data = elastic_results['aggregations']['date_chunks']['buckets']
    date_names = ['x']
    for entry in chunk_data[0]['articles_over_time']['buckets']:
        date_names.append(entry['key_as_string'].replace('T00:00:00.000Z', ''))

    all_date_data.append(date_names)

    all_source_names = []
    for bucket in chunk_data:
        provider_row = [bucket['key']]
        all_source_names.append(bucket['key'])
        for internal in bucket['articles_over_time']['buckets']:
            provider_row.append(internal['doc_count'])
        provider_cumulitive = [sum(provider_row[1:i+2]) for i in range(len(provider_row[1:]))]
        provider_cumulitive = [bucket['key']] + provider_cumulitive
        all_date_data.append(provider_cumulitive)

    for_charts['date_totals'] = {}
    for_charts['date_totals']['date_numbers'] = all_date_data
    for_charts['date_totals']['group_names'] = ['x'] + all_source_names

    all_data = {}
    all_data['raw_aggregations'] = elastic_results['aggregations']
    all_data['for_charts'] = for_charts
<commit_msg>filters on date range for last 3 months<commit_after>from time import gmtime
from calendar import timegm
from datetime import datetime

from dateutil.relativedelta import relativedelta

from elasticsearch import Elasticsearch

from website import settings

share_es = Elasticsearch(
    settings.SHARE_ELASTIC_URI,
    request_timeout=settings.ELASTIC_TIMEOUT
)

def search(query):
    # Run the real query and get the results
    raw_results = share_es.search(index='share', doc_type=None, body=query)

    results = [hit['_source'] for hit in raw_results['hits']['hits']]
    return {
        'results': results,
        'count': raw_results['hits']['total'],
    }

def count(query):
    if query.get('from') is not None:
        del query['from']
    if query.get('size') is not None:
        del query['size']

    count = share_es.count(index='share', body=query)

    return {
        'results': [],
        'count': count['count']
    }


def stats(query=dict()):
    three_months_ago = timegm((datetime.now() + relativedelta(months=-3)).timetuple()) * 1000
    query['aggs'] = {
        "sources": {
            "terms": {
                "field": "_type",
                "size": 0,
                "exclude": "of|and|or"
            }
        },
        "doisMissing": {
            "filter": {
                "missing": {
                    "field": "id.doi"
                }
            },
            "aggs": {
                "sources": {
                    "terms": {
                        "field": "_type",
                        "size": 0
                    }
                }
            }
        },
        "dois": {
            "filter": {
                "exists": {
                    "field": "id.doi"
                }
            },
            "aggs": {
                "sources": {
                    "terms": {
                        "field": "_type",
                        "size": 0
                    }
                }
            }
        },
        "date_chunks": {
            "terms": {
                "field": "_type",
                "size": 0,
                "exclude": "of|and|or"
            },
            "aggs": {
                "articles_over_time": {
                    "date_histogram": {
                        "field": "dateUpdated",
                        "interval": "month",
                        "min_doc_count": 0,
                        "extended_bounds": {
                            "min": three_months_ago,
                            "max": timegm(gmtime()) * 1000
                        }
                    },
                    "aggs": {
                        "date_filter": {
                            "filter": {
                                "range": {
                                    "dateUpdated": {
                                        "lt": three_months_ago
                                    }
                                }
                            }
                        }
                    }
                }
            }
        }
    }


    results = share_es.search(index='share', body=query)

    chart_results = data_for_charts(results)

    return chart_results


def data_for_charts(elastic_results):
    import json
    with open('results.json', 'w') as f: f.write(json.dumps(elastic_results, indent=4))
    source_data = elastic_results['aggregations']['sources']['buckets']
    for_charts = {}

    ## for the donut graph list of many lists, source and count
    source_and_counts = [[item['key'], item['doc_count']] for item in source_data]
    for_charts['donut_chart'] = source_and_counts

    all_date_data = []
    # for the date aggregations
    chunk_data = elastic_results['aggregations']['date_chunks']['buckets']
    date_names = ['x']
    for entry in chunk_data[0]['articles_over_time']['buckets']:
        date_names.append(entry['key_as_string'].replace('T00:00:00.000Z', ''))

    all_date_data.append(date_names)

    all_source_names = []
    for bucket in chunk_data:
        provider_row = [bucket['key']]
        all_source_names.append(bucket['key'])
        for internal in bucket['articles_over_time']['buckets']:
            provider_row.append(internal['doc_count'])
        provider_cumulative = [sum(provider_row[1:i+2]) for i in range(len(provider_row[1:]))]
        provider_cumulative = [bucket['key']] + provider_cumulative
        all_date_data.append(provider_cumulative)

    for_charts['date_totals'] = {}
    for_charts['date_totals']['date_numbers'] = all_date_data
    for_charts['date_totals']['group_names'] = ['x'] + all_source_names

    all_data = {}
    all_data['raw_aggregations'] = elastic_results['aggregations']
    all_data['for_charts'] = for_charts

{'fraction_non_alphanumeric': 0.12851360467731054, 'fraction_numerical': 0.006296379581740499, 'mean_word_length': 1.5985977212971079, 'pattern_counts': {'":': 80, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6177303', 'n_tokens_mistral': 2677, 'n_tokens_neox': 2456, 'n_words': 459}