<commit_before>import collections
import json
import time

import isoweek

from django.http import HttpResponse
from django.views.decorators.cache import cache_page, cache_control
from django.shortcuts import get_object_or_404

from packages.models import Package, Release, DownloadDelta


def fetch_stats(package):
    releases = list(Release.objects.filter(package=package).order_by("order"))
    specific_releases = set([x.version for x in releases[-8:]])

    deltas = list(DownloadDelta.objects.filter(file__release__in=releases).order_by("date").select_related("file", "file__release"))

    # @@@ Sanity Checks
    if not deltas:
        return [{}]

    data = [{"name": "Other", "data": []}] + [{"name": release.version, "data": []} for release in releases if release.version in specific_releases]

    # Get First Week
    start_week = isoweek.Week.withdate(deltas[0].date)
    end_week = isoweek.Week.thisweek()

    current = isoweek.Week(start_week.year, start_week.week)

    while current.year <= end_week.year and current.week < end_week.week:
        for x in data:
            x["data"].append({"x": int(time.mktime(current.day(0).timetuple()))})
        current = isoweek.Week(current.year, current.week + 1)

    _data = collections.defaultdict(dict)<commit_msg>update fetch_stats to save on memory<commit_after>import collections
import json
import time

import isoweek

from django.http import HttpResponse
from django.views.decorators.cache import cache_page, cache_control
from django.shortcuts import get_object_or_404

from packages.models import Package, Release, DownloadDelta


def fetch_stats(package):
    releases = list(Release.objects.filter(package=package).only("version", "order").order_by("order"))
    specific_releases = set([x.version for x in releases[-8:]])

    deltas = list(DownloadDelta.objects.filter(file__release__in=releases).only("delta", "file__release__version").order_by("date").select_related("file", "file__release"))

    # @@@ Sanity Checks
    if not deltas:
        return [{}]

    data = [{"name": "Other", "data": []}] + [{"name": release.version, "data": []} for release in releases if release.version in specific_releases]

    # Get First Week
    start_week = isoweek.Week.withdate(deltas[0].date)
    end_week = isoweek.Week.thisweek()

    current = isoweek.Week(start_week.year, start_week.week)

    while current.year <= end_week.year and current.week < end_week.week:
        for x in data:
            x["data"].append({"x": int(time.mktime(current.day(0).timetuple()))})
        current = isoweek.Week(current.year, current.week + 1)

    _data = collections.defaultdict(dict)
{'fraction_non_alphanumeric': 0.13046757164404224, 'fraction_numerical': 0.005279034690799397, 'mean_word_length': 4.414285714285715, 'pattern_counts': {'":': 10, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2546475', 'n_tokens_mistral': 840, 'n_tokens_neox': 800, 'n_words': 184}