<commit_before>from __future__ import division

import numpy as np
import scipy
import scipy.stats as spstats

import util

class SpreadSpan(object):
    
    def __init__(self, data):
        '''Initialize spread/span statistics with VideoData.'''
        # Initialize statistics
        self.data = data
        self.spread_values = list()
        self.span_values = list()
        self.span_by_vid = {}
        self.spread_by_vid = {}
        
        # Calculate statistics
        for vid, dates_by_cid in data.dates_vid_cid.iteritems():
            spread = self.spread(dates_by_cid)
            span = self.span(dates_by_cid)
            self.spread_values.append(spread)
            self.span_values.append(span)
            self.span_by_vid[vid] = span
            self.spread_by_vid[vid] = spread
        self.min_spread = round(min(self.spread_values))
        self.max_spread = round(max(self.spread_values))
        self.min_span = round(min(self.span_values))
        self.max_span = round(max(self.span_values))
        self.spread_bins = self.max_spread - self.min_spread + 1
        self.span_bins = self.max_span - self.min_span + 1
    
    def span(self, dates_by_cid):
        '''Lifespan of a video.'''
        dates = sorted(set.union(*[set(dates) for dates in dates_by_cid.itervalues()]), reverse=True)
        high = low = dates.pop()
        # Videos may trend multiple times
        spans = []
        while dates:
            d = dates.pop()
            days_skipped = (d - high).days - 1
            lifespan = (high - low).days + 1
            if days_skipped > max([lifespan, 6]):
                spans.append(lifespan)
                low = d
            high = d
        spans.append((high - low).days + 1)
        return float(sum(spans) / len(spans))

    def spread(self, dates_by_cid):
        return len(dates_by_cid)

    def spread_hist(self):
        results = list()
        values = self.spread_values
        counts, bins = np.histogram(values, np.arange(0.5,max(values)+0.5))
        for i, count in enumerate(counts):
            results.append((bins[i], count / len(self.data.videos)))
        return results
    
    def span_hist(self):
        results = list()
        values = self.span_values
        counts, bins = np.histogram(values, np.arange(0.5,round(max(values))+0.5))
        for i, count in enumerate(counts):
            results.append((bins[i], count / len(self.data.videos)))
        return results
    
    def bin_edges(self):
        xedges = np.arange(self.min_spread - 0.5, self.max_spread + 1.5)
        yedges = np.arange(self.min_span - 0.5, self.max_span + 1.5)
        return (xedges, yedges)
    
    def spread_span_hist(self):
        xedges, yedges = self.bin_edges()
        h, xe, ye = np.histogram2d(
            self.spread_values
            , self.span_values
            , [xedges, yedges]
        )
        return h
    
    def mean_span(self):
        return sum(self.span_values) / len(self.span_values)
    
    def mean_spread(self):
        return sum(self.spread_values) / len(self.spread_values)
    
    def country_mean_span(self, cid):
        spans = []
        clu = self.data.country_lookup
        vlu = self.data.video_lookup
        # Count each video once for each day it trends
        for vid in self.data.vids_by_cid[cid]:
            span = self.span_by_vid[vid]
            count = self.data.counts[clu.tok2id[cid],vlu.tok2id[vid]]
            spans += [span] * count
        return sum(spans) / len(spans)
    
    def country_mean_spread(self, cid):
        spreads = []
        clu = self.data.country_lookup
        vlu = self.data.video_lookup
        # Count each video once for each day it trends
        for vid in self.data.vids_by_cid[cid]:
            spread = self.spread_by_vid[vid]
            count = self.data.counts[clu.tok2id[cid],vlu.tok2id[vid]]
            spreads += [spread] * count
        return sum(spreads) / len(spreads)
    
    def pearsonr(self):
        return spstats.pearsonr(self.spread_values, self.span_values)
<commit_msg>Add code to find spread/span distribution for a single country.<commit_after>from __future__ import division

import numpy as np
import scipy
import scipy.stats as spstats

import util

class SpreadSpan(object):
    
    def __init__(self, data):
        '''Initialize spread/span statistics with VideoData.'''
        # Initialize statistics
        self.data = data
        self.spread_values = list()
        self.span_values = list()
        self.span_by_vid = {}
        self.spread_by_vid = {}
        
        # Calculate statistics
        for vid, dates_by_cid in data.dates_vid_cid.iteritems():
            spread = self.spread(dates_by_cid)
            span = self.span(dates_by_cid)
            self.spread_values.append(spread)
            self.span_values.append(span)
            self.span_by_vid[vid] = span
            self.spread_by_vid[vid] = spread
        self.min_spread = round(min(self.spread_values))
        self.max_spread = round(max(self.spread_values))
        self.min_span = round(min(self.span_values))
        self.max_span = round(max(self.span_values))
        self.spread_bins = self.max_spread - self.min_spread + 1
        self.span_bins = self.max_span - self.min_span + 1
    
    def span(self, dates_by_cid):
        '''Lifespan of a video.'''
        dates = sorted(set.union(*[set(dates) for dates in dates_by_cid.itervalues()]), reverse=True)
        high = low = dates.pop()
        # Videos may trend multiple times
        spans = []
        while dates:
            d = dates.pop()
            days_skipped = (d - high).days - 1
            lifespan = (high - low).days + 1
            if days_skipped > max([lifespan, 6]):
                spans.append(lifespan)
                low = d
            high = d
        spans.append((high - low).days + 1)
        return float(sum(spans) / len(spans))

    def spread(self, dates_by_cid):
        return len(dates_by_cid)

    def spread_hist(self):
        results = list()
        values = self.spread_values
        counts, bins = np.histogram(values, np.arange(0.5,max(values)+0.5))
        for i, count in enumerate(counts):
            results.append((bins[i], count / len(self.data.videos)))
        return results
    
    def span_hist(self):
        results = list()
        values = self.span_values
        counts, bins = np.histogram(values, np.arange(0.5,round(max(values))+0.5))
        for i, count in enumerate(counts):
            results.append((bins[i], count / len(self.data.videos)))
        return results
    
    def bin_edges(self):
        xedges = np.arange(self.min_spread - 0.5, self.max_spread + 1.5)
        yedges = np.arange(self.min_span - 0.5, self.max_span + 1.5)
        return (xedges, yedges)
    
    def spread_span_hist(self):
        xedges, yedges = self.bin_edges()
        h, xe, ye = np.histogram2d(
            self.spread_values
            , self.span_values
            , [xedges, yedges]
        )
        return h
    
    def country_spread_span_hist(self, country):
        xedges, yedges = self.bin_edges()
        spread_values = list()
        span_values = list()
        for vid in self.data.vids_by_cid[country]:
            spread_values.append(self.spread_by_vid[vid])
            span_values.append(self.span_by_vid[vid])
        h, xe, ye = np.histogram2d(
            spread_values
            , span_values
            , [xedges, yedges]
        )
        return h
    
    def mean_span(self):
        return sum(self.span_values) / len(self.span_values)
    
    def mean_spread(self):
        return sum(self.spread_values) / len(self.spread_values)
    
    def country_mean_span(self, cid):
        spans = []
        clu = self.data.country_lookup
        vlu = self.data.video_lookup
        # Count each video once for each day it trends
        for vid in self.data.vids_by_cid[cid]:
            span = self.span_by_vid[vid]
            count = self.data.counts[clu.tok2id[cid],vlu.tok2id[vid]]
            spans += [span] * count
        return sum(spans) / len(spans)
    
    def country_mean_spread(self, cid):
        spreads = []
        clu = self.data.country_lookup
        vlu = self.data.video_lookup
        # Count each video once for each day it trends
        for vid in self.data.vids_by_cid[cid]:
            spread = self.spread_by_vid[vid]
            count = self.data.counts[clu.tok2id[cid],vlu.tok2id[vid]]
            spreads += [spread] * count
        return sum(spreads) / len(spreads)
    
    def pearsonr(self):
        return spstats.pearsonr(self.spread_values, self.span_values)

{'fraction_non_alphanumeric': 0.10805500982318271, 'fraction_numerical': 0.006356177048422513, 'mean_word_length': 2.0579505300353356, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3655792', 'n_tokens_mistral': 2855, 'n_tokens_neox': 2772, 'n_words': 577}