<commit_before>package bitesized

import (
	"time"

	"github.com/garyburd/redigo/redis"
)

type Retention map[string][]int

func (b *Bitesized) Retention(e string, f, t time.Time, i Interval) ([]Retention, error) {
	if f.After(t) {
		return nil, ErrFromAfterTill
	}

	e = dasherize(e)
	retentions := []Retention{}

	start := t
	for {
		end := start
		counts := []int{}

		for {
			sKey := b.intervalkey(e, start, i)
			eKey := b.intervalkey(e, end, i)
			rKey := randSeq(5)

			c, err := redis.Int(b.store.Do("BITOP", "AND", rKey, sKey, eKey))
			if err != nil {
				return nil, err
			}

			counts = append(counts, c)

			if _, err := b.store.Do("DEL", rKey); err != nil {
				return nil, err<commit_msg>do another query to get bit count of reult key<commit_after>package bitesized

import (
	"time"

	"github.com/garyburd/redigo/redis"
)

type Retention map[string][]int

func (b *Bitesized) Retention(e string, f, t time.Time, i Interval) ([]Retention, error) {
	if f.After(t) {
		return nil, ErrFromAfterTill
	}

	e = dasherize(e)
	retentions := []Retention{}

	start := t
	for {
		end := start
		counts := []int{}

		for {
			sKey := b.intervalkey(e, start, i)
			eKey := b.intervalkey(e, end, i)
			rKey := randSeq(5)

			if _, err := b.store.Do("BITOP", "AND", rKey, sKey, eKey); err != nil {
				return nil, err
			}

			c, err := redis.Int(b.store.Do("BITCOUNT", rKey))
			if err != nil {
				return nil, err
			}

			counts = append(counts, c)

			if _, err := b.store.Do("DEL", rKey); err != nil {
				return nil, err
{'fraction_non_alphanumeric': 0.1662303664921466, 'fraction_numerical': 0.0013089005235602095, 'mean_word_length': 3.143631436314363, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '597423', 'n_tokens_mistral': 688, 'n_tokens_neox': 609, 'n_words': 161}