<commit_before>import sys
from functools import partial
from itertools import islice

from discodb import DiscoDB
from flask import Flask, render_template, request, g


app = Flask(__name__)


def count(iter):
  return sum(1 for _ in iter)

def query(db, q_str):
  if q_str:
    for subquery, values in db.metaquery(q_str):
      yield str(subquery), count(values)

# expects list to be sorted
def slice(items, size, dir, label="Other"):
  if dir == "ASC":
    i = iter(items)
  else:
    i = reversed(items)

  for item in islice(i, size):
    yield item
  yield label, sum(item[1] for item in i)


def top(items, size=5):
  return slice(items, size, "DESC")

def bottom(items, size=5):
  return slice(items, size, "ASC")

def value((k,count)):
  return k.split(':',1)[1], count
<commit_msg>slice() used by top() and bottom() now handles both directions<commit_after>import os
import sys
from functools import partial
from itertools import islice, chain

from discodb import DiscoDB
import boto
from flask import Flask, render_template, request, g


app = Flask(__name__)


def count(iter):
  return sum(1 for _ in iter)

def query(db, q_str):
  if q_str:
    for subquery, values in db.metaquery(q_str):
      yield str(subquery), count(values)

# expects list to be sorted
def slice(items, size, dir, label="Other"):
  if dir == "ASC":
    i = iter(items)
  elif dir == "DESC":
    i = reversed(items)
  else:
    raise ValueError("dir must be ASC|DESC")

  s     = islice(i, size)
  other = label, sum(item[1] for item in i)

  # note other is always last regardless of it's
  # value... not sure if that's proper or not
  if dir == "ASC":
    return chain(s, (other,))
  else:
    return chain(reversed(s), (other,))

def top(items, size=5):
  return slice(items, size, "DESC")

def bottom(items, size=5):
  return slice(items, size, "ASC")

def value((k,count)):
  return k.split(':',1)[1], count

{'fraction_non_alphanumeric': 0.12664214398318444, 'fraction_numerical': 0.006305832895428271, 'mean_word_length': 3.9583333333333335, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3455019', 'n_tokens_mistral': 669, 'n_tokens_neox': 655, 'n_words': 232}