<commit_before>from datetime import datetime
from grow.common import utils
from grow.pods import locales as locales_lib
from grow.pods.documents import collection
import collections
import csv as csv_lib
import itertools
import jinja2
import json as json_lib
import markdown
import re


@utils.memoize_tag
def categories(collection=None, collections=None, reverse=None, order_by=None,
               _pod=None):
  if isinstance(collection, collection.Collection):
    collection = collection
  elif isinstance(collection, basestring):
    collection = _pod.get_collection(collection)
  else:
    text = '{} must be a Collection instance or a collection path, found: {}.'
    raise ValueError(text.format(collection, type(collection)))

  category_list = collection.list_categories()

  def order_func(doc):
    return category_list.index(doc.category)

  docs = [doc for doc in collection.list_docs(reverse=reverse)]
  docs = sorted(docs, key=order_func)
  items = itertools.groupby(docs, key=order_func)
  return ((category_list[index], pages) for index, pages in items)


def LocaleIterator(iterator, locale):
  locale = str(locale)
  for i, line in enumerate(iterator):
    if i == 0 or line.startswith(locale):
      yield line


_no_locale = '__no_locale'


@utils.memoize_tag
def csv(path, locale=_no_locale, _pod=None):
  fp = _pod.open_file(path)
  if locale is not _no_locale:
    fp = LocaleIterator(fp, locale=locale)
  rows = []
  for row in csv_lib.DictReader(fp):
    data = {}
    for header, cell in row.iteritems():
      if cell is None:
        cell = ''
      data[header] = cell.decode('utf-8')
    rows.append(data)
  return rows


@utils.memoize_tag
def docs(collection, locale=None, order_by=None, _pod=None):
  collection = _pod.get_collection(collection)
  return collection.list_docs(locale=locale, order_by=order_by)


@utils.memoize_tag
def statics(pod_path, locale=None, _pod=None):
  return _pod.list_statics(pod_path, locale=locale)


def markdown_filter(value):
  try:
    if isinstance(value, unicode):
      value = value.decode('utf-8')
    return markdown.markdown(value)
  except UnicodeEncodeError:
    return markdown.markdown(value)<commit_msg>Fix caching issue with collections.<commit_after>from datetime import datetime
from grow.common import utils
from grow.pods import locales as locales_lib
from grow.pods.documents import collection as collection_lib
import collections
import csv as csv_lib
import itertools
import jinja2
import json as json_lib
import markdown
import re


def categories(collection=None, collections=None, reverse=None, order_by=None,
               _pod=None, use_cache=False):
  if isinstance(collection, collection_lib.Collection):
    collection = collection
  elif isinstance(collection, basestring):
    collection = _pod.get_collection(collection)
  else:
    text = '{} must be a Collection instance or a collection path, found: {}.'
    raise ValueError(text.format(collection, type(collection)))

  category_list = collection.list_categories()

  def order_func(doc):
    return category_list.index(doc.category)

  docs = [doc for doc in collection.list_docs(reverse=reverse)]
  docs = sorted(docs, key=order_func)
  items = itertools.groupby(docs, key=order_func)
  return ((category_list[index], pages) for index, pages in items)


def LocaleIterator(iterator, locale):
  locale = str(locale)
  for i, line in enumerate(iterator):
    if i == 0 or line.startswith(locale):
      yield line


_no_locale = '__no_locale'


@utils.memoize_tag
def csv(path, locale=_no_locale, _pod=None):
  fp = _pod.open_file(path)
  if locale is not _no_locale:
    fp = LocaleIterator(fp, locale=locale)
  rows = []
  for row in csv_lib.DictReader(fp):
    data = {}
    for header, cell in row.iteritems():
      if cell is None:
        cell = ''
      data[header] = cell.decode('utf-8')
    rows.append(data)
  return rows


@utils.memoize_tag
def docs(collection, locale=None, order_by=None, _pod=None):
  collection = _pod.get_collection(collection)
  return collection.list_docs(locale=locale, order_by=order_by)


@utils.memoize_tag
def collections(path, _pod=None):
  return _pod.get_collection(collection)


@utils.memoize_tag
def statics(pod_path, locale=None, _pod=None):
  return _pod.list_statics(pod_path, locale=locale)


def markdown_filter(value):
  try:
    if isinstance(value, unicode):
      value = value.decode('utf-8')
    return markdown.markdown(value)
  except UnicodeEncodeError:
    return markdown.markdown(value)
{'fraction_non_alphanumeric': 0.09915705412599822, 'fraction_numerical': 0.001774622892635315, 'mean_word_length': 4.601242236024845, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '916283', 'n_tokens_mistral': 1441, 'n_tokens_neox': 1397, 'n_words': 375}