<commit_before>import datetime
import json

from pylons import g, c
from sqlalchemy.sql.expression import select, distinct, func

from r2.lib.base import abort
from r2.lib.errors import errors
from r2.lib.hooks import HookRegistrar
from r2.lib.memoize import memoize
from r2.models import Subreddit, Link, Comment
from r2.models.gold import gold_table, ENGINE, TIMEZONE


hooks = HookRegistrar()


@memoize("gold-buyers", time=86400)
def gold_buyers_on(date):
    start_date = datetime.datetime.combine(date, datetime.time.min)
    end_date = datetime.datetime.combine(date, datetime.time.max)

    NON_REVENUE_STATUSES = ("declined", "chargeback", "fudge")
    date_expr = func.timezone(TIMEZONE.zone, gold_table.c.date)
    query = (select([distinct(gold_table.c.account_id)])
                .where(~ gold_table.c.status.in_(NON_REVENUE_STATUSES))
                .where(date_expr >= start_date)
                .where(date_expr <= end_date)
                .where(gold_table.c.pennies > 0)
            )
    rows = ENGINE.execute(query)
    return [int(account_id) for (account_id,) in rows.fetchall()]


def gold_buyers_yesterday():
    one_day = datetime.timedelta(days=1)
    yesterday = datetime.datetime.now(TIMEZONE).date() - one_day
    return gold_buyers_on(yesterday)


@hooks.on("reddit.request.begin")
def add_gold_hostname():
    c.gold_hostname = ""

    if g.gold_hostname_file:
        try:
            with open(g.gold_hostname_file) as f:
                c.gold_hostname = json.loads(f.read().strip())
        except IOError:
            pass


@hooks.on("subreddit.can_comment")
def nameaserver_can_comment(sr, user):
    if sr.name == g.gold_servername_sr:
        return sr.is_moderator(user) or user._id in gold_buyers_yesterday()
<commit_msg>temporary transition code to handle non-json<commit_after>import datetime
import json

from pylons import g, c
from sqlalchemy.sql.expression import select, distinct, func

from r2.lib.base import abort
from r2.lib.errors import errors
from r2.lib.hooks import HookRegistrar
from r2.lib.memoize import memoize
from r2.models import Subreddit, Link, Comment
from r2.models.gold import gold_table, ENGINE, TIMEZONE


hooks = HookRegistrar()


@memoize("gold-buyers", time=86400)
def gold_buyers_on(date):
    start_date = datetime.datetime.combine(date, datetime.time.min)
    end_date = datetime.datetime.combine(date, datetime.time.max)

    NON_REVENUE_STATUSES = ("declined", "chargeback", "fudge")
    date_expr = func.timezone(TIMEZONE.zone, gold_table.c.date)
    query = (select([distinct(gold_table.c.account_id)])
                .where(~ gold_table.c.status.in_(NON_REVENUE_STATUSES))
                .where(date_expr >= start_date)
                .where(date_expr <= end_date)
                .where(gold_table.c.pennies > 0)
            )
    rows = ENGINE.execute(query)
    return [int(account_id) for (account_id,) in rows.fetchall()]


def gold_buyers_yesterday():
    one_day = datetime.timedelta(days=1)
    yesterday = datetime.datetime.now(TIMEZONE).date() - one_day
    return gold_buyers_on(yesterday)


@hooks.on("reddit.request.begin")
def add_gold_hostname():
    c.gold_hostname = ""

    if g.gold_hostname_file:
        try:
            with open(g.gold_hostname_file) as f:
                contents = f.read().strip()
                try:
                    c.gold_hostname = json.loads(contents)
                except ValueError:
                    c.gold_hostname = {'name': contents,
                                       'permalink': 'http://www.reddit.com/r/' + g.gold_servername_sr}
        except IOError:
            pass


@hooks.on("subreddit.can_comment")
def nameaserver_can_comment(sr, user):
    if sr.name == g.gold_servername_sr:
        return sr.is_moderator(user) or user._id in gold_buyers_yesterday()

{'fraction_non_alphanumeric': 0.10986136541982736, 'fraction_numerical': 0.006800941668846455, 'mean_word_length': 3.0508474576271185, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1334825', 'n_tokens_mistral': 1317, 'n_tokens_neox': 1280, 'n_words': 252}