<commit_before>import django
import urllib2
import pynvest_connect
from . import models

import os


def populate(*investments, **kwargs):
    fix_existing = kwargs.pop('fix_existing', False)
    output = kwargs.pop('output', open(os.devnull, 'w'))
    investments = investments or models.Investment.objects.all()

    num_investments = len(investments)
    for (index, investment) in enumerate(investments):
        if isinstance(investment, basestring):
            investment, created = Investment.objects.get_or_create(symbol=investment, defaults={'name': 'Placeholder'})

        output.write('%3d/%d %-11s ' % (index + 1, num_investments, investment.symbol))

        try:
            prices = pynvest_connect.historical_prices(investment.symbol)
        except urllib2.HTTPError, e:
            if e.code != 404:
                raise
            else:
                # Not found but continue anyway
                output.write('<404>\n')
                continue

        all_snapshots = dict((snapshot.date, snapshot) for snapshot in investment.snapshot_set.all())
        dirty_dates = set()
        for row in prices:
            snapshot = all_snapshots.setdefault(row.date, models.Snapshot(investment=investment, date=row.date))
            if (snapshot.high  == row.high and
                snapshot.low   == row.low and
                snapshot.close == row.close):
                if fix_existing:
                    continue
                else:
                    break

            snapshot.high = row.high
            snapshot.low = row.low
            snapshot.close = row.close
            dirty_dates.add(snapshot.date)

        dividends = pynvest_connect.dividends(investment.symbol)
        for dividend in dividends:
            snapshot = all_snapshots[dividend.date]
            if snapshot.dividend == dividend.amount:
                if fix_existing:
                    continue
                else:
                    break

            snapshot.dividend = dividend.amount
            dirty_dates.add(snapshot.date)

        splits = pynvest_connect.splits(investment.symbol)
        for split in splits:
            snapshot = all_snapshots[split.date]
            if snapshot.split_before == split.before and snapshot.split_after == split.after:
                if fix_existing:
                    continue
                else:
                    break

            snapshot.split_before = split.before
            snapshot.split_after = split.after
            dirty_dates.add(snapshot.date)

        if dirty_dates:
            if len(dirty_dates) > 1:
                output.write(min(dirty_dates).strftime('%Y-%m-%d - '))
            output.write(max(dirty_dates).strftime('%Y-%m-%d'))

        with django.db.transaction.commit_on_success():
            for date in dirty_dates:
                all_snapshots[date].save()

        output.write('\n')<commit_msg>Refactored batch logic<commit_after>import django
import urllib2
import pynvest_connect
from . import models

import os


def _populate_prices(investment, fix_existing):
    all_snapshots = dict((snapshot.date, snapshot) for snapshot in investment.snapshot_set.all())
    dirty_dates = set()
    for row in pynvest_connect.historical_prices(investment.symbol):
        snapshot = all_snapshots.setdefault(row.date, models.Snapshot(investment=investment, date=row.date))
        if (snapshot.high  == row.high and
            snapshot.low   == row.low and
            snapshot.close == row.close):
            if fix_existing:
                continue
            else:
                break

        snapshot.high = row.high
        snapshot.low = row.low
        snapshot.close = row.close
        snapshot.save()
        dirty_dates.add(snapshot.date)
    return dirty_dates

def _populate_dividends(investment, fix_existing):
    all_snapshots = dict((snapshot.date, snapshot) for snapshot in investment.snapshot_set.all())
    dirty_dates = set()
    for dividend in pynvest_connect.dividends(investment.symbol):
        snapshot = all_snapshots[dividend.date]
        if snapshot.dividend == dividend.amount:
            if fix_existing:
                continue
            else:
                break

        snapshot.dividend = dividend.amount
        snapshot.save()
        dirty_dates.add(snapshot.date)
    return dirty_dates

def _populate_splits(investment, fix_existing):
    all_snapshots = dict((snapshot.date, snapshot) for snapshot in investment.snapshot_set.all())
    dirty_dates = set()
    for split in pynvest_connect.splits(investment.symbol):
        snapshot = all_snapshots[split.date]
        if snapshot.split_before == split.before and snapshot.split_after == split.after:
            if fix_existing:
                continue
            else:
                break

        snapshot.split_before = split.before
        snapshot.split_after = split.after
        snapshot.save()
        dirty_dates.add(snapshot.date)
    return dirty_dates


def populate(*investments, **kwargs):
    fix_existing = kwargs.pop('fix_existing', False)
    output = kwargs.pop('output', open(os.devnull, 'w'))
    investments = investments or models.Investment.objects.all()

    num_investments = len(investments)
    for (index, investment) in enumerate(investments):
        if isinstance(investment, basestring):
            investment, created = Investment.objects.get_or_create(symbol=investment, defaults={'name': 'Placeholder'})

        output.write('%3d/%d %-11s ' % (index + 1, num_investments, investment.symbol))

        dirty_dates = set()
        try:
            with django.db.transaction.commit_on_success():
                dirty_dates.update(_populate_prices(investment, fix_existing))
                dirty_dates.update(_populate_dividends(investment, fix_existing))
                dirty_dates.update(_populate_splits(investment, fix_existing))
        except urllib2.HTTPError, e:
            if e.code != 404:
                raise
            else:
                # Not found but continue anyway
                output.write('<404>\n')
                continue

        if dirty_dates:
            if len(dirty_dates) > 1:
                output.write(min(dirty_dates).strftime('%Y-%m-%d - '))
            output.write(max(dirty_dates).strftime('%Y-%m-%d'))

        output.write('\n')
{'fraction_non_alphanumeric': 0.09578664983430646, 'fraction_numerical': 0.004102887801798958, 'mean_word_length': 2.1298765432098765, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1757327', 'n_tokens_mistral': 1812, 'n_tokens_neox': 1745, 'n_words': 354}