<commit_before>#!/usr/bin/env python

import argparse
import collections
import itertools
import sys

import cloud
import numpy as np
import pandas as pd
import requests
import twitter

from lxml import etree


HTTP_TIMEOUT = 10
SECTOR_URL = 'http://biz.yahoo.com/p/sum_conameu.html'
INDUSTRY_URL = 'http://biz.yahoo.com/p/%(id)sconameu.html'
KEYSTATS_URL = 'http://finance.yahoo.com/q/ks?s=%(ticker)s'

EXCLUDED_INDUSTRIES = ('Accident & Health Insurance',
                       'Asset Management',
                       'Closed-End Fund - Debt',
                       'Closed-End Fund - Equity',
                       'Closed-End Fund - Foreign',
                       'Credit Services',
                       'Diversified Investments',
                       'Diversified Utilities',
                       'Electric Utilities',
                       'Foreign Money Center Banks',
                       'Foreign Regional Banks',
                       'Foreign Utilities',
                       'Gas Utilities',
                       'Independent Oil & Gas',
                       'Investment Brokerage - Regional',
                       'Investment Brokerage - Regional',
                       'Money Center Banks',
                       'Mortgage Investment',
                       'Regional - Mid-Atlantic Banks',
                       'Regional - Midwest Banks',
                       'Regional - Northeast Banks',
                       'Regional - Pacific Banks',
                       'Regional - Southeast Banks',
                       'Regional - Southwest Banks',
                       'Water Utilites')


Pipeline = collections.namedtuple('Pipeline',
                                 ('industry', 'ticker', 'keystats'))


def publish_to_twitter(df, prefix='MF', **kwargs):
    api = twitter.Api(**kwargs)
    msg = ' '.join(['$%s' % s for s in df.keys()])
    msg = '%s: %s' % (prefix, msg)
    return api.PostUpdate(msg)


def pairwise(iterable):
    args = [iter(iterable)] * 2
    return itertools.izip_longest(fillvalue=None, *args)


def chunk_list(lst, chunk_size):
    """split a lst into multiple lists of length chunk_size"""
    return [lst[i:i+chunk_size] for i in range(0, len(lst), chunk_size)]<commit_msg>Ensure print order is index order<commit_after>#!/usr/bin/env python

import argparse
import collections
import itertools
import sys

import cloud
import numpy as np
import pandas as pd
import requests
import twitter

from lxml import etree


HTTP_TIMEOUT = 10
SECTOR_URL = 'http://biz.yahoo.com/p/sum_conameu.html'
INDUSTRY_URL = 'http://biz.yahoo.com/p/%(id)sconameu.html'
KEYSTATS_URL = 'http://finance.yahoo.com/q/ks?s=%(ticker)s'

EXCLUDED_INDUSTRIES = ('Accident & Health Insurance',
                       'Asset Management',
                       'Closed-End Fund - Debt',
                       'Closed-End Fund - Equity',
                       'Closed-End Fund - Foreign',
                       'Credit Services',
                       'Diversified Investments',
                       'Diversified Utilities',
                       'Electric Utilities',
                       'Foreign Money Center Banks',
                       'Foreign Regional Banks',
                       'Foreign Utilities',
                       'Gas Utilities',
                       'Independent Oil & Gas',
                       'Investment Brokerage - Regional',
                       'Investment Brokerage - Regional',
                       'Money Center Banks',
                       'Mortgage Investment',
                       'Regional - Mid-Atlantic Banks',
                       'Regional - Midwest Banks',
                       'Regional - Northeast Banks',
                       'Regional - Pacific Banks',
                       'Regional - Southeast Banks',
                       'Regional - Southwest Banks',
                       'Water Utilites')


Pipeline = collections.namedtuple('Pipeline',
                                 ('industry', 'ticker', 'keystats'))


def publish_to_twitter(df, prefix='MF', **kwargs):
    api = twitter.Api(**kwargs)
    msg = ' '.join(['$%s' % s for s in df.index])
    msg = '%s: %s' % (prefix, msg)
    return api.PostUpdate(msg)


def pairwise(iterable):
    args = [iter(iterable)] * 2
    return itertools.izip_longest(fillvalue=None, *args)


def chunk_list(lst, chunk_size):
    """split a lst into multiple lists of length chunk_size"""
    return [lst[i:i+chunk_size] for i in range(0, len(lst), chunk_size)]
{'fraction_non_alphanumeric': 0.10651933701657458, 'fraction_numerical': 0.0017679558011049724, 'mean_word_length': 1.6283391405342624, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1984212', 'n_tokens_mistral': 1296, 'n_tokens_neox': 1162, 'n_words': 318}