<commit_before>import os
import logging
from csv import DictReader
from itertools import groupby
from operator import itemgetter
from StringIO import StringIO
from datetime import datetime
from urllib2 import HTTPError, URLError
from urlparse import urlparse
from random import shuffle
from argparse import ArgumentParser
from time import time
from re import match

from requests import get, exceptions
from dateutil.tz import tzoffset
import feedparser

from feeds import get_first_working_feed_link
from app import db, Project, Organization, Story, Event, Error, Issue, Label, is_safe_name


# Logging Setup
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
requests_log = logging.getLogger("requests")
requests_log.setLevel(logging.WARNING)

# :NOTE: debug
# import warnings
# warnings.filterwarnings('error')

# Org sources can be csv or yaml
# They should be lists of organizations you want included at /organizations
# columns should be name, website, events_url, rss, projects_list_url, city, latitude, longitude, type
# set use_test to True to use test_org_sources.csv instead of org_sources.csv
use_test = True
ORG_SOURCES = u'{}org_sources.csv'.format(u'test_' if use_test else u'')

if 'GITHUB_TOKEN' in os.environ:
    github_auth = (os.environ['GITHUB_TOKEN'], '')
else:
    github_auth = None

if 'MEETUP_KEY' in os.environ:<commit_msg>no longer using test org sources<commit_after>import os
import logging
from csv import DictReader
from itertools import groupby
from operator import itemgetter
from StringIO import StringIO
from datetime import datetime
from urllib2 import HTTPError, URLError
from urlparse import urlparse
from random import shuffle
from argparse import ArgumentParser
from time import time
from re import match

from requests import get, exceptions
from dateutil.tz import tzoffset
import feedparser

from feeds import get_first_working_feed_link
from app import db, Project, Organization, Story, Event, Error, Issue, Label, is_safe_name


# Logging Setup
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
requests_log = logging.getLogger("requests")
requests_log.setLevel(logging.WARNING)

# :NOTE: debug
# import warnings
# warnings.filterwarnings('error')

# Org sources can be csv or yaml
# They should be lists of organizations you want included at /organizations
# columns should be name, website, events_url, rss, projects_list_url, city, latitude, longitude, type
# set use_test to True to use test_org_sources.csv instead of org_sources.csv
use_test = False
ORG_SOURCES = u'{}org_sources.csv'.format(u'test_' if use_test else u'')

if 'GITHUB_TOKEN' in os.environ:
    github_auth = (os.environ['GITHUB_TOKEN'], '')
else:
    github_auth = None

if 'MEETUP_KEY' in os.environ:
{'fraction_non_alphanumeric': 0.06777217015140591, 'fraction_numerical': 0.0007209805335255948, 'mean_word_length': 5.972361809045226, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5673128', 'n_tokens_mistral': 828, 'n_tokens_neox': 811, 'n_words': 304}