# Path: context/config.py
# def get_twitter_keys(section='context'):
#     """Return namedtuple of Twitter config data"""
#     config = configuration()
#     TwitterKeys = collections.namedtuple('TwitterKeys', [
#         'consumer_key',
#         'consumer_secret',
#         'access_token',
#         'access_token_secret'])
#     k = TwitterKeys(
#         config.get(section, 'twitter_consumer_key'),
#         config.get(section, 'twitter_consumer_secret'),
#         config.get(section, 'twitter_access_token'),
#         config.get(section, 'twitter_access_token_secret'))
#     return k
# 
# def get_named_stoplist(name):
#     """Return named stoplist."""
#     config = configuration()
#     stoplist = config.get('stoplists', name)
#     return [s.strip() for s in CONFIG_LIST_REGEX.split(stoplist) if s.strip()]
# 
# Path: context/content.py
# def all_page_text(url):
#     try:
#         html = requests.get(url).text
#         doc = BeautifulSoup.BeautifulSoup(html)
#         return ' '.join([t for t in doc.findAll(text=True) if t.strip()])
#     except requests.ConnectionError:
#         return ''
# 
# Path: context/nlp/keywords.py
# def check_keywords(timeline, keywords):
#     tweets = []
#     keywords = [k['keyword'] for k in keywords]
#     for tweet in timeline:
#         score = 0
#         for kw in keywords:
#             if kw.lower() in tweet.text.lower():
#                 score += 1
#         if score > 0:
#             tweets.append({
#                 'tweet': tweet,
#                 'score': score})
#     tweets = sorted(tweets, key = lambda x: x['score'], reverse=True)
#     return tweets
# 
# Path: context/nlp/entities.py
# def collapse_entities(entity_list):
#     """Return unique list of entity names collapsed into shortest forms"""
#     name_list = []
# 
#     for e in entity_list:
#         name_forms = sorted(e['name_forms'], key=lambda s: len(s))
#         while name_forms:
#             name = name_forms.pop(0)
#             for i in range(len(name_forms) - 1, -1, -1):
#                 s = name_forms[i]
#                 if s.find(name) > -1:
#                     del name_forms[i]
#             if name not in name_list:
#                 name_list.append(name)
#         name_list.extend([s for s in name_forms if s not in name_list])
# 
#     return name_list
# 
# Path: context/nlp/util.py
# def tokenize(text):
#     """Return tokenized string"""
#     text = text.replace('@', '__at__').replace('#', '__hash__')
#     tokens = nltk.word_tokenize(text)
#     tokens = [t.replace('__at__', '@').replace('__hash__', '#')
#         for t in tokens]
#     return tokens
# 
# def compare_names(namepartsA, namepartsB):
#     """Takes two name-parts lists (as lists of words) and returns a score."""
#     complement = set(namepartsA) ^ set(namepartsB)
#     intersection = set(namepartsA) & set(namepartsB)
#     score = float(len(intersection))/(len(intersection)+len(complement))
#     return score
# 
# def name_parts(names, flat=False):
#     """Return list of words within a name"""
#     parts = []
#     for name in names:
#         n = ALNUM_PAT.findall(name)
#         if flat:
#             parts.extend(n)
#         else:
#             parts.append(n)
#     return parts
# 
# ALNUM_PAT = re.compile(r'[A-Za-z0-9]+')


import copy
import os
import re
import urllib
import requests
import string
import birdy.twitter
from collections import defaultdict, OrderedDict
from birdy.twitter import TwitterAuthError
from ..config import get_twitter_keys, get_named_stoplist
from ..content import all_page_text
from ..nlp.keywords import check_keywords
from ..nlp.entities import collapse_entities
from ..nlp.util import tokenize, compare_names, name_parts, ALNUM_PAT
from nltk import ngrams
from nltk.corpus import stopwords
# -*- coding: utf-8 -*-

URL_PAT = re.compile(r'http[^ ]+', re.I)
USER_PAT = re.compile(r'@\w{1,15}')    

LEAD_PAT = re.compile(u'^(RT)?\s*[.?!:;,\'"‘’“”—…\s]*', re.U | re.I)     
TRAIL_PAT = re.compile(u'[\s.:—…]+$', re.U)


class UserClient(birdy.twitter.UserClient):
    """Twitter UserClient"""
    def __repr__(self):
        return dict(self).__repr__


def get_client(credentials=None):
    """Get credentialed Twitter UserClient"""
    tk = get_twitter_keys()._replace(**credentials or {})
    return UserClient(*tk)


def best_users(entity_names, twitter_objs):
    twitter_name_parts = name_parts([obj['name'] for obj in twitter_objs])

# Path: context/config.py
# def get_twitter_keys(section='context'):
#     """Return namedtuple of Twitter config data"""
#     config = configuration()
#     TwitterKeys = collections.namedtuple('TwitterKeys', [
#         'consumer_key',
#         'consumer_secret',
#         'access_token',
#         'access_token_secret'])
#     k = TwitterKeys(
#         config.get(section, 'twitter_consumer_key'),
#         config.get(section, 'twitter_consumer_secret'),
#         config.get(section, 'twitter_access_token'),
#         config.get(section, 'twitter_access_token_secret'))
#     return k
# 
# def get_named_stoplist(name):
#     """Return named stoplist."""
#     config = configuration()
#     stoplist = config.get('stoplists', name)
#     return [s.strip() for s in CONFIG_LIST_REGEX.split(stoplist) if s.strip()]
# 
# Path: context/content.py
# def all_page_text(url):
#     try:
#         html = requests.get(url).text
#         doc = BeautifulSoup.BeautifulSoup(html)
#         return ' '.join([t for t in doc.findAll(text=True) if t.strip()])
#     except requests.ConnectionError:
#         return ''
# 
# Path: context/nlp/keywords.py
# def check_keywords(timeline, keywords):
#     tweets = []
#     keywords = [k['keyword'] for k in keywords]
#     for tweet in timeline:
#         score = 0
#         for kw in keywords:
#             if kw.lower() in tweet.text.lower():
#                 score += 1
#         if score > 0:
#             tweets.append({
#                 'tweet': tweet,
#                 'score': score})
#     tweets = sorted(tweets, key = lambda x: x['score'], reverse=True)
#     return tweets
# 
# Path: context/nlp/entities.py
# def collapse_entities(entity_list):
#     """Return unique list of entity names collapsed into shortest forms"""
#     name_list = []
# 
#     for e in entity_list:
#         name_forms = sorted(e['name_forms'], key=lambda s: len(s))
#         while name_forms:
#             name = name_forms.pop(0)
#             for i in range(len(name_forms) - 1, -1, -1):
#                 s = name_forms[i]
#                 if s.find(name) > -1:
#                     del name_forms[i]
#             if name not in name_list:
#                 name_list.append(name)
#         name_list.extend([s for s in name_forms if s not in name_list])
# 
#     return name_list
# 
# Path: context/nlp/util.py
# def tokenize(text):
#     """Return tokenized string"""
#     text = text.replace('@', '__at__').replace('#', '__hash__')
#     tokens = nltk.word_tokenize(text)
#     tokens = [t.replace('__at__', '@').replace('__hash__', '#')
#         for t in tokens]
#     return tokens
# 
# def compare_names(namepartsA, namepartsB):
#     """Takes two name-parts lists (as lists of words) and returns a score."""
#     complement = set(namepartsA) ^ set(namepartsB)
#     intersection = set(namepartsA) & set(namepartsB)
#     score = float(len(intersection))/(len(intersection)+len(complement))
#     return score
# 
# def name_parts(names, flat=False):
#     """Return list of words within a name"""
#     parts = []
#     for name in names:
#         n = ALNUM_PAT.findall(name)
#         if flat:
#             parts.extend(n)
#         else:
#             parts.append(n)
#     return parts
# 
# ALNUM_PAT = re.compile(r'[A-Za-z0-9]+')


# Path: context/services/twitter.py
import copy
import os
import re
import urllib
import requests
import string
import birdy.twitter
from collections import defaultdict, OrderedDict
from birdy.twitter import TwitterAuthError
from ..config import get_twitter_keys, get_named_stoplist
from ..content import all_page_text
from ..nlp.keywords import check_keywords
from ..nlp.entities import collapse_entities
from ..nlp.util import tokenize, compare_names, name_parts, ALNUM_PAT
from nltk import ngrams
from nltk.corpus import stopwords

# -*- coding: utf-8 -*-

URL_PAT = re.compile(r'http[^ ]+', re.I)
USER_PAT = re.compile(r'@\w{1,15}')    

LEAD_PAT = re.compile(u'^(RT)?\s*[.?!:;,\'"‘’“”—…\s]*', re.U | re.I)     
TRAIL_PAT = re.compile(u'[\s.:—…]+$', re.U)


class UserClient(birdy.twitter.UserClient):
    """Twitter UserClient"""
    def __repr__(self):
        return dict(self).__repr__


def get_client(credentials=None):
    """Get credentialed Twitter UserClient"""
    tk = get_twitter_keys()._replace(**credentials or {})
    return UserClient(*tk)


def best_users(entity_names, twitter_objs):
    twitter_name_parts = name_parts([obj['name'] for obj in twitter_objs])

    entity_name_parts = name_parts(entity_names, flat=True)