# Path: newsApp/cachingHelper.py
# def getCache():
#   servers = os.environ.get('MEMCACHIER_SERVERS', '').split(',')
#   user = os.environ.get('MEMCACHIER_USERNAME', '')
#   passw = os.environ.get('MEMCACHIER_PASSWORD', '')
# 
#   return bmemcached.Client(servers, user, passw)
# 
# Path: newsApp/cluster.py
# class Cluster(set):
#   """
#   Represents a cluster(set of similar documents)
#   """
# 
#   def __init__(self, docList):
#     """
#     Instantiates a new cluster object.
#     Requires 'docList': a list of docIds comprising the cluster
#     """
# 
#     set.__init__(self, docList)
# 
#     docList.sort()
#     self.id = hashlib.md5("-".join(docList).encode("latin1")).hexdigest().upper()
#     self.isCurrent = 'unknown'
#     self.isProcessed = False
# 
#   def process(self):
#     """
#     Processes the cluster to include metadata of consisting documents,
#     and overall metadata of cluster like category, location, feeds, etc.
#     """
# 
#     self.categories = []
#     self.countries = []
#     self.locales = []
#     self.publishers = []
#     self.languages = []
#     self.articles = [] # contains non-duplicate articles
#     self.duplicates = []
#     self.lastPubTime = 0
# 
#     docKeys = list(super(Cluster, self).__iter__())
#     docs = _getDocsInParallel(docKeys)
#     docsAdded = []
#     for doc in docs:
#       docKey = doc.key
#       if not _isDuplicateArticle(doc, docsAdded):
#         # note that this gets passed all the way till browser
#         # don't add any internal stuff here
#         self.articles.append({
#           'id': docKey,
#           'title': _getDocTitle(doc),
#           'publisher': doc.tags.get(TAG_PUBLISHER_DETAILS, ""),
#           'link': doc.tags.get(DOCTAG_URL, "#"),
#           'summaryText': _getDocSummary(doc),
#           'highlights': doc.tags.get(LINKTAG_HIGHLIGHTS, []),
#           'images': _getImagesForDoc(doc),
#           'lang': doc.tags.get(FEEDTAG_LANG, ""),
#           'publishedOn': doc.tags.get(LINKTAG_PUBTIME, 0)
#         })
#         docsAdded.append(doc)
#       else:
#         self.duplicates.append(docKey)
# 
#       if doc.tags.get(FEEDTAG_CATEGORY):
#         self.categories.append(doc.tags[FEEDTAG_CATEGORY])
#       if doc.tags.get(FEEDTAG_COUNTRY):
#         self.countries.append(doc.tags[FEEDTAG_COUNTRY])
#       if doc.tags.get(FEEDTAG_LOCALE):
#         self.locales.append(doc.tags[FEEDTAG_LOCALE])
#       if doc.tags.get(TAG_PUBLISHER):
#         self.publishers.append(doc.tags[TAG_PUBLISHER])
#       if doc.tags.get(FEEDTAG_LANG):
#         self.languages.append(doc.tags[FEEDTAG_LANG])
#       if doc.tags.get(LINKTAG_PUBTIME, 0) > self.lastPubTime:
#         self.lastPubTime =  doc.tags.get(LINKTAG_PUBTIME)
# 
#     nArticles = len(self.articles) + len(self.duplicates)
#     self.articles.sort(key=itemgetter('publishedOn'), reverse=True)
# 
#     #remove duplicates
#     self.categories = _removeDuplicatesAndOutliers(self.categories, nArticles)
#     self.countries = _removeDuplicatesAndOutliers(self.countries, nArticles)
#     self.locales = _removeDuplicatesAndOutliers(self.locales, nArticles)
#     self.publishers = list(set(self.publishers))
#     self.languages = list(set(self.languages))
#     self.isProcessed = True
# 
#   def deserializeFromString(self, clusterString):
#     clusterDict = json.loads(clusterString)
#     self.__init__(clusterDict['docKeys'])
#     self.articles = clusterDict['articles']
#     self.categories = clusterDict['categories']
#     self.countries = clusterDict['countries']
#     self.locales = clusterDict['locales']
#     self.publishers = clusterDict['publishers']
#     self.languages = clusterDict['languages']
#     self.duplicates = clusterDict['duplicates']
#     self.isCurrent = clusterDict['isCurrent']
#     self.lastPubTime = float(clusterDict['lastPubTime'])
#     self.isProcessed = True
# 
#   def serializeToString(self):
#     """
#     Serialize cluster to string. Only works for processed clusters
#     """
# 
#     if not self.isProcessed:
#       raise ValueError('Only processed clusters can be serialized')
# 
#     return json.dumps({
#       'clusterId': self.id,
#       'docKeys': list(super(Cluster, self).__iter__()),
#       'articles': self.articles,
#       'categories': self.categories,
#       'countries': self.countries,
#       'locales': self.locales,
#       'publishers': self.publishers,
#       'languages': self.languages,
#       'duplicates': self.duplicates,
#       'isCurrent': self.isCurrent,
#       'lastPubTime': self.lastPubTime
#     })
# 
# Path: newsApp/dbhelper.py
# def parseConnectionString(connectionString):
#     """
#     Parses the table connection string. Returns a dict of parameters found.
#     """
# 
#     return dict(item.split('=') for item in connectionString.split(';'))
# 
# def getS3Connection(connectionString):
#     connectionParams = parseConnectionString(connectionString)
# 
#     connection = S3Connection(
#         connectionParams['accessKeyId'],
#         connectionParams['secretAccessKey'])
# 
#     return connection


import json
import logging
import os
from boto.s3.connection import S3Connection
from boto.exception import S3ResponseError
from boto.s3.key import Key
from .cachingHelper import getCache
from .cluster import Cluster
from .dbhelper import parseConnectionString, getS3Connection



class ProcessedClusterStore:
  """
  Store for processed clusters.

  Contains functions for CRUD operations on processed clusters
  """

  def __init__(self):
    self.bucketConnString = os.environ['PROCESSEDCLUSTERS_BUCKET']

# Path: newsApp/cachingHelper.py
# def getCache():
#   servers = os.environ.get('MEMCACHIER_SERVERS', '').split(',')
#   user = os.environ.get('MEMCACHIER_USERNAME', '')
#   passw = os.environ.get('MEMCACHIER_PASSWORD', '')
# 
#   return bmemcached.Client(servers, user, passw)
# 
# Path: newsApp/cluster.py
# class Cluster(set):
#   """
#   Represents a cluster(set of similar documents)
#   """
# 
#   def __init__(self, docList):
#     """
#     Instantiates a new cluster object.
#     Requires 'docList': a list of docIds comprising the cluster
#     """
# 
#     set.__init__(self, docList)
# 
#     docList.sort()
#     self.id = hashlib.md5("-".join(docList).encode("latin1")).hexdigest().upper()
#     self.isCurrent = 'unknown'
#     self.isProcessed = False
# 
#   def process(self):
#     """
#     Processes the cluster to include metadata of consisting documents,
#     and overall metadata of cluster like category, location, feeds, etc.
#     """
# 
#     self.categories = []
#     self.countries = []
#     self.locales = []
#     self.publishers = []
#     self.languages = []
#     self.articles = [] # contains non-duplicate articles
#     self.duplicates = []
#     self.lastPubTime = 0
# 
#     docKeys = list(super(Cluster, self).__iter__())
#     docs = _getDocsInParallel(docKeys)
#     docsAdded = []
#     for doc in docs:
#       docKey = doc.key
#       if not _isDuplicateArticle(doc, docsAdded):
#         # note that this gets passed all the way till browser
#         # don't add any internal stuff here
#         self.articles.append({
#           'id': docKey,
#           'title': _getDocTitle(doc),
#           'publisher': doc.tags.get(TAG_PUBLISHER_DETAILS, ""),
#           'link': doc.tags.get(DOCTAG_URL, "#"),
#           'summaryText': _getDocSummary(doc),
#           'highlights': doc.tags.get(LINKTAG_HIGHLIGHTS, []),
#           'images': _getImagesForDoc(doc),
#           'lang': doc.tags.get(FEEDTAG_LANG, ""),
#           'publishedOn': doc.tags.get(LINKTAG_PUBTIME, 0)
#         })
#         docsAdded.append(doc)
#       else:
#         self.duplicates.append(docKey)
# 
#       if doc.tags.get(FEEDTAG_CATEGORY):
#         self.categories.append(doc.tags[FEEDTAG_CATEGORY])
#       if doc.tags.get(FEEDTAG_COUNTRY):
#         self.countries.append(doc.tags[FEEDTAG_COUNTRY])
#       if doc.tags.get(FEEDTAG_LOCALE):
#         self.locales.append(doc.tags[FEEDTAG_LOCALE])
#       if doc.tags.get(TAG_PUBLISHER):
#         self.publishers.append(doc.tags[TAG_PUBLISHER])
#       if doc.tags.get(FEEDTAG_LANG):
#         self.languages.append(doc.tags[FEEDTAG_LANG])
#       if doc.tags.get(LINKTAG_PUBTIME, 0) > self.lastPubTime:
#         self.lastPubTime =  doc.tags.get(LINKTAG_PUBTIME)
# 
#     nArticles = len(self.articles) + len(self.duplicates)
#     self.articles.sort(key=itemgetter('publishedOn'), reverse=True)
# 
#     #remove duplicates
#     self.categories = _removeDuplicatesAndOutliers(self.categories, nArticles)
#     self.countries = _removeDuplicatesAndOutliers(self.countries, nArticles)
#     self.locales = _removeDuplicatesAndOutliers(self.locales, nArticles)
#     self.publishers = list(set(self.publishers))
#     self.languages = list(set(self.languages))
#     self.isProcessed = True
# 
#   def deserializeFromString(self, clusterString):
#     clusterDict = json.loads(clusterString)
#     self.__init__(clusterDict['docKeys'])
#     self.articles = clusterDict['articles']
#     self.categories = clusterDict['categories']
#     self.countries = clusterDict['countries']
#     self.locales = clusterDict['locales']
#     self.publishers = clusterDict['publishers']
#     self.languages = clusterDict['languages']
#     self.duplicates = clusterDict['duplicates']
#     self.isCurrent = clusterDict['isCurrent']
#     self.lastPubTime = float(clusterDict['lastPubTime'])
#     self.isProcessed = True
# 
#   def serializeToString(self):
#     """
#     Serialize cluster to string. Only works for processed clusters
#     """
# 
#     if not self.isProcessed:
#       raise ValueError('Only processed clusters can be serialized')
# 
#     return json.dumps({
#       'clusterId': self.id,
#       'docKeys': list(super(Cluster, self).__iter__()),
#       'articles': self.articles,
#       'categories': self.categories,
#       'countries': self.countries,
#       'locales': self.locales,
#       'publishers': self.publishers,
#       'languages': self.languages,
#       'duplicates': self.duplicates,
#       'isCurrent': self.isCurrent,
#       'lastPubTime': self.lastPubTime
#     })
# 
# Path: newsApp/dbhelper.py
# def parseConnectionString(connectionString):
#     """
#     Parses the table connection string. Returns a dict of parameters found.
#     """
# 
#     return dict(item.split('=') for item in connectionString.split(';'))
# 
# def getS3Connection(connectionString):
#     connectionParams = parseConnectionString(connectionString)
# 
#     connection = S3Connection(
#         connectionParams['accessKeyId'],
#         connectionParams['secretAccessKey'])
# 
#     return connection


# Path: newsApp/processedClusterStore.py
import json
import logging
import os
from boto.s3.connection import S3Connection
from boto.exception import S3ResponseError
from boto.s3.key import Key
from .cachingHelper import getCache
from .cluster import Cluster
from .dbhelper import parseConnectionString, getS3Connection




class ProcessedClusterStore:
  """
  Store for processed clusters.

  Contains functions for CRUD operations on processed clusters
  """

  def __init__(self):
    self.bucketConnString = os.environ['PROCESSEDCLUSTERS_BUCKET']

    self.cache = getCache()