# Path: newsApp/docHelper.py
# def getDocComparisionScore(jobInfo, doc1, doc2):
#     score = 0
#     if (doc1.tags[FEEDTAG_LANG] == LANG_ENGLISH) and \
#         (doc2.tags[FEEDTAG_LANG] == LANG_ENGLISH):
#         score = computeEnglishDocsSimScore(jobInfo, doc1, doc2)
#         logger.info("Compared using shingles. %s", jobInfo)
#     else:
#         score = computeDocSimScoreUsingEntities(jobInfo, doc1, doc2)
#         # make it slightly easier for non-english docs to get clustered.
#         score = score * 1.15
#         logger.info("Compared using entities. %s", jobInfo)
# 
#     if FEEDTAG_LOCALE in doc1.tags and FEEDTAG_LOCALE in doc2.tags and \
#         doc1.tags[FEEDTAG_LOCALE] != doc2.tags[FEEDTAG_LOCALE]:
# 
#         logger.info(
#             "The two docs are from different locations. Adding penalty. %s",
#             jobInfo)
#         score = score - 0.4
#         if score < 0:
#             score = 0
# 
#     if doc1.tags[TAG_PUBLISHER] != doc2.tags[TAG_PUBLISHER]:
#         # make it slightly easier for different-publisher-docs to get clustered
#         score = score * 1.1
# 
#     doc1Publisher = doc1.tags[TAG_PUBLISHER]
#     doc2Publisher = doc2.tags[TAG_PUBLISHER]
#     if doc1Publisher == doc2Publisher:
#         penaltyPublishers = ['TOI', 'TelanganaToday', 'Hindu']
#         if doc1Publisher in penaltyPublishers :
#             logger.info(
#                 "Adding penalty for same publisher %s. %s",
#                 doc1Publisher,
#                 jobInfo)
#             score = score*0.8
# 
#     if score > 1:
#         score = 1
# 
#     logger.info("Comparision score: %s. %s", str(score), jobInfo)
#     return score
# 
# def getDocEnglishContentSimilarity(doc1, doc2):
#     return th.compareUsingShingles(
#         getDocEnglishContent(doc1),
#         getDocEnglishContent(doc2))
# 
# Path: newsApp/docManager.py
# class DocManager:
#     """
#     Manage documents stored in cloud.
# 
#     Contains functions for CRUD operations on documents
#     """
# 
#     def __init__(self):
#         """
#         Instantiates a new instance of DocManager class
# 
#         'bucketConnString' : connection string of s3 bucket in which docs
#         are stored.
#         """
# 
#         self.bucketConnString = os.environ['DOCSBUCKET_CONNECTIONSTRING']
#         self.cache = getCache()
#         self.__cacheExpiry= 900
# 
#     def __getBucket(self):
#         bucketConnParams = parseConnectionString(self.bucketConnString)
#         conn = getS3Connection(self.bucketConnString)
# 
#         return conn.get_bucket(bucketConnParams['bucketName'], validate=False)
# 
#     def __isDocNew(self, key, timeLimit):
#         if _getEpochSecs(key.last_modified) < timeLimit:
#             return False
# 
#         doc = self.get(key.name)
#         return (doc.tags[LINKTAG_PUBTIME] > timeLimit) and \
#             (FEEDTAG_DO_NOT_CLUSTER not in doc.tags)
# 
#     def put(self, doc):
#         k = Key(self.__getBucket())
#         k.key = doc.key
# 
#         # not storing tags directly in blob's metadata as the maximum size
#         # allowed there is only 2kb.
#         tags = dict(doc.tags)
#         tags['content'] = doc.content
#         keyContents = json.dumps(tags)
#         k.set_contents_from_string(keyContents)
#         self.cache.set(k.key, keyContents, self.__cacheExpiry)
# 
#     def get(self, docKey):
#         keyContents = self.cache.get(docKey)
#         if not keyContents:
#             k = Key(self.__getBucket())
#             k.key = docKey
#             keyContents = k.get_contents_as_string()
#             self.cache.set(docKey, keyContents, self.__cacheExpiry)
# 
#         storedTags = json.loads(keyContents)
#         content = storedTags.pop('content', None)
#         tags = storedTags
# 
#         return Doc(docKey, content, tags)
# 
#     def delete(self, docKey):
#         k = Key(self.__getBucket())
#         k.key = docKey
#         k.delete()
#         self.cache.delete(docKey)


import hashlib
import json
import queue
from operator import itemgetter
from threading import Thread
from .constants import *
from .docHelper import getDocComparisionScore,getDocEnglishContentSimilarity
from .docManager import DocManager
from . import textHelper as th
DOC_DUPLICATION_CONTENT_THRESHOLD = 0.8
def _removeDuplicatesAndOutliers(items, articleCount):
  d = {}
  for item in items:
    if item in d:
      d[item] = d[item] + 1
    else:
      d[item] = 1

  return [item for item in list(d.keys()) if d[item] > 0.4 * articleCount]

def _isDuplicateArticle(doc, docsAdded):
  for addedDoc in docsAdded:
    if doc.tags[FEEDTAG_LANG] == addedDoc.tags[FEEDTAG_LANG]:
      if doc.tags[TAG_PUBLISHER] != addedDoc.tags[TAG_PUBLISHER]:
        return False

      distance = getDocComparisionScore("processCluster", doc, addedDoc)
      if distance >= DOC_DUPLICATION_SCORE_THRESHOLD:
        return True

      englishContentSim = getDocEnglishContentSimilarity(doc, addedDoc)
      if englishContentSim >= DOC_DUPLICATION_CONTENT_THRESHOLD:
        return True

  return False

def _getDocsInParallel(docKeys):
  que = queue.Queue()
  threads_list = list()

# Path: newsApp/docHelper.py
# def getDocComparisionScore(jobInfo, doc1, doc2):
#     score = 0
#     if (doc1.tags[FEEDTAG_LANG] == LANG_ENGLISH) and \
#         (doc2.tags[FEEDTAG_LANG] == LANG_ENGLISH):
#         score = computeEnglishDocsSimScore(jobInfo, doc1, doc2)
#         logger.info("Compared using shingles. %s", jobInfo)
#     else:
#         score = computeDocSimScoreUsingEntities(jobInfo, doc1, doc2)
#         # make it slightly easier for non-english docs to get clustered.
#         score = score * 1.15
#         logger.info("Compared using entities. %s", jobInfo)
# 
#     if FEEDTAG_LOCALE in doc1.tags and FEEDTAG_LOCALE in doc2.tags and \
#         doc1.tags[FEEDTAG_LOCALE] != doc2.tags[FEEDTAG_LOCALE]:
# 
#         logger.info(
#             "The two docs are from different locations. Adding penalty. %s",
#             jobInfo)
#         score = score - 0.4
#         if score < 0:
#             score = 0
# 
#     if doc1.tags[TAG_PUBLISHER] != doc2.tags[TAG_PUBLISHER]:
#         # make it slightly easier for different-publisher-docs to get clustered
#         score = score * 1.1
# 
#     doc1Publisher = doc1.tags[TAG_PUBLISHER]
#     doc2Publisher = doc2.tags[TAG_PUBLISHER]
#     if doc1Publisher == doc2Publisher:
#         penaltyPublishers = ['TOI', 'TelanganaToday', 'Hindu']
#         if doc1Publisher in penaltyPublishers :
#             logger.info(
#                 "Adding penalty for same publisher %s. %s",
#                 doc1Publisher,
#                 jobInfo)
#             score = score*0.8
# 
#     if score > 1:
#         score = 1
# 
#     logger.info("Comparision score: %s. %s", str(score), jobInfo)
#     return score
# 
# def getDocEnglishContentSimilarity(doc1, doc2):
#     return th.compareUsingShingles(
#         getDocEnglishContent(doc1),
#         getDocEnglishContent(doc2))
# 
# Path: newsApp/docManager.py
# class DocManager:
#     """
#     Manage documents stored in cloud.
# 
#     Contains functions for CRUD operations on documents
#     """
# 
#     def __init__(self):
#         """
#         Instantiates a new instance of DocManager class
# 
#         'bucketConnString' : connection string of s3 bucket in which docs
#         are stored.
#         """
# 
#         self.bucketConnString = os.environ['DOCSBUCKET_CONNECTIONSTRING']
#         self.cache = getCache()
#         self.__cacheExpiry= 900
# 
#     def __getBucket(self):
#         bucketConnParams = parseConnectionString(self.bucketConnString)
#         conn = getS3Connection(self.bucketConnString)
# 
#         return conn.get_bucket(bucketConnParams['bucketName'], validate=False)
# 
#     def __isDocNew(self, key, timeLimit):
#         if _getEpochSecs(key.last_modified) < timeLimit:
#             return False
# 
#         doc = self.get(key.name)
#         return (doc.tags[LINKTAG_PUBTIME] > timeLimit) and \
#             (FEEDTAG_DO_NOT_CLUSTER not in doc.tags)
# 
#     def put(self, doc):
#         k = Key(self.__getBucket())
#         k.key = doc.key
# 
#         # not storing tags directly in blob's metadata as the maximum size
#         # allowed there is only 2kb.
#         tags = dict(doc.tags)
#         tags['content'] = doc.content
#         keyContents = json.dumps(tags)
#         k.set_contents_from_string(keyContents)
#         self.cache.set(k.key, keyContents, self.__cacheExpiry)
# 
#     def get(self, docKey):
#         keyContents = self.cache.get(docKey)
#         if not keyContents:
#             k = Key(self.__getBucket())
#             k.key = docKey
#             keyContents = k.get_contents_as_string()
#             self.cache.set(docKey, keyContents, self.__cacheExpiry)
# 
#         storedTags = json.loads(keyContents)
#         content = storedTags.pop('content', None)
#         tags = storedTags
# 
#         return Doc(docKey, content, tags)
# 
#     def delete(self, docKey):
#         k = Key(self.__getBucket())
#         k.key = docKey
#         k.delete()
#         self.cache.delete(docKey)


# Path: newsApp/cluster.py
import hashlib
import json
import queue
from operator import itemgetter
from threading import Thread
from .constants import *
from .docHelper import getDocComparisionScore,getDocEnglishContentSimilarity
from .docManager import DocManager
from . import textHelper as th

DOC_DUPLICATION_CONTENT_THRESHOLD = 0.8
def _removeDuplicatesAndOutliers(items, articleCount):
  d = {}
  for item in items:
    if item in d:
      d[item] = d[item] + 1
    else:
      d[item] = 1

  return [item for item in list(d.keys()) if d[item] > 0.4 * articleCount]

def _isDuplicateArticle(doc, docsAdded):
  for addedDoc in docsAdded:
    if doc.tags[FEEDTAG_LANG] == addedDoc.tags[FEEDTAG_LANG]:
      if doc.tags[TAG_PUBLISHER] != addedDoc.tags[TAG_PUBLISHER]:
        return False

      distance = getDocComparisionScore("processCluster", doc, addedDoc)
      if distance >= DOC_DUPLICATION_SCORE_THRESHOLD:
        return True

      englishContentSim = getDocEnglishContentSimilarity(doc, addedDoc)
      if englishContentSim >= DOC_DUPLICATION_CONTENT_THRESHOLD:
        return True

  return False

def _getDocsInParallel(docKeys):
  que = queue.Queue()
  threads_list = list()

  docManager = DocManager()