# Path: nordlys/storage/mongo.py
# class Mongo(object):
#     """Manages the MongoDB connection and operations."""
#     ID_FIELD = "_id"
# 
#     def __init__(self, host, db, collection):
#         self.client = MongoClient(host)
#         self.db = self.client[db]
#         self.collection = self.db[collection]
#         self.db_name = db
#         self.collection_name = collection
#         print "Connected to " + self.db_name + "." + self.collection_name
# 
#     @staticmethod
#     def escape(s):
#         """Escapes string (to be used as key or fieldname).
#         Replaces . and $ with their unicode eqivalents."""
#         return s.replace(".", "\u002e").replace("$", "\u0024")
# 
#     @staticmethod
#     def unescape(s):
#         """Unescapes string."""
#         return s.replace("\u002e", ".").replace("\u0024", "$")
# 
#     def find_by_id(self, doc_id):
#         """Returns all document content for a given document id."""
#         return self.get_doc(self.collection.find_one({Mongo.ID_FIELD: self.escape(doc_id)}))
# 
#     def get_doc(self, mdoc):
#         """Returns document contents with with keys and _id field unescaped."""
#         if mdoc is None:
#             return None
# 
#         doc = {}
#         for f in mdoc:
#             if f == Mongo.ID_FIELD:
#                 doc[f] = self.unescape(mdoc[f])
#             else:
#                 doc[self.unescape(f)] = mdoc[f]
# 
#         return doc
# 
#     @staticmethod
#     def print_doc(doc):
#         print "_id: " + doc[Mongo.ID_FIELD]
#         for key, value in doc.iteritems():
#             if key == Mongo.ID_FIELD: continue  # ignore the id key
#             if type(value) is list:
#                 print key + ":"
#                 for v in value:
#                     print "\t" + str(v)
#             else:
#                 print key + ": " + str(value)
# 
# Path: nordlys/wikipedia/utils.py
# class WikipediaUtils(object):
#     mongo = None
#     
#     @staticmethod
#     def wiki_title_to_uri(title):
#         """
#         Converts wiki page title to wiki_uri
#         based on https://en.wikipedia.org/wiki/Wikipedia:Page_name#Spaces.2C_underscores_and_character_coding
#         encoding based on http://dbpedia.org/services-resources/uri-encoding
#         """
#         if title:
#             wiki_uri = "<wikipedia:" + quote(title, ' !$&\'()*+,-./:;=@_~').replace(' ', '_') + ">"
#             return wiki_uri
#         else:
#             return None
# 
#     @staticmethod
#     def wiki_uri_to_dbp_uri(wiki_uri):
#         """Converts Wikipedia uri to DBpedia URI."""
#         return wiki_uri.replace("<wikipedia:", "<dbpedia:")


import argparse
import os
import json
from urllib import unquote
from nordlys.storage.mongo import Mongo
from nordlys.wikipedia.utils import WikipediaUtils
"""
Reads surface forms extracted from titles, redirects, and anchors and merge them into a single json file,
which can be directly imported to Mongodb using this command:

 mongoimport --db <db_name> --collection surfaceforms_wiki_YYYYMMDD --file <path_to_json_file> --jsonArray

@author: Faegheh Hasibi (faegheh.hasibi@idi.ntnu.no)
"""



class Merger(object):

    def __init__(self):
        self.all_sfs = {}

    def merge_all(self, titles_file, redirects_file, anchors_file, out_file):
        self.add_anchors(anchors_file)
        self.add_titles(titles_file)
        self.add_redirects(redirects_file)

        # Converting all surface forms to mongo format
        print "Converting to mongodb format ..."
        sf_mongo_entries = []
        i = 0
        for sf, en_sources in self.all_sfs.iteritems():

# Path: nordlys/storage/mongo.py
# class Mongo(object):
#     """Manages the MongoDB connection and operations."""
#     ID_FIELD = "_id"
# 
#     def __init__(self, host, db, collection):
#         self.client = MongoClient(host)
#         self.db = self.client[db]
#         self.collection = self.db[collection]
#         self.db_name = db
#         self.collection_name = collection
#         print "Connected to " + self.db_name + "." + self.collection_name
# 
#     @staticmethod
#     def escape(s):
#         """Escapes string (to be used as key or fieldname).
#         Replaces . and $ with their unicode eqivalents."""
#         return s.replace(".", "\u002e").replace("$", "\u0024")
# 
#     @staticmethod
#     def unescape(s):
#         """Unescapes string."""
#         return s.replace("\u002e", ".").replace("\u0024", "$")
# 
#     def find_by_id(self, doc_id):
#         """Returns all document content for a given document id."""
#         return self.get_doc(self.collection.find_one({Mongo.ID_FIELD: self.escape(doc_id)}))
# 
#     def get_doc(self, mdoc):
#         """Returns document contents with with keys and _id field unescaped."""
#         if mdoc is None:
#             return None
# 
#         doc = {}
#         for f in mdoc:
#             if f == Mongo.ID_FIELD:
#                 doc[f] = self.unescape(mdoc[f])
#             else:
#                 doc[self.unescape(f)] = mdoc[f]
# 
#         return doc
# 
#     @staticmethod
#     def print_doc(doc):
#         print "_id: " + doc[Mongo.ID_FIELD]
#         for key, value in doc.iteritems():
#             if key == Mongo.ID_FIELD: continue  # ignore the id key
#             if type(value) is list:
#                 print key + ":"
#                 for v in value:
#                     print "\t" + str(v)
#             else:
#                 print key + ": " + str(value)
# 
# Path: nordlys/wikipedia/utils.py
# class WikipediaUtils(object):
#     mongo = None
#     
#     @staticmethod
#     def wiki_title_to_uri(title):
#         """
#         Converts wiki page title to wiki_uri
#         based on https://en.wikipedia.org/wiki/Wikipedia:Page_name#Spaces.2C_underscores_and_character_coding
#         encoding based on http://dbpedia.org/services-resources/uri-encoding
#         """
#         if title:
#             wiki_uri = "<wikipedia:" + quote(title, ' !$&\'()*+,-./:;=@_~').replace(' ', '_') + ">"
#             return wiki_uri
#         else:
#             return None
# 
#     @staticmethod
#     def wiki_uri_to_dbp_uri(wiki_uri):
#         """Converts Wikipedia uri to DBpedia URI."""
#         return wiki_uri.replace("<wikipedia:", "<dbpedia:")


# Path: nordlys/wikipedia/merge_sf.py
import argparse
import os
import json
from urllib import unquote
from nordlys.storage.mongo import Mongo
from nordlys.wikipedia.utils import WikipediaUtils

"""
Reads surface forms extracted from titles, redirects, and anchors and merge them into a single json file,
which can be directly imported to Mongodb using this command:

 mongoimport --db <db_name> --collection surfaceforms_wiki_YYYYMMDD --file <path_to_json_file> --jsonArray

@author: Faegheh Hasibi (faegheh.hasibi@idi.ntnu.no)
"""



class Merger(object):

    def __init__(self):
        self.all_sfs = {}

    def merge_all(self, titles_file, redirects_file, anchors_file, out_file):
        self.add_anchors(anchors_file)
        self.add_titles(titles_file)
        self.add_redirects(redirects_file)

        # Converting all surface forms to mongo format
        print "Converting to mongodb format ..."
        sf_mongo_entries = []
        i = 0
        for sf, en_sources in self.all_sfs.iteritems():

            escaped_sf = Mongo.escape(sf)