# Path: nordlys/storage/mongo.py
# class Mongo(object):
#     """Manages the MongoDB connection and operations."""
#     ID_FIELD = "_id"
# 
#     def __init__(self, host, db, collection):
#         self.client = MongoClient(host)
#         self.db = self.client[db]
#         self.collection = self.db[collection]
#         self.db_name = db
#         self.collection_name = collection
#         print "Connected to " + self.db_name + "." + self.collection_name
# 
#     @staticmethod
#     def escape(s):
#         """Escapes string (to be used as key or fieldname).
#         Replaces . and $ with their unicode eqivalents."""
#         return s.replace(".", "\u002e").replace("$", "\u0024")
# 
#     @staticmethod
#     def unescape(s):
#         """Unescapes string."""
#         return s.replace("\u002e", ".").replace("\u0024", "$")
# 
#     def find_by_id(self, doc_id):
#         """Returns all document content for a given document id."""
#         return self.get_doc(self.collection.find_one({Mongo.ID_FIELD: self.escape(doc_id)}))
# 
#     def get_doc(self, mdoc):
#         """Returns document contents with with keys and _id field unescaped."""
#         if mdoc is None:
#             return None
# 
#         doc = {}
#         for f in mdoc:
#             if f == Mongo.ID_FIELD:
#                 doc[f] = self.unescape(mdoc[f])
#             else:
#                 doc[self.unescape(f)] = mdoc[f]
# 
#         return doc
# 
#     @staticmethod
#     def print_doc(doc):
#         print "_id: " + doc[Mongo.ID_FIELD]
#         for key, value in doc.iteritems():
#             if key == Mongo.ID_FIELD: continue  # ignore the id key
#             if type(value) is list:
#                 print key + ":"
#                 for v in value:
#                     print "\t" + str(v)
#             else:
#                 print key + ": " + str(value)
# 
# Path: nordlys/wikipedia/utils.py
# class WikipediaUtils(object):
#     mongo = None
#     
#     @staticmethod
#     def wiki_title_to_uri(title):
#         """
#         Converts wiki page title to wiki_uri
#         based on https://en.wikipedia.org/wiki/Wikipedia:Page_name#Spaces.2C_underscores_and_character_coding
#         encoding based on http://dbpedia.org/services-resources/uri-encoding
#         """
#         if title:
#             wiki_uri = "<wikipedia:" + quote(title, ' !$&\'()*+,-./:;=@_~').replace(' ', '_') + ">"
#             return wiki_uri
#         else:
#             return None
# 
#     @staticmethod
#     def wiki_uri_to_dbp_uri(wiki_uri):
#         """Converts Wikipedia uri to DBpedia URI."""
#         return wiki_uri.replace("<wikipedia:", "<dbpedia:")


import argparse
import os
import json
from urllib import unquote
from nordlys.storage.mongo import Mongo
from nordlys.wikipedia.utils import WikipediaUtils
        else:
            self.all_sfs[sf][pred][en] = count

    # ============== ANCHORS ==============

    def add_anchors(self, anchor_file):
        print "Adding anchors ..."
        i = 0
        infile = open(anchor_file, "r")
        for line in infile:
            # print line
            cols = line.strip().split("\t")
            sf = cols[0].strip()
            count = int(cols[2])
            wiki_uri = WikipediaUtils.wiki_title_to_uri(unquote(cols[1].strip()))
            self.__add_to_dict(sf, "anchor", wiki_uri, count)
            i += 1
            if i % 1000000 == 0:
                print "Processed", i, "th anchor!"

    # ============== REDIRECTS ==============

    def add_redirects(self, redirect_file):
        """Adds redirect pages to the surface form dictionary."""
        print "Adding redirects ..."
        redirects = open(redirect_file, "r")
        count = 0
        for line in redirects:
            cols = line.strip().split("\t")
            sf = cols[0].strip().lower()

# Path: nordlys/storage/mongo.py
# class Mongo(object):
#     """Manages the MongoDB connection and operations."""
#     ID_FIELD = "_id"
# 
#     def __init__(self, host, db, collection):
#         self.client = MongoClient(host)
#         self.db = self.client[db]
#         self.collection = self.db[collection]
#         self.db_name = db
#         self.collection_name = collection
#         print "Connected to " + self.db_name + "." + self.collection_name
# 
#     @staticmethod
#     def escape(s):
#         """Escapes string (to be used as key or fieldname).
#         Replaces . and $ with their unicode eqivalents."""
#         return s.replace(".", "\u002e").replace("$", "\u0024")
# 
#     @staticmethod
#     def unescape(s):
#         """Unescapes string."""
#         return s.replace("\u002e", ".").replace("\u0024", "$")
# 
#     def find_by_id(self, doc_id):
#         """Returns all document content for a given document id."""
#         return self.get_doc(self.collection.find_one({Mongo.ID_FIELD: self.escape(doc_id)}))
# 
#     def get_doc(self, mdoc):
#         """Returns document contents with with keys and _id field unescaped."""
#         if mdoc is None:
#             return None
# 
#         doc = {}
#         for f in mdoc:
#             if f == Mongo.ID_FIELD:
#                 doc[f] = self.unescape(mdoc[f])
#             else:
#                 doc[self.unescape(f)] = mdoc[f]
# 
#         return doc
# 
#     @staticmethod
#     def print_doc(doc):
#         print "_id: " + doc[Mongo.ID_FIELD]
#         for key, value in doc.iteritems():
#             if key == Mongo.ID_FIELD: continue  # ignore the id key
#             if type(value) is list:
#                 print key + ":"
#                 for v in value:
#                     print "\t" + str(v)
#             else:
#                 print key + ": " + str(value)
# 
# Path: nordlys/wikipedia/utils.py
# class WikipediaUtils(object):
#     mongo = None
#     
#     @staticmethod
#     def wiki_title_to_uri(title):
#         """
#         Converts wiki page title to wiki_uri
#         based on https://en.wikipedia.org/wiki/Wikipedia:Page_name#Spaces.2C_underscores_and_character_coding
#         encoding based on http://dbpedia.org/services-resources/uri-encoding
#         """
#         if title:
#             wiki_uri = "<wikipedia:" + quote(title, ' !$&\'()*+,-./:;=@_~').replace(' ', '_') + ">"
#             return wiki_uri
#         else:
#             return None
# 
#     @staticmethod
#     def wiki_uri_to_dbp_uri(wiki_uri):
#         """Converts Wikipedia uri to DBpedia URI."""
#         return wiki_uri.replace("<wikipedia:", "<dbpedia:")


# Path: nordlys/wikipedia/merge_sf.py
import argparse
import os
import json
from urllib import unquote
from nordlys.storage.mongo import Mongo
from nordlys.wikipedia.utils import WikipediaUtils

        else:
            self.all_sfs[sf][pred][en] = count

    # ============== ANCHORS ==============

    def add_anchors(self, anchor_file):
        print "Adding anchors ..."
        i = 0
        infile = open(anchor_file, "r")
        for line in infile:
            # print line
            cols = line.strip().split("\t")
            sf = cols[0].strip()
            count = int(cols[2])
            wiki_uri = WikipediaUtils.wiki_title_to_uri(unquote(cols[1].strip()))
            self.__add_to_dict(sf, "anchor", wiki_uri, count)
            i += 1
            if i % 1000000 == 0:
                print "Processed", i, "th anchor!"

    # ============== REDIRECTS ==============

    def add_redirects(self, redirect_file):
        """Adds redirect pages to the surface form dictionary."""
        print "Adding redirects ..."
        redirects = open(redirect_file, "r")
        count = 0
        for line in redirects:
            cols = line.strip().split("\t")
            sf = cols[0].strip().lower()

            wiki_uri = WikipediaUtils.wiki_title_to_uri(cols[1].strip())