# Path: nytimesfetch.py
# def nytimesFetchInfo(url):
#     #print >> sys.stderr, 'url: ' + url
#     #http://api.nytimes.com/svc/search/v1/article?format=json&query=url%3Ahttp%3A%2F%2Fwww.nytimes.com%2F2009%2F03%2F26%2Fgarden%2F26slow.html&fields=title%2C+nytd_title%2C+date%2C+byline%2C+page_facet%2C+section_page_facet%2C+source_facet&api-key=#### 
#     urlfixed = re.sub('\?.*$', '', url)
#     #print >> sys.stderr, urlfixed
#     querystring = urllib.urlencode({'format': 'json',
#                                     'query': 'url:'+urlfixed,
#                                     'fields': 'title, date, byline, page_facet, section_page_facet, source_facet, column_facet',
#                                     'api-key': nyt_apikey})
#     nyturl = "http://api.nytimes.com/svc/search/v1/article?" + querystring
#     jsonstring = cachedFetch(nyturl, 3600)
#     if jsonstring == '':
#         error('Bad data from NYTimes')
#     
#     nytdata = json.loads(jsonstring)
#     #print >> sys.stderr, nytdata
#     if len(nytdata['results']) == 0:
#         sys.stderr.write("No data from " + nyturl + "\n")
#         return {}
#     
#     result = nytdata['results'][0]
#     citedata = {'work': 'The New York Times'}
#     citedata['title'] = result['title']
#     if result.has_key('page_facet'):
#         citedata['page'] = result['page_facet']
#     if result.has_key('date'):
#         m = re.search('(\d\d\d\d)(\d\d)(\d\d)', result['date'])
#         if m:
#             citedata['date'] = m.group(1) + '-' + m.group(2) + '-' + m.group(3)
#     if result.has_key('byline'):
#         byline = result['byline']
#         byline = re.sub('(?i)^By\s+', '', byline)
#         byline = re.sub('(?i) And ', ', ', byline)
#         byline = byline.title()
#         authors = re.split('\s*,\s*', byline)
#         citedata['authors'] = authors
# 
#     
#     return citedata
# 
# Path: urltitlefetch.py
# def urlTitleFetch(url):
#     key = 'urltitle/' + url
#     data = memcache.get(key)
#     if data is not None:
#         #print >> sys.stderr, "Title from memcache: " + url
#         return data
#     else:
#         #print >> sys.stderr, "Title from fetch: " + url
#         htmldata = ''
#         try:
#             u = urllib2.urlopen(url)
#             htmldata = u.read()
#             u.close()
#         except urllib2.URLError, e:
#             #print >> sys.stderr, 'URLError'
#             #raise
#             return ''
#         except:
#             return ''
#         
#         if htmldata == '' or not re.search('<html', htmldata, re.I):
#             #error('Bad html data')
#             return ''
#             
#         #Strip everything from <body and on, we only need the head
#         htmldata = re.split('(?i)<body', htmldata, maxsplit=1)[0]
#         
#         parseOnly = SoupStrainer('title')
#         soup = BeautifulSoup(htmldata, parseOnlyThese=parseOnly, convertEntities=BeautifulSoup.HTML_ENTITIES)
#         #print >> sys.stderr, soup
#         if soup.title and soup.title.string:
#             title = soup.title.string.extract()    # extract() makes it an ordinary string
#         else:
#             return ''
# #        title = ''
# #        m = re.search('<title>(.*?)<\/title>', htmldata, re.I | re.DOTALL)
# #        if m:
# #            title = m.group(1)
# #            print >> sys.stderr, 'title:' + title
# #            #title = re.sub('^[\s\n]+|[\s\n]+$', '', title)
# #            title = title.strip()
# #            title = unescape(title)
# #            print >> sys.stderr, 'aftertitle:' + title
# #        else:
# #            print >> sys.stderr, 'none'
# #            pass
#         
#         memcache.add(key, title, 3600)
#         return title
# 
# Path: RedirectTarget.py
# def getRedirectTarget(title):
#     querystring = urllib.urlencode({'action': 'query',
#                                     'format': 'json',
#                                     'redirects': '1',
#                                     'titles':title})
#     url = "http://en.wikipedia.org/w/api.php?" + querystring
#     #url = "http://en.wikipedia.org/w/api.php?action=query&prop=info&titles=appspot.com&redirects&format=json"
#     #print >> sys.stderr, "url: " + url 
#     jsonstring = cachedFetch(url, 3600)
#     #print >> sys.stderr, jsonstring
#     apidata = json.loads(jsonstring)
#     try:
#         return apidata['query']['redirects'][0]['to']
#     except KeyError, e:
#         return ''
# 
# Path: DomainParser.py
# def domainFromURL(url):
#     return getDomain(url,tlds)


from nytimesfetch import nytimesFetchInfo
from django.utils import simplejson as json
from urltitlefetch import urlTitleFetch
from RedirectTarget import getRedirectTarget
from DomainParser import domainFromURL
import cgi
import re
import sys

def error(message):
    print 'Content-Type: text/plain'
    print ''
    print 'Error: ', message

 
def main():
    form = cgi.FieldStorage()
    if (not form.has_key("url")):
        error('No URL.'); return
    url = form["url"].value

    if (not form.has_key("callback")):
        error('No callback.'); return
    callback = form["callback"].value
 
    if re.search('^http:\/\/[^\/]*nytimes\.com\/', url, re.I):

# Path: nytimesfetch.py
# def nytimesFetchInfo(url):
#     #print >> sys.stderr, 'url: ' + url
#     #http://api.nytimes.com/svc/search/v1/article?format=json&query=url%3Ahttp%3A%2F%2Fwww.nytimes.com%2F2009%2F03%2F26%2Fgarden%2F26slow.html&fields=title%2C+nytd_title%2C+date%2C+byline%2C+page_facet%2C+section_page_facet%2C+source_facet&api-key=#### 
#     urlfixed = re.sub('\?.*$', '', url)
#     #print >> sys.stderr, urlfixed
#     querystring = urllib.urlencode({'format': 'json',
#                                     'query': 'url:'+urlfixed,
#                                     'fields': 'title, date, byline, page_facet, section_page_facet, source_facet, column_facet',
#                                     'api-key': nyt_apikey})
#     nyturl = "http://api.nytimes.com/svc/search/v1/article?" + querystring
#     jsonstring = cachedFetch(nyturl, 3600)
#     if jsonstring == '':
#         error('Bad data from NYTimes')
#     
#     nytdata = json.loads(jsonstring)
#     #print >> sys.stderr, nytdata
#     if len(nytdata['results']) == 0:
#         sys.stderr.write("No data from " + nyturl + "\n")
#         return {}
#     
#     result = nytdata['results'][0]
#     citedata = {'work': 'The New York Times'}
#     citedata['title'] = result['title']
#     if result.has_key('page_facet'):
#         citedata['page'] = result['page_facet']
#     if result.has_key('date'):
#         m = re.search('(\d\d\d\d)(\d\d)(\d\d)', result['date'])
#         if m:
#             citedata['date'] = m.group(1) + '-' + m.group(2) + '-' + m.group(3)
#     if result.has_key('byline'):
#         byline = result['byline']
#         byline = re.sub('(?i)^By\s+', '', byline)
#         byline = re.sub('(?i) And ', ', ', byline)
#         byline = byline.title()
#         authors = re.split('\s*,\s*', byline)
#         citedata['authors'] = authors
# 
#     
#     return citedata
# 
# Path: urltitlefetch.py
# def urlTitleFetch(url):
#     key = 'urltitle/' + url
#     data = memcache.get(key)
#     if data is not None:
#         #print >> sys.stderr, "Title from memcache: " + url
#         return data
#     else:
#         #print >> sys.stderr, "Title from fetch: " + url
#         htmldata = ''
#         try:
#             u = urllib2.urlopen(url)
#             htmldata = u.read()
#             u.close()
#         except urllib2.URLError, e:
#             #print >> sys.stderr, 'URLError'
#             #raise
#             return ''
#         except:
#             return ''
#         
#         if htmldata == '' or not re.search('<html', htmldata, re.I):
#             #error('Bad html data')
#             return ''
#             
#         #Strip everything from <body and on, we only need the head
#         htmldata = re.split('(?i)<body', htmldata, maxsplit=1)[0]
#         
#         parseOnly = SoupStrainer('title')
#         soup = BeautifulSoup(htmldata, parseOnlyThese=parseOnly, convertEntities=BeautifulSoup.HTML_ENTITIES)
#         #print >> sys.stderr, soup
#         if soup.title and soup.title.string:
#             title = soup.title.string.extract()    # extract() makes it an ordinary string
#         else:
#             return ''
# #        title = ''
# #        m = re.search('<title>(.*?)<\/title>', htmldata, re.I | re.DOTALL)
# #        if m:
# #            title = m.group(1)
# #            print >> sys.stderr, 'title:' + title
# #            #title = re.sub('^[\s\n]+|[\s\n]+$', '', title)
# #            title = title.strip()
# #            title = unescape(title)
# #            print >> sys.stderr, 'aftertitle:' + title
# #        else:
# #            print >> sys.stderr, 'none'
# #            pass
#         
#         memcache.add(key, title, 3600)
#         return title
# 
# Path: RedirectTarget.py
# def getRedirectTarget(title):
#     querystring = urllib.urlencode({'action': 'query',
#                                     'format': 'json',
#                                     'redirects': '1',
#                                     'titles':title})
#     url = "http://en.wikipedia.org/w/api.php?" + querystring
#     #url = "http://en.wikipedia.org/w/api.php?action=query&prop=info&titles=appspot.com&redirects&format=json"
#     #print >> sys.stderr, "url: " + url 
#     jsonstring = cachedFetch(url, 3600)
#     #print >> sys.stderr, jsonstring
#     apidata = json.loads(jsonstring)
#     try:
#         return apidata['query']['redirects'][0]['to']
#     except KeyError, e:
#         return ''
# 
# Path: DomainParser.py
# def domainFromURL(url):
#     return getDomain(url,tlds)


# Path: urlfetchjs.py
from nytimesfetch import nytimesFetchInfo
from django.utils import simplejson as json
from urltitlefetch import urlTitleFetch
from RedirectTarget import getRedirectTarget
from DomainParser import domainFromURL
import cgi
import re
import sys


def error(message):
    print 'Content-Type: text/plain'
    print ''
    print 'Error: ', message

 
def main():
    form = cgi.FieldStorage()
    if (not form.has_key("url")):
        error('No URL.'); return
    url = form["url"].value

    if (not form.has_key("callback")):
        error('No callback.'); return
    callback = form["callback"].value
 
    if re.search('^http:\/\/[^\/]*nytimes\.com\/', url, re.I):

        citedata = nytimesFetchInfo(url)