# Path: blango/spider.py
# class Spider(BaseSpider):
#     def __init__(self, uri, data = ''):
#         super(Spider, self).__init__(uri)
#         self.data = data
# 
#     def pingback(self, source):
#         if not self._require_fetch():
#             return False
#         pburi = self.stream.headers.get('X-Pingback')
#         if not pburi:
#             r = re.compile('<link.*?rel="pingback".*?href="([^"]+)" ?/?>', re.I)
#             m = r.search(self.data)
#             if not m:
#                 return False
#             pburi = m.group(1)
#         server = ServerProxy(pburi)
#         try:
#             server.pingback.ping(source, self.uri)
#             return True
#         except Error, v:
#             return False
# 
#     def trackback(self, url, page_name='', title='', excerpt=''):
#         if not self._require_fetch():
#             print self.code
#             return False
# 
#         r = re.compile('trackback:ping="([^"]+)"')
#         m = r.search(self.data)
#         if not m:
#             return False
#         tburi = m.group(1)
#         t = Spider(tburi)
#         t.request.add_header('Content-Type', 'application/x-www-form-urlencoded; charset=utf-8')
#         values = { 'title': title,
#             'excerpt': excerpt,
#             'url': url,
#             'blog_name': page_name }
#         data = urllib.urlencode(values)
#         t.request = urllib2.Request(tburi, data)
#         t.fetch()
#         return t.data.find('<error>0</error>') != -1
# 
#     def backlinks(self, back):
#         if not self._require_fetch():
#             return False
# 
#         r = re.compile('<a.*?href="%s".*>' % back)
#         return r.search(self.data) != None
# 
# Path: stem.py
# class Stemmer(object):
#     def __init__(self, lang, encoding=None):
#         self.stemmer = libstemmer.sb_stemmer_new(lang, encoding)
#         if not self.stemmer:
#             raise RuntimeError('Lang "%s" is not supported by this stemmer' % lang)
# 
#         self.mutex = Lock()
# 
#     def __del__(self):
#         libstemmer.sb_stemmer_delete(self.stemmer)
#         # No need to call the parent class, since object.__del__ does not exist
# 
#     def stem(self, word):
#         if isinstance(word, unicode):
#             w = word.encode('utf8')
#         else:
#             w = word
# 
#         # sb_stemmer_stem stemmer is not thread safe
#         self.mutex.acquire()
#         stem = libstemmer.sb_stemmer_stem(self.stemmer, w.lower(), len(w))
#         self.mutex.release()
#         return stem
# 
# HAS_LIBSTEMMER = True


import re
import os
import urlparse
from datetime import datetime, timedelta
from cStringIO import StringIO
from django.db import models
from django.contrib.auth.models import User
from django.template.defaultfilters import slugify, capfirst, force_escape
from django.utils.translation import ugettext, ugettext_lazy as _
from django.utils.safestring import mark_safe
from django.core.files.base import ContentFile
from django.conf import settings
from PIL import Image
from blango.spider import Spider
from stem import Stemmer, HAS_LIBSTEMMER
from markdown import markdown
                    common_stems.append(stem.pk)
                    max_dist += stem.value ** 2 + entry_stem.value ** 2
                except EntryStems.DoesNotExist:
                    increment = entry_stem.value ** 2
                    abs_dist += increment
                    max_dist += increment

            for entry_stem in self.stems.exclude(pk__in=common_stems):
                increment = entry_stem.value ** 2
                abs_dist += increment
                max_dist += increment

            if abs_dist/max_dist < settings.RELATED_THRESHOLD:
                self.related.add(e)

    def get_admin_absolute_url(self):
        try:
            return self.linkentry.get_admin_absolute_url()
        except:
            pass
        return '/admin/%s/%s/%s/' % (self._meta.app_label, self._meta.module_name, self.pk)

class LinkEntry(Entry):
    class Meta:
        verbose_name = _('link entry')
        verbose_name_plural = _('link entries')

    link = models.URLField(_('link'), max_length=512, verify_exists=False)

    def save(self, *args, **kwargs):

# Path: blango/spider.py
# class Spider(BaseSpider):
#     def __init__(self, uri, data = ''):
#         super(Spider, self).__init__(uri)
#         self.data = data
# 
#     def pingback(self, source):
#         if not self._require_fetch():
#             return False
#         pburi = self.stream.headers.get('X-Pingback')
#         if not pburi:
#             r = re.compile('<link.*?rel="pingback".*?href="([^"]+)" ?/?>', re.I)
#             m = r.search(self.data)
#             if not m:
#                 return False
#             pburi = m.group(1)
#         server = ServerProxy(pburi)
#         try:
#             server.pingback.ping(source, self.uri)
#             return True
#         except Error, v:
#             return False
# 
#     def trackback(self, url, page_name='', title='', excerpt=''):
#         if not self._require_fetch():
#             print self.code
#             return False
# 
#         r = re.compile('trackback:ping="([^"]+)"')
#         m = r.search(self.data)
#         if not m:
#             return False
#         tburi = m.group(1)
#         t = Spider(tburi)
#         t.request.add_header('Content-Type', 'application/x-www-form-urlencoded; charset=utf-8')
#         values = { 'title': title,
#             'excerpt': excerpt,
#             'url': url,
#             'blog_name': page_name }
#         data = urllib.urlencode(values)
#         t.request = urllib2.Request(tburi, data)
#         t.fetch()
#         return t.data.find('<error>0</error>') != -1
# 
#     def backlinks(self, back):
#         if not self._require_fetch():
#             return False
# 
#         r = re.compile('<a.*?href="%s".*>' % back)
#         return r.search(self.data) != None
# 
# Path: stem.py
# class Stemmer(object):
#     def __init__(self, lang, encoding=None):
#         self.stemmer = libstemmer.sb_stemmer_new(lang, encoding)
#         if not self.stemmer:
#             raise RuntimeError('Lang "%s" is not supported by this stemmer' % lang)
# 
#         self.mutex = Lock()
# 
#     def __del__(self):
#         libstemmer.sb_stemmer_delete(self.stemmer)
#         # No need to call the parent class, since object.__del__ does not exist
# 
#     def stem(self, word):
#         if isinstance(word, unicode):
#             w = word.encode('utf8')
#         else:
#             w = word
# 
#         # sb_stemmer_stem stemmer is not thread safe
#         self.mutex.acquire()
#         stem = libstemmer.sb_stemmer_stem(self.stemmer, w.lower(), len(w))
#         self.mutex.release()
#         return stem
# 
# HAS_LIBSTEMMER = True


# Path: blango/models.py
import re
import os
import urlparse
from datetime import datetime, timedelta
from cStringIO import StringIO
from django.db import models
from django.contrib.auth.models import User
from django.template.defaultfilters import slugify, capfirst, force_escape
from django.utils.translation import ugettext, ugettext_lazy as _
from django.utils.safestring import mark_safe
from django.core.files.base import ContentFile
from django.conf import settings
from PIL import Image
from blango.spider import Spider
from stem import Stemmer, HAS_LIBSTEMMER
from markdown import markdown

                    common_stems.append(stem.pk)
                    max_dist += stem.value ** 2 + entry_stem.value ** 2
                except EntryStems.DoesNotExist:
                    increment = entry_stem.value ** 2
                    abs_dist += increment
                    max_dist += increment

            for entry_stem in self.stems.exclude(pk__in=common_stems):
                increment = entry_stem.value ** 2
                abs_dist += increment
                max_dist += increment

            if abs_dist/max_dist < settings.RELATED_THRESHOLD:
                self.related.add(e)

    def get_admin_absolute_url(self):
        try:
            return self.linkentry.get_admin_absolute_url()
        except:
            pass
        return '/admin/%s/%s/%s/' % (self._meta.app_label, self._meta.module_name, self.pk)

class LinkEntry(Entry):
    class Meta:
        verbose_name = _('link entry')
        verbose_name_plural = _('link entries')

    link = models.URLField(_('link'), max_length=512, verify_exists=False)

    def save(self, *args, **kwargs):

        spider = Spider(self.link)