# Path: storylist/models.py
# class Story(models.Model):
#     headline = models.CharField(max_length=100)
#     url = models.CharField(max_length=250)
#     source = models.CharField(max_length=50)
#     added = models.DateTimeField()
# 
#     def __unicode__(self):
#     	return u'%s: %s' % (self.source, self.headline)


import sys
import os
from storylist.models import Story
from scrapy.contrib.djangoitem import DjangoItem
"""
Helper functions for scrapy project
dupCheck():		Delete duplicate entries.
				Called after each reactor.run() in crimelog.py

"""
sys.path.insert(0, '/home/amirkurtovic/crimeline')

os.environ['DJANGO_SETTINGS_MODULE'] = 'crimeline.settings'



def dupCheck():
	'''
	Searches database for duplicate URL entries before saving item.
	item: Story object passed by each spider
	'''


# Path: storylist/models.py
# class Story(models.Model):
#     headline = models.CharField(max_length=100)
#     url = models.CharField(max_length=250)
#     source = models.CharField(max_length=50)
#     added = models.DateTimeField()
# 
#     def __unicode__(self):
#     	return u'%s: %s' % (self.source, self.headline)


# Path: crimelog/crimelog/helpers.py
import sys
import os
from storylist.models import Story
from scrapy.contrib.djangoitem import DjangoItem

"""
Helper functions for scrapy project
dupCheck():		Delete duplicate entries.
				Called after each reactor.run() in crimelog.py

"""
sys.path.insert(0, '/home/amirkurtovic/crimeline')

os.environ['DJANGO_SETTINGS_MODULE'] = 'crimeline.settings'



def dupCheck():
	'''
	Searches database for duplicate URL entries before saving item.
	item: Story object passed by each spider
	'''


	for url in Story.objects.values_list('url', flat=True).distinct():