# Path: wikipedia/models.py
# class BulkImport(models.Model):
#     slug = models.SlugField(max_length=255)
#     created_at = models.DateTimeField(auto_now_add=True)
#     text = models.TextField()
# 
#     def __unicode__(self):
#         return str(self.created_at)
# 
# class ImportedNewspaper(models.Model):
#     bulk_import = models.ForeignKey(BulkImport)
#     confirmed_endorser = models.ForeignKey(Endorser, blank=True, null=True)
#     section = models.PositiveSmallIntegerField(choices=NEWSPAPER_SECTIONS)
#     name = models.CharField(max_length=100)
#     endorsement_2016 = models.CharField(max_length=15)
#     endorsement_2012 = models.CharField(max_length=15, blank=True, null=True)
#     circulation = models.PositiveIntegerField(blank=True, null=True)
#     date = models.DateField(blank=True, null=True)
#     city = models.CharField(max_length=20, blank=True, null=True)
#     state = models.CharField(max_length=20, blank=True, null=True)
#     url = models.URLField(blank=True, null=True, max_length=255)
# 
#     def __unicode__(self):
#         return self.name
# 
#     def get_likely_endorser(self):
#         name = self.name.lower()
# 
#         query = Endorser.objects.filter(
#             name__iexact=name,
#         )
#         if query.exists():
#             return query.first()
# 
#         # See if putting "The" in front (or removing it) helps.
#         if name.startswith('the '):
#             query = Endorser.objects.filter(
#                 name__iexact=name[4:],
#             )
#             if query.exists():
#                 return query.first()
#         else:
#             query = Endorser.objects.filter(
#                 name__iexact='the ' + name,
#             )
#             if query.exists():
#                 return query.first()
# 
# NEWSPAPER_SECTIONS = (
#     (3, 'Daily newspapers'),
#     (7, 'Weekly newspapers'),
#     #(11, 'Magazines'),
#     #(15, 'College and university newspapers'),
#     #(19, 'Foreign publications'),
# )


from datetime import datetime
from BeautifulSoup import BeautifulSoup as BS
from django.core.management.base import BaseCommand, CommandError
from wikipedia.models import BulkImport, ImportedNewspaper, NEWSPAPER_SECTIONS
import requests
                    cell_text = cell_text.replace(',', '')
                    if cell_text.isdigit():
                        circulation = int(cell_text)

                cell_value = table_cells[3]['data-sort-value']
                date = datetime.strptime(cell_value, '%Y-%m-%d').date()

                a_tags = table_cells[4].findAll('a')
                if not a_tags:
                    # National.
                    city = None
                else:
                    city = a_tags[0].string

                state = table_cells[5].find(text=True)
                if state == 'National':
                    state = None

                endorsement_2012 = table_cells[6].get('data-sort-value')

                row = {
                    'name': name,
                    '2016': endorsement_2016,
                    'circulation': circulation,
                    'date': date,
                    'city': city,
                    'state': state,
                    '2012': endorsement_2012,
                    'ref_id': ref_id,
                }

# Path: wikipedia/models.py
# class BulkImport(models.Model):
#     slug = models.SlugField(max_length=255)
#     created_at = models.DateTimeField(auto_now_add=True)
#     text = models.TextField()
# 
#     def __unicode__(self):
#         return str(self.created_at)
# 
# class ImportedNewspaper(models.Model):
#     bulk_import = models.ForeignKey(BulkImport)
#     confirmed_endorser = models.ForeignKey(Endorser, blank=True, null=True)
#     section = models.PositiveSmallIntegerField(choices=NEWSPAPER_SECTIONS)
#     name = models.CharField(max_length=100)
#     endorsement_2016 = models.CharField(max_length=15)
#     endorsement_2012 = models.CharField(max_length=15, blank=True, null=True)
#     circulation = models.PositiveIntegerField(blank=True, null=True)
#     date = models.DateField(blank=True, null=True)
#     city = models.CharField(max_length=20, blank=True, null=True)
#     state = models.CharField(max_length=20, blank=True, null=True)
#     url = models.URLField(blank=True, null=True, max_length=255)
# 
#     def __unicode__(self):
#         return self.name
# 
#     def get_likely_endorser(self):
#         name = self.name.lower()
# 
#         query = Endorser.objects.filter(
#             name__iexact=name,
#         )
#         if query.exists():
#             return query.first()
# 
#         # See if putting "The" in front (or removing it) helps.
#         if name.startswith('the '):
#             query = Endorser.objects.filter(
#                 name__iexact=name[4:],
#             )
#             if query.exists():
#                 return query.first()
#         else:
#             query = Endorser.objects.filter(
#                 name__iexact='the ' + name,
#             )
#             if query.exists():
#                 return query.first()
# 
# NEWSPAPER_SECTIONS = (
#     (3, 'Daily newspapers'),
#     (7, 'Weekly newspapers'),
#     #(11, 'Magazines'),
#     #(15, 'College and university newspapers'),
#     #(19, 'Foreign publications'),
# )


# Path: wikipedia/management/commands/importnewspapers.py
from datetime import datetime
from BeautifulSoup import BeautifulSoup as BS
from django.core.management.base import BaseCommand, CommandError
from wikipedia.models import BulkImport, ImportedNewspaper, NEWSPAPER_SECTIONS
import requests

                    cell_text = cell_text.replace(',', '')
                    if cell_text.isdigit():
                        circulation = int(cell_text)

                cell_value = table_cells[3]['data-sort-value']
                date = datetime.strptime(cell_value, '%Y-%m-%d').date()

                a_tags = table_cells[4].findAll('a')
                if not a_tags:
                    # National.
                    city = None
                else:
                    city = a_tags[0].string

                state = table_cells[5].find(text=True)
                if state == 'National':
                    state = None

                endorsement_2012 = table_cells[6].get('data-sort-value')

                row = {
                    'name': name,
                    '2016': endorsement_2016,
                    'circulation': circulation,
                    'date': date,
                    'city': city,
                    'state': state,
                    '2012': endorsement_2012,
                    'ref_id': ref_id,
                }

                if ImportedNewspaper.objects.filter(