<commit_before>import logging
from optparse import make_option
import re
from bs4 import BeautifulSoup
import couchdb
from django.core.management import BaseCommand
from django.conf import settings
from django.utils.text import slugify
import requests
from bilanci.utils import UnicodeDictReader
__author__ = 'guglielmo'

class Command(BaseCommand):


    option_list = BaseCommand.option_list + (
        make_option('--dry-run',
                    dest='dryrun',
                    action='store_true',
                    default=False,
                    help='Set the dry-run command mode: nothing is written in the couchdb'),
        make_option('--years',
                    dest='years',
                    default='',
                    help='Years to fetch. From 2002 to 2012. Use one of this formats: 2012 or 2003-2006 or 2002,2004,2006'),
        make_option('--cities',
                    dest='cities',
                    default='',
                    help='Cities codes or slugs. Use comma to separate values: Roma,Napoli,Torino'),
        make_option('--base-url',
                    dest='base_url',
                    default='http://finanzalocale.mirror.openpolis.it',
                    help='Base URL for HTML files (mirror)')

    )

    help = 'Import the data for given cities and years, from HTML code into Couchdb'

    logger = logging.getLogger('management')
    comuni_dicts = {}


    def handle(self, *args, **options):
        verbosity = options['verbosity']
        if verbosity == '0':
            self.logger.setLevel(logging.ERROR)
        elif verbosity == '1':
            self.logger.setLevel(logging.WARNING)
        elif verbosity == '2':
            self.logger.setLevel(logging.INFO)
        elif verbosity == '3':
            self.logger.setLevel(logging.DEBUG)

        dryrun = options['dryrun']

        cities_codes = options['cities']
        if not cities_codes:
            raise Exception("Missing city parameter")

        cities = self.get_cities(cities_codes)
        self.logger.info("Scraping cities: {0}".format(cities))


        years = options['years']
        if not years:
            raise Exception("Missing years parameter")

        if "-" in years:
            (start_year, end_year) = years.split("-")
            years = range(int(start_year), int(end_year)+1)
        else:
            years = [int(y.strip()) for y in years.split(",") if 2001 < int(y.strip()) < 2013]

        if not years:
            raise Exception("No suitable year found in {0}".format(years))

        self.logger.info("Scraping years: {0}".format(years))

        base_url = options['base_url']

        # setup couchdb connection
        couch_server = couchdb.Server()

        for city in cities:
            for year in years:

                # Preventivo
                q_list = ["%02d" % (i,) for i in range(1, 10)]
                url = "{0}/{1}/{2}/Preventivo".format(
                        base_url, year, city
                )
                self.logger.info("Scraping Preventivo: {0}".format(url))
                preventivo = self.scrape(
                    url,
                    pages=q_list
                )

                # Consuntivo
                q_list = ["%02d" % (i,) for i in range(1, 20)]
                q_list.append("50")
                url = "{0}/{1}/{2}/Consuntivo".format(
                    base_url, year, city
                )
                self.logger.info("Scraping Consuntivo: {0}".format(url))
                consuntivo = self.scrape(
                    url,
                    pages=q_list
                )

                # prepare global data
                bilancio_id = "{0}_{1}".format(year, city)
                bilancio_data = {
                    'anno': year,
                    'city_name': city.split("--")[0],
                    'preventivo': preventivo,
                    'consuntivo': consuntivo
                }

                if not dryrun:
                    bilanci_db = couch_server['bilanci']

                    # create or update budget data on couchdb
                    if bilancio_id in bilanci_db:
                        bilancio_data['_rev'] = bilanci_db[bilancio_id].rev

                    bilanci_db[bilancio_id] = bilancio_data
                    self.logger.info("Data written to couchdb")
                else:
                    self.logger.info("Couchdb writing skipped because of --dry-run")


    def get_comuni_dicts(self):
        """
        read list of comuni from CSV into a single dictionary
        having the code as key and the slug as value
        """
        lista_comuni_csv = settings.LISTA_COMUNI_PATH
        try:
            udr = UnicodeDictReader(f=open(lista_comuni_csv, mode='r'), dialect="excel_quote_all",)
        except IOError:
            raise Exception("Impossible to open file:%s" % lista_comuni_csv)

        comuni_by_codes = {}
        comuni_by_slugs = {}
        for row in udr:
            complete = "{0}--{1}".format(row['NOME_COMUNE'].upper(), row['CODICE_COMUNE'])
            comuni_by_codes[row['CODICE_COMUNE']] = complete
            comuni_by_slugs[row['NOME_COMUNE']] = complete

        return {'codes': comuni_by_codes, 'slugs': comuni_by_slugs}


    _digits = re.compile('\d')
    def contains_digits(self, d):
        return bool(self._digits.search(d))

    def get_cities(self, codes):
        """
        Returns the list of complete names of the cities, used in the html files
        starting from codes or slugs.

        The type of strings passed is automatically guessed.
        """
        if not self.comuni_dicts:
            self.comuni_dicts = self.get_comuni_dicts()

        codes = [c.strip().upper() for c in codes.split(",")]

        ret = []
        for code in codes:
            try:
                if self.contains_digits(code):
                    # it's a code
                    ret.append(self.comuni_dicts['codes'][code])
                else:
                    # it's a slug
                    ret.append(self.comuni_dicts['slugs'][code])
            except KeyError:
                continue

        return ret


    def scrape(self, path, pages=None):
        """
        Create and return data out of HTML code, for each page in pages list
        """
        data = {}
        for quadro in pages:
            data[quadro]={}
            self.logger.debug("Scraping quadro:{0}.html".format(quadro))<commit_msg>all shortcut added to cities parameter in html2couch script<commit_after>import logging
from optparse import make_option
import re
from bs4 import BeautifulSoup
import couchdb
from django.core.management import BaseCommand
from django.conf import settings
from django.utils.text import slugify
import requests
from bilanci.utils import UnicodeDictReader
__author__ = 'guglielmo'

class Command(BaseCommand):


    option_list = BaseCommand.option_list + (
        make_option('--dry-run',
                    dest='dryrun',
                    action='store_true',
                    default=False,
                    help='Set the dry-run command mode: nothing is written in the couchdb'),
        make_option('--years',
                    dest='years',
                    default='',
                    help='Years to fetch. From 2002 to 2012. Use one of this formats: 2012 or 2003-2006 or 2002,2004,2006'),
        make_option('--cities',
                    dest='cities',
                    default='',
                    help='Cities codes or slugs. Use comma to separate values: Roma,Napoli,Torino or  "All"'),
        make_option('--base-url',
                    dest='base_url',
                    default='http://finanzalocale.mirror.openpolis.it',
                    help='Base URL for HTML files (mirror)')

    )

    help = 'Import the data for given cities and years, from HTML code into Couchdb'

    logger = logging.getLogger('management')
    comuni_dicts = {}


    def handle(self, *args, **options):
        verbosity = options['verbosity']
        if verbosity == '0':
            self.logger.setLevel(logging.ERROR)
        elif verbosity == '1':
            self.logger.setLevel(logging.WARNING)
        elif verbosity == '2':
            self.logger.setLevel(logging.INFO)
        elif verbosity == '3':
            self.logger.setLevel(logging.DEBUG)

        dryrun = options['dryrun']

        cities_codes = options['cities']
        if not cities_codes:
            raise Exception("Missing city parameter")

        cities = self.get_cities(cities_codes)
        if cities_codes.lower() != 'all':
            self.logger.info("Scraping cities: {0}".format(cities))


        years = options['years']
        if not years:
            raise Exception("Missing years parameter")

        if "-" in years:
            (start_year, end_year) = years.split("-")
            years = range(int(start_year), int(end_year)+1)
        else:
            years = [int(y.strip()) for y in years.split(",") if 2001 < int(y.strip()) < 2013]

        if not years:
            raise Exception("No suitable year found in {0}".format(years))

        self.logger.info("Scraping years: {0}".format(years))

        base_url = options['base_url']

        # setup couchdb connection
        couch_server = couchdb.Server()

        for city in cities:
            for year in years:

                # Preventivo
                q_list = ["%02d" % (i,) for i in range(1, 10)]
                url = "{0}/{1}/{2}/Preventivo".format(
                        base_url, year, city
                )
                self.logger.info("Scraping Preventivo: {0}".format(url))
                preventivo = self.scrape(
                    url,
                    pages=q_list
                )

                # Consuntivo
                q_list = ["%02d" % (i,) for i in range(1, 20)]
                q_list.append("50")
                url = "{0}/{1}/{2}/Consuntivo".format(
                    base_url, year, city
                )
                self.logger.info("Scraping Consuntivo: {0}".format(url))
                consuntivo = self.scrape(
                    url,
                    pages=q_list
                )

                # prepare global data
                bilancio_id = "{0}_{1}".format(year, city)
                bilancio_data = {
                    'anno': year,
                    'city_name': city.split("--")[0],
                    'preventivo': preventivo,
                    'consuntivo': consuntivo
                }

                if not dryrun:
                    bilanci_db = couch_server['bilanci']

                    # create or update budget data on couchdb
                    if bilancio_id in bilanci_db:
                        bilancio_data['_rev'] = bilanci_db[bilancio_id].rev

                    bilanci_db[bilancio_id] = bilancio_data
                    self.logger.info("Data written to couchdb")
                else:
                    self.logger.info("Couchdb writing skipped because of --dry-run")


    def get_comuni_dicts(self):
        """
        read list of comuni from CSV into a single dictionary
        having the code as key and the slug as value
        """
        lista_comuni_csv = settings.LISTA_COMUNI_PATH
        try:
            udr = UnicodeDictReader(f=open(lista_comuni_csv, mode='r'), dialect="excel_quote_all",)
        except IOError:
            raise Exception("Impossible to open file:%s" % lista_comuni_csv)

        comuni_by_codes = {}
        comuni_by_slugs = {}
        for row in udr:
            complete = "{0}--{1}".format(row['NOME_COMUNE'].upper(), row['CODICE_COMUNE'])
            comuni_by_codes[row['CODICE_COMUNE']] = complete
            comuni_by_slugs[row['NOME_COMUNE']] = complete

        return {'codes': comuni_by_codes, 'slugs': comuni_by_slugs}


    _digits = re.compile('\d')
    def contains_digits(self, d):
        return bool(self._digits.search(d))

    def get_cities(self, codes):
        """
        Returns the list of complete names of the cities, used in the html files
        starting from codes or slugs.

        The type of strings passed is automatically guessed.

        Return the complete list, if the All shortcut is used.
        """
        if not self.comuni_dicts:
            self.comuni_dicts = self.get_comuni_dicts()

        if codes.lower() == 'all':
            return self.comuni_dicts['codes'].values()

        codes = [c.strip().upper() for c in codes.split(",")]

        ret = []
        for code in codes:
            try:
                if self.contains_digits(code):
                    # it's a code
                    ret.append(self.comuni_dicts['codes'][code])
                else:
                    # it's a slug
                    ret.append(self.comuni_dicts['slugs'][code])
            except KeyError:
                continue

        return ret


    def scrape(self, path, pages=None):
        """
        Create and return data out of HTML code, for each page in pages list
        """
        data = {}
        for quadro in pages:
            data[quadro]={}
            self.logger.debug("Scraping quadro:{0}.html".format(quadro))
{'fraction_non_alphanumeric': 0.10701581775524105, 'fraction_numerical': 0.011428138954060396, 'mean_word_length': 1.6592875830146911, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2272614', 'n_tokens_mistral': 3920, 'n_tokens_neox': 3607, 'n_words': 921}