<commit_before># -*- encoding: utf8 -*-
import datetime
import logging
import os
import time
import subprocess
from optparse import make_option
from django.core.management.base import BaseCommand, CommandError
from django.db.models import Q
from cvn.models import Usuario, Proyecto, Publicacion, Congreso, Convenio
from string_utils.stringcmp import do_stringcmp

logger = logging.getLogger(__name__)


def difering_fields(obj1, obj2, EXCLUDE_FIELDS=[]):
    # return:
    #  - how many of them are different
    # fields where one or two of the values is None,
    # are not counted as different
    # pass a EXCLUDE_FIELDS list with the fields you want
    # to ignore in the comparison
    assert (type(obj1) == type(obj2),
            "The types of the objects are not the same, dude")
    difering = []
    tipo = type(obj1)
    fields = tipo._meta.get_all_field_names()
    for f in fields:
        if f not in EXCLUDE_FIELDS:
            f1 = obj1.__getattribute__(f)
            f2 = obj2.__getattribute__(f)
            if f1 and f2 and f1 != f2:
                difering.append(f)
    return len(difering)


def log_print(message):
    print message
    logger.info(message)

def backupDatabase(username, dbname, port):
    backupdir = os.environ['HOME'] + '/backup'
    date = time.strftime('%Y-%m-%d.%H:%M:%S')
    filepath = "%s/%s.%s.gz" % (backupdir, dbname, date)
    params = "pg_dump -U%s -p%s -W %s | gzip -9 -c > %s" % (username, port, dbname, filepath)

    if not os.path.exists(backupdir):
        os.mkdir(backupdir)
    proc = subprocess.Popen([params], stderr=subprocess.PIPE, shell=True)
    (out, err) = proc.communicate()
    if err:
        os.system("rm " + filepath)
    return err

class Command(BaseCommand):
    help = u'Encuentra registros sospechosos de estar duplicados'
    option_list = BaseCommand.option_list + (
        make_option(
            "-u",
            "--document",
            dest="usuario",
            default=False,
            help="specify ID document for duplicate control. Use 'all'.",
        ),
        make_option(
            "-t",
            "--table",
            dest="table",
            help="specify table for duplicate control",
        ),
        make_option(
            "-d",
            "--diff",
            dest="differing_pairs",
            help="specify the number of differing pairs for duplicate control",
        ),
        make_option(
            "-y",
            "--year",
            dest="year",
            default=False,
            help="specify the year for searching in format XXXX or use 'all'",
        ),
    )

    TABLES = {'Proyecto': Proyecto,
              'Publicacion': Publicacion,
              'Congreso': Congreso,
              'Convenio': Convenio}
    NAME_FIELDS = {'Proyecto': 'denominacion_del_proyecto',
                   'Publicacion': 'titulo',
                   'Congreso': 'titulo',
                   'Convenio': 'denominacion_del_proyecto'}

    TIMESTAMP_FIELDS = ['updated_at', 'created_at', ]

    DONT_SET_FIELDS = ['id', 'usuario', ]

    DONT_CHECK_FIELDS = TIMESTAMP_FIELDS + DONT_SET_FIELDS

    # mínima similitud para considerar dos registros como duplicados
    LIMIT = 0.75

    DIFFERING_PAIRS = 0  # Default value. No contamos la denominación.

    FIELD_WIDTH = 28
    COLWIDTH = 50

    def print_cabecera_registro(self, pry1, pry2, duplicates, pair, model_fields):
        os.system("clear")
        log_print("=============================================")
        log_print(" ID1 = {0} comparado con ID2 = {1} ({2:2.2f}%)"
            .format(pry1.id, pry2.id, duplicates[pair]*100))
        log_print("=============================================")
        # overview of the two registers
        log_print("Field".ljust(self.FIELD_WIDTH)
            + "ID1".ljust(self.COLWIDTH)
            + "ID2".ljust(self.COLWIDTH))
        log_print("-" * (self.FIELD_WIDTH + 2 * self.COLWIDTH))
        for f in model_fields:
            if f not in (self.DONT_SET_FIELDS +
                self.TIMESTAMP_FIELDS):
                f1 = pry1.__getattribute__(f)
                f1 = "" if f1 is None else f1
                f2 = pry2.__getattribute__(f)
                f2 = "" if f2 is None else f2
                if any([f1, f2]):
                    log_print(unicode(f)[:self.FIELD_WIDTH-1]
                        .ljust(self.FIELD_WIDTH)
                        + unicode(f1)[:self.COLWIDTH-1]
                        .ljust(self.COLWIDTH)
                        + unicode(f2)[:self.COLWIDTH-1]
                        .ljust(self.COLWIDTH))
        log_print("-" * (self.FIELD_WIDTH + 2 * self.COLWIDTH))

    def checkArgs(self, options):
        #Esta funcion chequea los argumentos pasados por el usuario
        if options['table'] is None:
            raise CommandError("Option `--table=...` must be specified.")
        else:
            if options['table'] in self.TABLES:
                TABLE = self.TABLES[options['table']]
                NAME_FIELD = self.NAME_FIELDS[options['table']]
            else:
                raise CommandError("\"{0}\" is not a table. Use Proyecto, " +
                                   "Convenio, Publicacion or Congreso "
                                   .format(options['table']))
        if options['differing_pairs'] is None:
            raise CommandError("Option `--diff=0, 1...` must be specified.")
        else:
            try:
                self.DIFFERING_PAIRS = int(options['differing_pairs'])
            except:
                raise CommandError("Option `--diff needs an integer 0,1,...")
        return TABLE, NAME_FIELD

    def runQueries(self, options, TABLE):
        log_print("Buscando duplicados en el modelo " +
                  "{0}".format(TABLE.__name__))
        if not options['year']:
            registros = TABLE.objects.exclude(usuario=None)
        else:
            self.YEAR = options['year']
            if TABLE == Proyecto:
                # --------------------------- PROYECTOS -------------------- #
                # PROYECTOS vigentes en 2012
                registros = TABLE.objects.filter(
                    Q(fecha_de_inicio__lte=datetime.date(self.YEAR, 12, 31)) &
                    Q(fecha_de_fin__gt=datetime.date(self.YEAR, 1, 1)))
                # excluye los registros ya detectados previamente
                # y que están huérfanos de usuario
                registros = registros.exclude(usuario=None)
                # ---------------------------------------------------------- #

            elif TABLE == Publicacion:
                # ---------------------- PUBLICACIONES --------------------- #
                # PUBLICACIONES en 2012
                registros = TABLE.objects.filter(fecha__year=self.YEAR)
                # excluye los registros ya detectados previamente
                # y que están huérfanos de usuario
                registros = registros.exclude(usuario=None)
                # ---------------------------------------------------------- #
<commit_msg>Firltrado por tipo de publicacion en el script find_pairs<commit_after># -*- encoding: utf8 -*-
import datetime
import logging
import os
import time
import subprocess
from optparse import make_option
from django.core.management.base import BaseCommand, CommandError
from django.db.models import Q
from cvn.models import Usuario, Proyecto, Publicacion, Congreso, Convenio
from string_utils.stringcmp import do_stringcmp

logger = logging.getLogger(__name__)


def difering_fields(obj1, obj2, EXCLUDE_FIELDS=[]):
    # return:
    #  - how many of them are different
    # fields where one or two of the values is None,
    # are not counted as different
    # pass a EXCLUDE_FIELDS list with the fields you want
    # to ignore in the comparison
    assert (type(obj1) == type(obj2),
            "The types of the objects are not the same, dude")
    difering = []
    tipo = type(obj1)
    fields = tipo._meta.get_all_field_names()
    for f in fields:
        if f not in EXCLUDE_FIELDS:
            f1 = obj1.__getattribute__(f)
            f2 = obj2.__getattribute__(f)
            if f1 and f2 and f1 != f2:
                difering.append(f)
    return len(difering)


def log_print(message):
    print message
    logger.info(message)

def backupDatabase(username, dbname, port):
    backupdir = os.environ['HOME'] + '/backup'
    date = time.strftime('%Y-%m-%d.%H:%M:%S')
    filepath = "%s/%s.%s.gz" % (backupdir, dbname, date)
    params = "pg_dump -U%s -p%s -W %s | gzip -9 -c > %s" % (username, port, dbname, filepath)

    if not os.path.exists(backupdir):
        os.mkdir(backupdir)
    proc = subprocess.Popen([params], stderr=subprocess.PIPE, shell=True)
    (out, err) = proc.communicate()
    if err:
        os.system("rm " + filepath)
    return err

class Command(BaseCommand):
    help = u'Encuentra registros sospechosos de estar duplicados'
    option_list = BaseCommand.option_list + (
        make_option(
            "-u",
            "--document",
            dest="usuario",
            default=False,
            help="specify ID document for duplicate control. Use 'all'.",
        ),
        make_option(
            "-t",
            "--table",
            dest="table",
            help="specify table for duplicate control",
        ),
        make_option(
            "-d",
            "--diff",
            dest="differing_pairs",
            help="specify the number of differing pairs for duplicate control",
        ),
        make_option(
            "-y",
            "--year",
            dest="year",
            default=False,
            help="specify the year for searching in format XXXX or use 'all'",
        ),
        make_option(
            "-p",
            "--publication",
            dest="publication",
            default=False,
            help="specify the type of publication for searching",
        ),
    )
    
    PUBLICATION_TYPES = {'libro' : u'Libro', 'capitulo' : u'Capítulo de Libro', 'articulo' : u'Artículo'}

    TABLES = {'Proyecto': Proyecto,
              'Publicacion': Publicacion,
              'Congreso': Congreso,
              'Convenio': Convenio}
    NAME_FIELDS = {'Proyecto': 'denominacion_del_proyecto',
                   'Publicacion': 'titulo',
                   'Congreso': 'titulo',
                   'Convenio': 'denominacion_del_proyecto'}

    TIMESTAMP_FIELDS = ['updated_at', 'created_at', ]

    DONT_SET_FIELDS = ['id', 'usuario', ]

    DONT_CHECK_FIELDS = TIMESTAMP_FIELDS + DONT_SET_FIELDS

    # mínima similitud para considerar dos registros como duplicados
    LIMIT = 0.75

    DIFFERING_PAIRS = 0  # Default value. No contamos la denominación.

    FIELD_WIDTH = 28
    COLWIDTH = 50

    def print_cabecera_registro(self, pry1, pry2, duplicates, pair, model_fields):
        os.system("clear")
        log_print("=============================================")
        log_print(" ID1 = {0} comparado con ID2 = {1} ({2:2.2f}%)"
            .format(pry1.id, pry2.id, duplicates[pair]*100))
        log_print("=============================================")
        # overview of the two registers
        log_print("Field".ljust(self.FIELD_WIDTH)
            + "ID1".ljust(self.COLWIDTH)
            + "ID2".ljust(self.COLWIDTH))
        log_print("-" * (self.FIELD_WIDTH + 2 * self.COLWIDTH))
        for f in model_fields:
            if f not in (self.DONT_SET_FIELDS +
                self.TIMESTAMP_FIELDS):
                f1 = pry1.__getattribute__(f)
                f1 = "" if f1 is None else f1
                f2 = pry2.__getattribute__(f)
                f2 = "" if f2 is None else f2
                if any([f1, f2]):
                    log_print(unicode(f)[:self.FIELD_WIDTH-1]
                        .ljust(self.FIELD_WIDTH)
                        + unicode(f1)[:self.COLWIDTH-1]
                        .ljust(self.COLWIDTH)
                        + unicode(f2)[:self.COLWIDTH-1]
                        .ljust(self.COLWIDTH))
        log_print("-" * (self.FIELD_WIDTH + 2 * self.COLWIDTH))

    def checkArgs(self, options):
        #Esta funcion chequea los argumentos pasados por el usuario
        if options['publication']:
            if options['publication'] not in self.PUBLICATION_TYPES:
                raise CommandError('Option publication: allowed values: libro, capitulo, articulo')
        if options['table'] is None:
            raise CommandError("Option `--table=...` must be specified.")
        else:
            if options['table'] in self.TABLES:
                TABLE = self.TABLES[options['table']]
                NAME_FIELD = self.NAME_FIELDS[options['table']]
            else:
                raise CommandError("\"{0}\" is not a table. Use Proyecto, " +
                                   "Convenio, Publicacion or Congreso "
                                   .format(options['table']))
        if options['differing_pairs'] is None:
            raise CommandError("Option `--diff=0, 1...` must be specified.")
        else:
            try:
                self.DIFFERING_PAIRS = int(options['differing_pairs'])
            except:
                raise CommandError("Option `--diff needs an integer 0,1,...")
        return TABLE, NAME_FIELD

    def runQueries(self, options, TABLE):
        log_print("Buscando duplicados en el modelo " +
                  "{0}".format(TABLE.__name__)) 
        if not options['year']:
            registros = TABLE.objects.exclude(usuario=None)
        else:
            self.YEAR = options['year']
            if TABLE == Proyecto:
                # --------------------------- PROYECTOS -------------------- #
                # PROYECTOS vigentes en 2012
                registros = TABLE.objects.filter(
                    Q(fecha_de_inicio__lte=datetime.date(self.YEAR, 12, 31)) &
                    Q(fecha_de_fin__gt=datetime.date(self.YEAR, 1, 1)))
                # excluye los registros ya detectados previamente
                # y que están huérfanos de usuario
                registros = registros.exclude(usuario=None)
                # ---------------------------------------------------------- #

            elif TABLE == Publicacion:
                # ---------------------- PUBLICACIONES --------------------- #
                # PUBLICACIONES en 2012
                registros = TABLE.objects.filter(fecha__year=self.YEAR)
                # excluye los registros ya detectados previamente
                # y que están huérfanos de usuario
                registros = registros.exclude(usuario=None)
                # ---------------------------------------------------------- #
                if options['publication']:
                    registros = registros.filter(tipo_de_produccion=self.PUBLICATION_TYPES[options['publication']])

{'fraction_non_alphanumeric': 0.14429507532256974, 'fraction_numerical': 0.010538404377491048, 'mean_word_length': 1.761425107256109, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3382829', 'n_tokens_mistral': 4471, 'n_tokens_neox': 4261, 'n_words': 1044}