<commit_before>from __future__ import unicode_literals

from django.core.management.base import BaseCommand
from django.db import connection, transaction

from geotrek.core.models import Path


class Command(BaseCommand):
    help = """Remove all duplicate path (same geom)."""
    """Do not remove path with topology."""

    def handle(self, *args, **options):
        verbosity = options['verbosity']
        cursor = connection.cursor()
        query = """with relations as (SELECT t1.id as t1_id,
                                             t2.id as t2_id
                                      FROM l_t_troncon t1
                                      JOIN l_t_troncon t2 ON t1.id < t2.id AND ST_OrderingEquals(t1.geom, t2.geom)
                                      ORDER BY t1.id, t2.id)
                      SELECT * FROM relations WHERE t1_id NOT IN (SELECT t2_id FROM relations) """
        cursor.execute(query)
        list_topologies = cursor.fetchall()

        path_deleted = []

        with transaction.atomic():
            try:
                for path1_pk, path2_pk in list_topologies:
                    path2 = Path.objects.get(pk=path2_pk)
                    path2.aggregations.update(path_id=path1_pk)
                    path_deleted.append(path2)
                    if verbosity > 1:
                        self.stdout.write("Deleting path %s" % path2)
                    path2.delete()

            except Exception as exc:
                self.stdout.write(self.style.ERROR("{}".format(exc)))
<commit_msg>Include invisible in remove duplicate paths<commit_after>from __future__ import unicode_literals

from django.core.management.base import BaseCommand
from django.db import connection, transaction

from geotrek.core.models import Path


class Command(BaseCommand):
    help = """Remove all duplicate path (same geom)."""
    """Do not remove path with topology."""

    def handle(self, *args, **options):
        verbosity = options['verbosity']
        cursor = connection.cursor()
        query = """with relations as (SELECT t1.id as t1_id,
                                             t2.id as t2_id
                                      FROM l_t_troncon t1
                                      JOIN l_t_troncon t2 ON t1.id < t2.id AND ST_OrderingEquals(t1.geom, t2.geom)
                                      ORDER BY t1.id, t2.id)
                      SELECT * FROM relations WHERE t1_id NOT IN (SELECT t2_id FROM relations) """
        cursor.execute(query)
        list_topologies = cursor.fetchall()

        path_deleted = []

        with transaction.atomic():
            try:
                for path1_pk, path2_pk in list_topologies:
                    path2 = Path.include_invisible.get(pk=path2_pk)
                    path2.aggregations.update(path_id=path1_pk)
                    path_deleted.append(path2)
                    if verbosity > 1:
                        self.stdout.write("Deleting path %s" % path2)
                    path2.delete()

            except Exception as exc:
                self.stdout.write(self.style.ERROR("{}".format(exc)))

{'fraction_non_alphanumeric': 0.08525641025641026, 'fraction_numerical': 0.015384615384615385, 'mean_word_length': 1.6315345699831365, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5626289', 'n_tokens_mistral': 865, 'n_tokens_neox': 816, 'n_words': 219}