<commit_before># This script is used to cleanup the URLs that disqus has comments associated.
# Annoyingly Disqus does not appear to strip out commonly added parameters such
# as `gclid` (the Google Click Identifier that AdWords adds to URLs).
#
# Leaving aside that though - when the default page layout changed from tab
# based to individual pages the URLs to the comments all changed, and we started
# to use the disqus_identifier properly. So the URLs would have needed changing
# anyway.
#
# This script takes a CSV file that can be requested on this page:
#
#    http://YOUR_SHORTNAME.disqus.com/admin/tools/migrate/
#    ( click on 'Start URL Mapper' and then on 'you can download a CSV here')
#
# It the outputs (to STDOUT) a CSV that can be uploaded (ie rows with "old_url,
# new_url").
#
# It does the following:
#
#  * Strips off all query parameters
#  * adds '/comments' to the end if needed
#
#  No checking or correcting of the URLs occurs.
#
# The resulting CSV can then be uploaded on the 'URL Mapper' page that the input
# CSV was downloaded from.

import re
import sys
import csv
from django.core.management.base import LabelCommand

class Command(LabelCommand):
    help = 'Correct URLs that Disqus has for comments'
    args = '<CSV requested from Disqus>'

    def handle_label(self,  filename, **options):

        csv_reader = csv.reader(open(filename, 'rb'))
        csv_writer = csv.writer(sys.stdout)

        for row in csv_reader:
            old_url = row[0]
            new_url = old_url

            # strip query
            new_url = re.sub(r'\?.*$', '', new_url)

            # add '/comment/' if needed
            if not re.search(r'/comments/$', new_url):
                new_url += 'comments/'

            csv_writer.writerow([old_url, new_url])<commit_msg>Update the command for generating a Disqus old->new thread URL mapping<commit_after># This script is used to cleanup the URLs that disqus has comments associated.
# Annoyingly Disqus does not appear to strip out commonly added parameters such
# as `gclid` (the Google Click Identifier that AdWords adds to URLs), and we've
# got separate threads for each sub-page of a person, whereas they should
# really just be a single thread.
#
# This script takes a CSV file that can be requested on this page:
#
#    http://YOUR_SHORTNAME.disqus.com/admin/tools/migrate/
#    ( click on 'Start URL Mapper' and then on 'you can download a CSV here')
#
# It the outputs (to STDOUT) a CSV that can be uploaded (ie rows with "old_url,
# new_url").
#
# It does the following:
#
#  * Identifies the type of the object (person, place or organisation) and
#    its slug (even if it's in a percent-encoded query parameter from a proxy)
#  * Reconstructs the canonical URL for that object
#
# This doesn't actually check that the URL works, but attempts to
# remove anything extraneous to produce the canonical URL for the
# person.
#
# The resulting CSV can then be uploaded on the 'URL Mapper' page that the input
# CSV was downloaded from.

from __future__ import unicode_literals, print_function

import re
import sys
import csv
from urllib import unquote
from urlparse import urlsplit
from django.core.management.base import LabelCommand

class Command(LabelCommand):
    help = 'Correct URLs that Disqus has for comments'
    args = '<CSV requested from Disqus>'

    def rewrite_url(self, old_url):
        split_url = urlsplit(old_url)
        path_re = r'/+(hansard/+)?(?P<type>person|place|organisation)/+([^/?]*?)(/| |$)'
        if split_url.netloc == 'info.mzalendo.com':
            # Look for the straightforward pattern of person, organisation
            # or place page or sub-pages:
            m = re.search('^' + path_re, split_url.path)
        else:
            # Otherwise it's probably a proxy URL, in which case we
            # can sometimes find the path by removing the
            # percent-encoding from it:
            unquoted = unquote(old_url)
            m = re.search(path_re, unquoted)
        if m:
            return 'http://info.mzalendo.com/{1}/{2}/'.format(*m.groups())

    def handle_label(self,  filename, **options):
        verbose = options['verbosity'] > 1

        new_mapping = []

        with open(filename, 'r') as f:
            reader = csv.reader(f)
            for row in reader:
                old_url = row[0]
                new_url = self.rewrite_url(old_url)
                if new_url is None:
                    if verbose:
                        print("Skipping: {0}".format(old_url), file=sys.stderr)
                else:
                    new_mapping.append([old_url, new_url])

        writer = csv.writer(sys.stdout)
        for old, new in new_mapping:
            writer.writerow([old, new])
{'fraction_non_alphanumeric': 0.08889825152728038, 'fraction_numerical': 0.0012639561828523278, 'mean_word_length': 2.759303246239113, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2012740', 'n_tokens_mistral': 1422, 'n_tokens_neox': 1346, 'n_words': 548}