<commit_before>from django.core.management.base import BaseCommand, CommandError
from django.core.files import File
from django.contrib.auth import get_user_model
User = get_user_model()
from django.conf import settings
from django.contrib.auth.models import User
from django_comments.models import Comment
from django_comments_xtd.models import XtdComment
from django.contrib.contenttypes.models import ContentType
from django.contrib.sites.models import Site
from base64 import b64encode
from blog.models import BlogPage
from optparse import make_option
import urllib.request
import os
import json
import requests
try:
    import html
except ImportError:  # 2.x
    import HTMLParser
    html = HTMLParser.HTMLParser()
from bs4 import BeautifulSoup
from blog.models import (BlogPage, BlogTag, BlogPageTag, BlogIndexPage,
                         BlogCategory, BlogCategoryBlogPage)
from wagtail.wagtailimages.models import Image


class Command(BaseCommand):
    """
    This is a management command to migrate a Wordpress site to Wagtail.
    Two arguments should be used - the site to be migrated and the site it is
    being migrated to.

    Users will first need to make sure the WP REST API(WP API) plugin is
    installed on the self-hosted Wordpress site to migrate.
    Next users will need to create a BlogIndex object in this GUI.
    This will be used as a parent object for the child blog page objects.
    """

    def add_arguments(self, parser):
        """have to add this to use args in django 1.8"""
        parser.add_argument('blog_to_migrate',
                            help="Base url of wordpress instance")
        parser.add_argument('blog_index',
                            help="Title of blog index page to attach blogs")
        parser.add_argument('username',
                            default=False,
                            help='Username for basic Auth')
        parser.add_argument('password',
                            default=False,
                            help='Password for basic Auth')
        parser.add_argument('--import-comments',
                           action='store_false',
                           default=False,
                           help="import Wordpress comments to Django Xtd")
    
    def handle(self, *args, **options):
        """gets data from WordPress site"""
        if 'username' in options:
            self.username = options['username']
        else:
            self.username = None
        if 'password' in options:
            self.password = options['password']
        else:
            self.password = None
        try:
            blog_index = BlogIndexPage.objects.get(
                title__icontains=options['blog_index'])
        except BlogIndexPage.DoesNotExist:
            raise CommandError("Have you created an index yet?")
        if options['blog_to_migrate'] == "just_testing":
            with open('test-data.json') as test_json:
                posts = json.load(test_json)
        else:
            posts = self.get_posts_data(options['blog_to_migrate'])
        if '--import-comments' in options:
            print('we are really importing comments')
        self.create_blog_pages(posts, blog_index)

    def convert_html_entities(self, text, *args, **options):
        """converts html symbols so they show up correctly in wagtail"""
        return html.unescape(text)

    def get_posts_data(self, blog, id=None, get_comments=False, *args, **options):
        self.url = blog
        headers = {
            'Content-Type': 'application/json',
            'Accept': 'application/json',
        }
        if self.username and self.password:
            auth = b64encode(
                str.encode('{}:{}'.format(self.username, self.password)))
            headers['Authorization'] = 'Basic {}'.format(auth)
        if self.url.startswith('http://'):
            base_url = self.url
        else:
            base_url = ''.join(('http://', self.url))
        posts_url = ''.join((base_url, '/wp-json/posts'))
        comments_url = ''.join((posts_url, '/%s/comments')) % id            
        if get_comments == True:
            comments_url = ''.join((posts_url, '/%s/comments')) % id
            fetched_comments = requests.get(comments_url)
            comments_data = fetched_comments.text
            comments_garbage = comments_data.split("[")[0]
            comments_data = comments_data.strip(comments_garbage)
            for bad_data in ['8db4ac', '\r\n', '\r\n0']:
                comments_data = comments_data.strip(bad_data)
            return json.loads(comments_data)
        else:
            fetched_posts = requests.get(posts_url, headers=headers)
            data = fetched_posts.text
            # I have no idea what this junk is
            garbage_data = data.split("[")[0]
            data = data.strip(garbage_data)
            for bad_data in ['8db4ac', '\r\n', '\r\n0']:
                data = data.strip(bad_data)
            return json.loads(data)

    def create_images_from_urls_in_content(self, body):
        """create Image objects and transfer image files to media root"""
        soup = BeautifulSoup(body, "html5lib")
        for img in soup.findAll('img'):
            old_url = img['src']
            if 'width' in img:
                width = img['width']
            if 'height' in img:
                height = img['height']
            else:
                width = 100
                height = 100
            path, file_ = os.path.split(img['src'])
            if not img['src']:
                continue  # Blank image
            try:
                remote_image = urllib.request.urlretrieve(img['src'])
            except urllib.error.HTTPError:
                print("Unable to import " + img['src'])
                continue
            except urllib.error.URLError:
                print("URL error - try again " + img['src'])
                continue
            image = Image(title=file_, width=width, height=height)
            image.file.save(file_, File(open(remote_image[0], 'rb')))
            image.save()
            new_url = image.file.url
            body = body.replace(old_url, new_url)
            body = self.convert_html_entities(body)
        return body

    def create_user(self, author):
        username = author['username']
        first_name = author['first_name']
        last_name = author['last_name']
        try:
            user = User.objects.get(username=username)
        except User.DoesNotExist:
            user = User.objects.create_user(
                username=username, first_name=first_name, last_name=last_name)
        return user

    def import_comments(self, post_id, slug, *args, **options):
        comments = self.get_posts_data('dev.swoonreads.com', post_id, get_comments=True)
        blog_post_type = ""
        for comment in comments:
            try:
                blog_post = BlogPage.objects.get(slug=slug)
                blog_post_type = ContentType.objects.get_for_model(blog_post)
            except BlogPage.DoesNotExist:
                print('cannot find this blog post')
                continue
            try:
                mysite = Site.objects.get_current()
                site_id = mysite.id
            except Site.DoesNotExist:
                print('site does not exist')
                continue
            comment_id = comment.get('ID')
            comment_status = comment.get('status')
            comment_parent = comment.get('parent')
            comment_text = comment.get('content')
            comment_text = self.convert_html_entities(comment_text)
            date = comment.get('date')[:10]
            status = comment.get('status')
            comment_author = comment.get('author')
            print(comment_author)
            new_comment = XtdComment.objects.get_or_create(site_id=site_id, content_type=blog_post_type, comment=comment_text, submit_date=date)[0]
            comment_parent = comment.get('parent')
            thread_level = 0
            if comment_parent is not None and comment_parent is not 0:
                while thread_level <= settings.COMMENTS_XTD_MAX_THREAD_LEVEL:
                    for parent_comment in comments:
                        if parent_comment['ID'] == comment_parent:
                            parent_comment_content = parent_comment['content']
                            grandparent = parent_comment['parent']
                            if grandparent is not None and grandparent is not 0:
                                for grandparent_comment in comments:
                                    if grandparent_comment['ID'] == grandparent:
                                        grandparent_content = grandparent_comment['content']
                                        grandparent_date = grandparent_comment['date'][:10]
                                        gp_comment = XtdComment.objects.get_or_create(site_id=site_id, content_type=blog_post_type, comment=grandparent_content, submit_date=grandparent_date)[0]
                                        gp_comment.thread_id = 2
                                        gp_comment.save()
                            parent = XtdComment.objects.get_or_create(site_id=site_id, content_type=blog_post_type, comment=parent_comment_content, submit_date=date)[0]
                            parent.thread_id = 1
                            parent.save()
                            thread_level +=1
                        else:
                            continue
            else:
                new_comment.thread_id = 0   
            if type(comment_author) is int:
                pass<commit_msg>updated blog<commit_after>from django.core.management.base import BaseCommand, CommandError
from django.core.files import File
from django.contrib.auth import get_user_model
User = get_user_model()
from django.conf import settings
from django.contrib.auth.models import User
from django_comments.models import Comment
from django_comments_xtd.models import XtdComment
from django.contrib.contenttypes.models import ContentType
from django.contrib.sites.models import Site
from base64 import b64encode
from blog.models import BlogPage
from optparse import make_option
import urllib.request
import os
import json
import requests
try:
    import html
except ImportError:  # 2.x
    import HTMLParser
    html = HTMLParser.HTMLParser()
from bs4 import BeautifulSoup
from blog.models import (BlogPage, BlogTag, BlogPageTag, BlogIndexPage,
                         BlogCategory, BlogCategoryBlogPage)
from wagtail.wagtailimages.models import Image


class Command(BaseCommand):
    """
    This is a management command to migrate a Wordpress site to Wagtail.
    Two arguments should be used - the site to be migrated and the site it is
    being migrated to.

    Users will first need to make sure the WP REST API(WP API) plugin is
    installed on the self-hosted Wordpress site to migrate.
    Next users will need to create a BlogIndex object in this GUI.
    This will be used as a parent object for the child blog page objects.
    """

    def add_arguments(self, parser):
        """have to add this to use args in django 1.8"""
        parser.add_argument('blog_to_migrate',
                            help="Base url of wordpress instance")
        parser.add_argument('blog_index',
                            help="Title of blog index page to attach blogs")
        parser.add_argument('username',
                            default=False,
                            help='Username for basic Auth')
        parser.add_argument('password',
                            default=False,
                            help='Password for basic Auth')
        parser.add_argument('--import-comments',
                           action='store_false',
                           default=False,
                           help="import Wordpress comments to Django Xtd")
    
    def handle(self, *args, **options):
        """gets data from WordPress site"""
        if 'username' in options:
            self.username = options['username']
        else:
            self.username = None
        if 'password' in options:
            self.password = options['password']
        else:
            self.password = <PASSWORD>
        try:
            blog_index = BlogIndexPage.objects.get(
                title__icontains=options['blog_index'])
        except BlogIndexPage.DoesNotExist:
            raise CommandError("Have you created an index yet?")
        if options['blog_to_migrate'] == "just_testing":
            with open('test-data.json') as test_json:
                posts = json.load(test_json)
        else:
            posts = self.get_posts_data(options['blog_to_migrate'])
        if '--import-comments' in options:
            print('we are really importing comments')
        self.create_blog_pages(posts, blog_index)

    def convert_html_entities(self, text, *args, **options):
        """converts html symbols so they show up correctly in wagtail"""
        return html.unescape(text)

    def get_posts_data(self, blog, id=None, get_comments=False, *args, **options):
        self.url = blog
        headers = {
            'Content-Type': 'application/json',
            'Accept': 'application/json',
        }
        if self.username and self.password:
            auth = b64encode(
                str.encode('{}:{}'.format(self.username, self.password)))
            headers['Authorization'] = 'Basic {}'.format(auth)
        if self.url.startswith('http://'):
            base_url = self.url
        else:
            base_url = ''.join(('http://', self.url))
        posts_url = ''.join((base_url, '/wp-json/posts'))
        comments_url = ''.join((posts_url, '/%s/comments')) % id            
        if get_comments == True:
            comments_url = ''.join((posts_url, '/%s/comments')) % id
            fetched_comments = requests.get(comments_url)
            comments_data = fetched_comments.text
            comments_garbage = comments_data.split("[")[0]
            comments_data = comments_data.strip(comments_garbage)
            for bad_data in ['8db4ac', '\r\n', '\r\n0']:
                comments_data = comments_data.strip(bad_data)
            return json.loads(comments_data)
        else:
            fetched_posts = requests.get(posts_url, headers=headers)
            data = fetched_posts.text
            # I have no idea what this junk is
            garbage_data = data.split("[")[0]
            data = data.strip(garbage_data)
            for bad_data in ['8db4ac', '\r\n', '\r\n0']:
                data = data.strip(bad_data)
            return json.loads(data)

    def create_images_from_urls_in_content(self, body):
        """create Image objects and transfer image files to media root"""
        soup = BeautifulSoup(body, "html5lib")
        for img in soup.findAll('img'):
            old_url = img['src']
            if 'width' in img:
                width = img['width']
            if 'height' in img:
                height = img['height']
            else:
                width = 100
                height = 100
            path, file_ = os.path.split(img['src'])
            if not img['src']:
                continue  # Blank image
            try:
                remote_image = urllib.request.urlretrieve(img['src'])
            except urllib.error.HTTPError:
                print("Unable to import " + img['src'])
                continue
            except urllib.error.URLError:
                print("URL error - try again " + img['src'])
                continue
            image = Image(title=file_, width=width, height=height)
            image.file.save(file_, File(open(remote_image[0], 'rb')))
            image.save()
            new_url = image.file.url
            body = body.replace(old_url, new_url)
            body = self.convert_html_entities(body)
        return body

    def create_user(self, author):
        username = author['username']
        first_name = author['first_name']
        last_name = author['last_name']
        try:
            user = User.objects.get(username=username)
        except User.DoesNotExist:
            user = User.objects.create_user(
                username=username, first_name=first_name, last_name=last_name)
        return user

    def import_comments(self, post_id, slug, *args, **options):
        comments = self.get_posts_data('dev.swoonreads.com', post_id, get_comments=True)
        blog_post_type = ""
        for comment in comments:
            try:
                blog_post = BlogPage.objects.get(slug=slug)
                blog_post_type = ContentType.objects.get_for_model(blog_post)
            except BlogPage.DoesNotExist:
                print('cannot find this blog post')
                continue
            try:
                mysite = Site.objects.get_current()
                site_id = mysite.id
            except Site.DoesNotExist:
                print('site does not exist')
                continue
            comment_id = comment.get('ID')
            comment_status = comment.get('status')
            comment_parent = comment.get('parent')
            comment_text = comment.get('content')
            comment_text = self.convert_html_entities(comment_text)
            date = comment.get('date')[:10]
            status = comment.get('status')
            comment_author = comment.get('author')
            try:
                new_comment = XtdComment.objects.get_or_create(site_id=site_id, content_type=blog_post_type, comment=comment_text, submit_date=date)[0]
            except ValueError:
                continue
            comment_parent = comment.get('parent')
            thread_level = 0
            if comment_parent is not None and comment_parent is not 0:
                while thread_level <= settings.COMMENTS_XTD_MAX_THREAD_LEVEL:
                    for parent_comment in comments:
                        if parent_comment['ID'] == comment_parent:
                            parent_comment_content = parent_comment['content']
                            grandparent = parent_comment['parent']
                            if grandparent is not None and grandparent is not 0:
                                for grandparent_comment in comments:
                                    if grandparent_comment['ID'] == grandparent:
                                        grandparent_content = grandparent_comment['content']
                                        grandparent_date = grandparent_comment['date'][:10]
                                        gp_comment = XtdComment.objects.get_or_create(site_id=site_id, content_type=blog_post_type, comment=grandparent_content, submit_date=grandparent_date)[0]
                                        gp_comment.thread_id = 2
                                        gp_comment.save()
                            parent = XtdComment.objects.get_or_create(site_id=site_id, content_type=blog_post_type, comment=parent_comment_content, submit_date=date)[0]
                            parent.thread_id = 1
                            parent.save()
                            thread_level +=1
                        else:
                            continue
            else:
                new_comment.thread_id = 0   
            if type(comment_author) is int:
                pass
{'fraction_non_alphanumeric': 0.08995254796781514, 'fraction_numerical': 0.004126263668248401, 'mean_word_length': 1.8576271186440678, 'pattern_counts': {'":': 2, '<': 6, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5318082', 'n_tokens_mistral': 5079, 'n_tokens_neox': 4953, 'n_words': 1265}