# Path: website/models.py
# class Annotation(models.Model):
#     """
#     Each record is a natural language <-> code translation annotated by a
#     programmer.
#     """
#     url = models.ForeignKey(URL, on_delete=models.CASCADE)
#     nl = models.ForeignKey(NL, on_delete=models.CASCADE)
#     cmd = models.ForeignKey(Command, on_delete=models.CASCADE)
#     annotator = models.ForeignKey(User, on_delete=models.CASCADE)
#     submission_time = models.DateTimeField(default=timezone.now)
# 
# class AnnotationUpdate(models.Model):
#     """
#     Each record is an update of an annotation submitted by a judger.
#     """
#     annotation = models.ForeignKey(Annotation, on_delete=models.CASCADE)
#     judger = models.ForeignKey(User, on_delete=models.CASCADE)
#     update_str = models.TextField()
#     update_type = models.TextField(default='nl')
#     comment = models.ForeignKey(Comment, on_delete=models.CASCADE)
#     submission_time = models.DateTimeField(default=timezone.now)
#     status = models.TextField(default='open')
# 
# class Command(models.Model):
#     """
#     Command line.
#     """
#     str = models.TextField(primary_key=True)
#     template = models.TextField(default='')
#     language = models.TextField(default='bash')
#     tags = models.ManyToManyField('Tag')
# 
# class Notification(models.Model):
#     """
#     Each record is a certain type of message issued from one annotator to
#     another.
#     """
#     sender = models.ForeignKey(User, related_name='notification_sender',
#                                on_delete=models.CASCADE)
#     receiver = models.ForeignKey(User, related_name='notification_receiver',
#                                  on_delete=models.CASCADE)
#     type = models.TextField(default='comment')
#     annotation_update = models.ForeignKey('AnnotationUpdate', null=True,
#                                           on_delete=models.CASCADE)
#     comment = models.ForeignKey('Comment', null=True, on_delete=models.CASCADE)
#     url = models.ForeignKey('URL', on_delete=models.CASCADE)
#     creation_time = models.DateTimeField(default=timezone.now)
#     status = models.TextField(default='issued')
# 
# class URL(models.Model):
#     """
#     URL.
# 
#     :member str: url address.
#     :member html_content: snapshot of the URL content at the time of annotation.
#     :member commands: commands in the URL (automatically extracted)
#     :member tags: tags of the URL (assigned based on user annotations)
#     """
#     str = models.TextField(primary_key=True)
#     html_content = models.TextField(default='')
#     commands = models.ManyToManyField('Command')
#     tags = models.ManyToManyField('Tag')
# 
# class URLTag(models.Model):
#     """
#     Each record stores a (url, tag) pair.
#     """
#     url = models.ForeignKey(URL, on_delete=models.CASCADE)
#     tag = models.TextField()
# 
# Path: website/utils.py
# def get_tag(tag_str):
#     tag, _ = Tag.objects.get_or_create(str=tag_str.strip())
#     return tag
# 
# def get_command(command_str):
#     command_str=command_str.strip()
#     if Command.objects.filter(str=command_str).exists():
#         cmd = Command.objects.get(str=command_str)
#     else:
#         cmd = Command.objects.create(str=command_str)
#         ast = data_tools.bash_parser(command_str)
#         for utility in data_tools.get_utilities(ast):
#             cmd.tags.add(get_tag(utility))    
#         template = data_tools.ast2template(
# 	        ast, loose_constraints=True)
#         cmd.template = template
#         cmd.save()
#     return cmd
# 
# def get_url(url_str):
#     url_str = url_str.strip()
#     try:
#         url = URL.objects.get(str=url_str)
#     except ObjectDoesNotExist:
#         html = extract_html(url_str)
#         if html is None:
#             url = URL.objects.create(str=url_str)
#         else:
#             url = URL.objects.create(str=url_str, html_content=html)
#     return url


import html
import os, sys
import pickle
import re
import sqlite3
from website.models import Annotation, AnnotationUpdate, Command, \
    Notification, URL, URLTag
from website.utils import get_tag, get_command, get_url
from bashlint import data_tools

def extract_code(text):
    for match in CODE_REGEX.findall(text):
        if match.strip():
            yield html.unescape(match.replace("<br>", "\n"))

def extract_oneliners_from_code(code_block):
    for cmd in code_block.splitlines():
        if cmd.startswith('$ '):
            cmd = cmd[2:]
        if cmd.startswith('# '):
            cmd = cmd[2:]
        comment = re.search(r'\s+#\s+', cmd)
        if comment:
            old_cmd = cmd
            cmd = cmd[:comment.start()]
            print('Remove comment: {} -> {}'.format(old_cmd, cmd))
        cmd = cmd.strip()

        # discard code block opening line
        if cmd and not cmd[-1] in ['{', '[', '(']:
            yield cmd

def load_urls(input_file_path):
    with open(input_file_path, 'rb') as f:
        urls_by_utility = pickle.load(f)

    for utility in urls_by_utility:
        for url in urls_by_utility[utility]:
            if not URLTag.objects.filter(url__str=url, tag=utility):

# Path: website/models.py
# class Annotation(models.Model):
#     """
#     Each record is a natural language <-> code translation annotated by a
#     programmer.
#     """
#     url = models.ForeignKey(URL, on_delete=models.CASCADE)
#     nl = models.ForeignKey(NL, on_delete=models.CASCADE)
#     cmd = models.ForeignKey(Command, on_delete=models.CASCADE)
#     annotator = models.ForeignKey(User, on_delete=models.CASCADE)
#     submission_time = models.DateTimeField(default=timezone.now)
# 
# class AnnotationUpdate(models.Model):
#     """
#     Each record is an update of an annotation submitted by a judger.
#     """
#     annotation = models.ForeignKey(Annotation, on_delete=models.CASCADE)
#     judger = models.ForeignKey(User, on_delete=models.CASCADE)
#     update_str = models.TextField()
#     update_type = models.TextField(default='nl')
#     comment = models.ForeignKey(Comment, on_delete=models.CASCADE)
#     submission_time = models.DateTimeField(default=timezone.now)
#     status = models.TextField(default='open')
# 
# class Command(models.Model):
#     """
#     Command line.
#     """
#     str = models.TextField(primary_key=True)
#     template = models.TextField(default='')
#     language = models.TextField(default='bash')
#     tags = models.ManyToManyField('Tag')
# 
# class Notification(models.Model):
#     """
#     Each record is a certain type of message issued from one annotator to
#     another.
#     """
#     sender = models.ForeignKey(User, related_name='notification_sender',
#                                on_delete=models.CASCADE)
#     receiver = models.ForeignKey(User, related_name='notification_receiver',
#                                  on_delete=models.CASCADE)
#     type = models.TextField(default='comment')
#     annotation_update = models.ForeignKey('AnnotationUpdate', null=True,
#                                           on_delete=models.CASCADE)
#     comment = models.ForeignKey('Comment', null=True, on_delete=models.CASCADE)
#     url = models.ForeignKey('URL', on_delete=models.CASCADE)
#     creation_time = models.DateTimeField(default=timezone.now)
#     status = models.TextField(default='issued')
# 
# class URL(models.Model):
#     """
#     URL.
# 
#     :member str: url address.
#     :member html_content: snapshot of the URL content at the time of annotation.
#     :member commands: commands in the URL (automatically extracted)
#     :member tags: tags of the URL (assigned based on user annotations)
#     """
#     str = models.TextField(primary_key=True)
#     html_content = models.TextField(default='')
#     commands = models.ManyToManyField('Command')
#     tags = models.ManyToManyField('Tag')
# 
# class URLTag(models.Model):
#     """
#     Each record stores a (url, tag) pair.
#     """
#     url = models.ForeignKey(URL, on_delete=models.CASCADE)
#     tag = models.TextField()
# 
# Path: website/utils.py
# def get_tag(tag_str):
#     tag, _ = Tag.objects.get_or_create(str=tag_str.strip())
#     return tag
# 
# def get_command(command_str):
#     command_str=command_str.strip()
#     if Command.objects.filter(str=command_str).exists():
#         cmd = Command.objects.get(str=command_str)
#     else:
#         cmd = Command.objects.create(str=command_str)
#         ast = data_tools.bash_parser(command_str)
#         for utility in data_tools.get_utilities(ast):
#             cmd.tags.add(get_tag(utility))    
#         template = data_tools.ast2template(
# 	        ast, loose_constraints=True)
#         cmd.template = template
#         cmd.save()
#     return cmd
# 
# def get_url(url_str):
#     url_str = url_str.strip()
#     try:
#         url = URL.objects.get(str=url_str)
#     except ObjectDoesNotExist:
#         html = extract_html(url_str)
#         if html is None:
#             url = URL.objects.create(str=url_str)
#         else:
#             url = URL.objects.create(str=url_str, html_content=html)
#     return url


# Path: website/scripts/db_changes.py
import html
import os, sys
import pickle
import re
import sqlite3
from website.models import Annotation, AnnotationUpdate, Command, \
    Notification, URL, URLTag
from website.utils import get_tag, get_command, get_url
from bashlint import data_tools


def extract_code(text):
    for match in CODE_REGEX.findall(text):
        if match.strip():
            yield html.unescape(match.replace("<br>", "\n"))

def extract_oneliners_from_code(code_block):
    for cmd in code_block.splitlines():
        if cmd.startswith('$ '):
            cmd = cmd[2:]
        if cmd.startswith('# '):
            cmd = cmd[2:]
        comment = re.search(r'\s+#\s+', cmd)
        if comment:
            old_cmd = cmd
            cmd = cmd[:comment.start()]
            print('Remove comment: {} -> {}'.format(old_cmd, cmd))
        cmd = cmd.strip()

        # discard code block opening line
        if cmd and not cmd[-1] in ['{', '[', '(']:
            yield cmd

def load_urls(input_file_path):
    with open(input_file_path, 'rb') as f:
        urls_by_utility = pickle.load(f)

    for utility in urls_by_utility:
        for url in urls_by_utility[utility]:
            if not URLTag.objects.filter(url__str=url, tag=utility):

                URLTag.objects.create(url=get_url(url), tag=utility)