# Path: website/models.py
# class Annotation(models.Model):
#     """
#     Each record is a natural language <-> code translation annotated by a
#     programmer.
#     """
#     url = models.ForeignKey(URL, on_delete=models.CASCADE)
#     nl = models.ForeignKey(NL, on_delete=models.CASCADE)
#     cmd = models.ForeignKey(Command, on_delete=models.CASCADE)
#     annotator = models.ForeignKey(User, on_delete=models.CASCADE)
#     submission_time = models.DateTimeField(default=timezone.now)
# 
# class AnnotationUpdate(models.Model):
#     """
#     Each record is an update of an annotation submitted by a judger.
#     """
#     annotation = models.ForeignKey(Annotation, on_delete=models.CASCADE)
#     judger = models.ForeignKey(User, on_delete=models.CASCADE)
#     update_str = models.TextField()
#     update_type = models.TextField(default='nl')
#     comment = models.ForeignKey(Comment, on_delete=models.CASCADE)
#     submission_time = models.DateTimeField(default=timezone.now)
#     status = models.TextField(default='open')
# 
# class Command(models.Model):
#     """
#     Command line.
#     """
#     str = models.TextField(primary_key=True)
#     template = models.TextField(default='')
#     language = models.TextField(default='bash')
#     tags = models.ManyToManyField('Tag')
# 
# class Notification(models.Model):
#     """
#     Each record is a certain type of message issued from one annotator to
#     another.
#     """
#     sender = models.ForeignKey(User, related_name='notification_sender',
#                                on_delete=models.CASCADE)
#     receiver = models.ForeignKey(User, related_name='notification_receiver',
#                                  on_delete=models.CASCADE)
#     type = models.TextField(default='comment')
#     annotation_update = models.ForeignKey('AnnotationUpdate', null=True,
#                                           on_delete=models.CASCADE)
#     comment = models.ForeignKey('Comment', null=True, on_delete=models.CASCADE)
#     url = models.ForeignKey('URL', on_delete=models.CASCADE)
#     creation_time = models.DateTimeField(default=timezone.now)
#     status = models.TextField(default='issued')
# 
# class URL(models.Model):
#     """
#     URL.
# 
#     :member str: url address.
#     :member html_content: snapshot of the URL content at the time of annotation.
#     :member commands: commands in the URL (automatically extracted)
#     :member tags: tags of the URL (assigned based on user annotations)
#     """
#     str = models.TextField(primary_key=True)
#     html_content = models.TextField(default='')
#     commands = models.ManyToManyField('Command')
#     tags = models.ManyToManyField('Tag')
# 
# class URLTag(models.Model):
#     """
#     Each record stores a (url, tag) pair.
#     """
#     url = models.ForeignKey(URL, on_delete=models.CASCADE)
#     tag = models.TextField()
# 
# Path: website/utils.py
# def get_tag(tag_str):
#     tag, _ = Tag.objects.get_or_create(str=tag_str.strip())
#     return tag
# 
# def get_command(command_str):
#     command_str=command_str.strip()
#     if Command.objects.filter(str=command_str).exists():
#         cmd = Command.objects.get(str=command_str)
#     else:
#         cmd = Command.objects.create(str=command_str)
#         ast = data_tools.bash_parser(command_str)
#         for utility in data_tools.get_utilities(ast):
#             cmd.tags.add(get_tag(utility))    
#         template = data_tools.ast2template(
# 	        ast, loose_constraints=True)
#         cmd.template = template
#         cmd.save()
#     return cmd
# 
# def get_url(url_str):
#     url_str = url_str.strip()
#     try:
#         url = URL.objects.get(str=url_str)
#     except ObjectDoesNotExist:
#         html = extract_html(url_str)
#         if html is None:
#             url = URL.objects.create(str=url_str)
#         else:
#             url = URL.objects.create(str=url_str, html_content=html)
#     return url


import html
import os, sys
import pickle
import re
import sqlite3
from website.models import Annotation, AnnotationUpdate, Command, \
    Notification, URL, URLTag
from website.utils import get_tag, get_command, get_url
from bashlint import data_tools
            print(cmd.str)
            ast = data_tools.bash_parser(cmd.str)
            for utility in data_tools.get_utilities(ast):
                print(utility)
                cmd.tags.add(get_tag(utility))
            cmd.save()

def populate_command_template():
    for cmd in Command.objects.all():
        if len(cmd.str) > 600:
            cmd.delete()
        else:
            ast = data_tools.bash_parser(cmd.str)
            template = data_tools.ast2template(ast, loose_constraints=True)
            cmd.template = template
            cmd.save()

def populate_tag_commands():
    for tag in Tag.objects.all():
        tag.commands.clear()
    for url_tag in URLTag.objects.all():
        print(url_tag.url.str)
        tag = get_tag(url_tag.tag)
        for cmd in url_tag.url.commands.all():
            if tag in cmd.tags.all():
                tag.commands.add(cmd)
        tag.save()

def populate_url_tags():
    for url in URL.objects.all():

# Path: website/models.py
# class Annotation(models.Model):
#     """
#     Each record is a natural language <-> code translation annotated by a
#     programmer.
#     """
#     url = models.ForeignKey(URL, on_delete=models.CASCADE)
#     nl = models.ForeignKey(NL, on_delete=models.CASCADE)
#     cmd = models.ForeignKey(Command, on_delete=models.CASCADE)
#     annotator = models.ForeignKey(User, on_delete=models.CASCADE)
#     submission_time = models.DateTimeField(default=timezone.now)
# 
# class AnnotationUpdate(models.Model):
#     """
#     Each record is an update of an annotation submitted by a judger.
#     """
#     annotation = models.ForeignKey(Annotation, on_delete=models.CASCADE)
#     judger = models.ForeignKey(User, on_delete=models.CASCADE)
#     update_str = models.TextField()
#     update_type = models.TextField(default='nl')
#     comment = models.ForeignKey(Comment, on_delete=models.CASCADE)
#     submission_time = models.DateTimeField(default=timezone.now)
#     status = models.TextField(default='open')
# 
# class Command(models.Model):
#     """
#     Command line.
#     """
#     str = models.TextField(primary_key=True)
#     template = models.TextField(default='')
#     language = models.TextField(default='bash')
#     tags = models.ManyToManyField('Tag')
# 
# class Notification(models.Model):
#     """
#     Each record is a certain type of message issued from one annotator to
#     another.
#     """
#     sender = models.ForeignKey(User, related_name='notification_sender',
#                                on_delete=models.CASCADE)
#     receiver = models.ForeignKey(User, related_name='notification_receiver',
#                                  on_delete=models.CASCADE)
#     type = models.TextField(default='comment')
#     annotation_update = models.ForeignKey('AnnotationUpdate', null=True,
#                                           on_delete=models.CASCADE)
#     comment = models.ForeignKey('Comment', null=True, on_delete=models.CASCADE)
#     url = models.ForeignKey('URL', on_delete=models.CASCADE)
#     creation_time = models.DateTimeField(default=timezone.now)
#     status = models.TextField(default='issued')
# 
# class URL(models.Model):
#     """
#     URL.
# 
#     :member str: url address.
#     :member html_content: snapshot of the URL content at the time of annotation.
#     :member commands: commands in the URL (automatically extracted)
#     :member tags: tags of the URL (assigned based on user annotations)
#     """
#     str = models.TextField(primary_key=True)
#     html_content = models.TextField(default='')
#     commands = models.ManyToManyField('Command')
#     tags = models.ManyToManyField('Tag')
# 
# class URLTag(models.Model):
#     """
#     Each record stores a (url, tag) pair.
#     """
#     url = models.ForeignKey(URL, on_delete=models.CASCADE)
#     tag = models.TextField()
# 
# Path: website/utils.py
# def get_tag(tag_str):
#     tag, _ = Tag.objects.get_or_create(str=tag_str.strip())
#     return tag
# 
# def get_command(command_str):
#     command_str=command_str.strip()
#     if Command.objects.filter(str=command_str).exists():
#         cmd = Command.objects.get(str=command_str)
#     else:
#         cmd = Command.objects.create(str=command_str)
#         ast = data_tools.bash_parser(command_str)
#         for utility in data_tools.get_utilities(ast):
#             cmd.tags.add(get_tag(utility))    
#         template = data_tools.ast2template(
# 	        ast, loose_constraints=True)
#         cmd.template = template
#         cmd.save()
#     return cmd
# 
# def get_url(url_str):
#     url_str = url_str.strip()
#     try:
#         url = URL.objects.get(str=url_str)
#     except ObjectDoesNotExist:
#         html = extract_html(url_str)
#         if html is None:
#             url = URL.objects.create(str=url_str)
#         else:
#             url = URL.objects.create(str=url_str, html_content=html)
#     return url


# Path: website/scripts/db_changes.py
import html
import os, sys
import pickle
import re
import sqlite3
from website.models import Annotation, AnnotationUpdate, Command, \
    Notification, URL, URLTag
from website.utils import get_tag, get_command, get_url
from bashlint import data_tools

            print(cmd.str)
            ast = data_tools.bash_parser(cmd.str)
            for utility in data_tools.get_utilities(ast):
                print(utility)
                cmd.tags.add(get_tag(utility))
            cmd.save()

def populate_command_template():
    for cmd in Command.objects.all():
        if len(cmd.str) > 600:
            cmd.delete()
        else:
            ast = data_tools.bash_parser(cmd.str)
            template = data_tools.ast2template(ast, loose_constraints=True)
            cmd.template = template
            cmd.save()

def populate_tag_commands():
    for tag in Tag.objects.all():
        tag.commands.clear()
    for url_tag in URLTag.objects.all():
        print(url_tag.url.str)
        tag = get_tag(url_tag.tag)
        for cmd in url_tag.url.commands.all():
            if tag in cmd.tags.all():
                tag.commands.add(cmd)
        tag.save()

def populate_url_tags():
    for url in URL.objects.all():

        for annotation in Annotation.objects.filter(url=url):