<commit_before># encoding: utf-8
import datetime
from south.db import db
from south.v2 import DataMigration
from django.db import models
import json

def synchronize_properties_cache(profile):
    result = {}
    for prop in profile["properties"]:
        type = "text"
        for tag in prop["tags"]:
            if tag.startswith("property:type=") and not tag=="property:type=shredded_list":
                type = tag[len("property:type="):]

        result[prop["property"]] = {
            "label": prop["label"],
            "valueType": type

        }
    return {"properties": result}

class Migration(DataMigration):

    def forwards(self, orm):
        "Write your forwards methods here."
        for ds in orm["dataset.Dataset"].objects.all():
            p = json.loads(ds.profile)
            for prop in p["properties"]:
                inx = 0
                while inx < len(prop["tags"]):
                    if prop["tags"][inx] == "property:type=datetime":
                        prop["tags"][inx] = "property:type=date"
                    inx += 1
            ds.profile = json.dumps(p)
            ds.properties_cache = json.dumps(synchronize_properties_cache(json.loads(ds.profile)))
            ds.save()




    def backwards(self, orm):
        for ds in orm["dataset.Dataset"].objects.all():
            p = json.loads(ds.profile)
            for prop in p["properties"]:
                inx = 0
                while inx < len(prop["tags"]):
                    if prop["tags"][inx] == "property:type=date":
                        prop["tags"][inx] = "property:type=datetime"
                    inx += 1<commit_msg>Massaging migrations to run on mysql<commit_after># encoding: utf-8
import datetime
from south.db import db
from south.v2 import DataMigration
from django.db import models
import json

def synchronize_properties_cache(profile):
    result = {}
    for prop in profile["properties"]:
        type = "text"
        for tag in prop.get("tags", []):
            if tag.startswith("property:type=") and not tag=="property:type=shredded_list":
                type = tag[len("property:type="):]

        result[prop["property"]] = {
            "label": prop["label"],
            "valueType": type

        }
    return {"properties": result}

class Migration(DataMigration):

    def forwards(self, orm):
        "Write your forwards methods here."
        for ds in orm["dataset.Dataset"].objects.all():
            p = json.loads(ds.profile)
            for prop in p["properties"]:
                inx = 0
                while inx < len(prop.get("tags", [])):
                    if prop["tags"][inx] == "property:type=datetime":
                        prop["tags"][inx] = "property:type=date"
                    inx += 1
            ds.profile = json.dumps(p)
            ds.properties_cache = json.dumps(synchronize_properties_cache(json.loads(ds.profile)))
            ds.save()




    def backwards(self, orm):
        for ds in orm["dataset.Dataset"].objects.all():
            p = json.loads(ds.profile)
            for prop in p["properties"]:
                inx = 0
                while inx < len(prop.get("tags", [])):
                    if prop["tags"][inx] == "property:type=date":
                        prop["tags"][inx] = "property:type=datetime"
                    inx += 1
{'fraction_non_alphanumeric': 0.1351270553064275, 'fraction_numerical': 0.003587443946188341, 'mean_word_length': 1.9299474605954465, 'pattern_counts': {'":': 12, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1884377', 'n_tokens_mistral': 964, 'n_tokens_neox': 920, 'n_words': 222}