<commit_before>"""
jobs.py - base notification routines

Author 
    <NAME> <<EMAIL>>

License
    Copyright (c) 2010 Massachusetts Institute of Technology.
    MIT License (cf. MIT-LICENSE.txt or http://www.opensource.org/licenses/mit-license.php)
"""
import sys,os
import datetime
if "." not in sys.path: 
    sys.path.append(".")
if "DJANGO_SETTINGS_MODULE" not in os.environ or __name__=="__main__": 
    os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
from django.conf import settings
import base.utils as utils, base.models as M
from django.template.loader import render_to_string
from django.core.mail import EmailMessage
from django.db.models import Max
from django.db.models.deletion import Collector
from django.db.utils import IntegrityError

VISIBILITY = {1: "Myself", 2: "Staff", 3: "Class"}


def extract_obj(o, from_class, cut_at):
    #inspired from from  http://stackoverflow.com/a/2315053/768104
    print "pulling objects related to %s" % (o,)
    links = [rel.get_accessor_name() for rel in o._meta.get_all_related_objects()]
    for link in links:
        rel_objects = getattr(o, link).all()
        for ro in rel_objects:
            classname = ro.__class__.__name__
            if classname not in extracted: 
                extracted[classname]={}
            if ro.id not in extracted[classname]:
                extracted[classname][ro.id]=1 
                extract_obj(ro, classname, cut_at)
from django.db.models.fields.related import ForeignKey
def duplicate(objs, using_src, using_dest, special_handlers):
    #adapted from http://stackoverflow.com/a/6064096/768104    
    collector = Collector(using_src)
    collector.collect(objs)
    collector.sort()
    related_models = collector.data.keys()
    duplicate_order = reversed(related_models)
    extracted = {}
    for model in duplicate_order:
        # Find all FKs on model that point to a related_model.
        fks = []
        for f in model._meta.fields:
            if isinstance(f, ForeignKey) and f.rel.to not in related_models:
                fks.append(f)
        # Replace each `sub_obj` with a duplicate.
        if model not in collector.data:
            continue
        sub_objects = collector.data[model]
        for obj in sub_objects:
            for fk in fks:
                rel_obj = getattr(obj, fk.name)
                rel_cls = rel_obj.__class__
                if rel_cls not in extracted:
                    extracted[rel_cls]={}
                if rel_obj is not None and rel_obj.id not in extracted[rel_cls]: 
                    extracted[rel_cls][rel_obj.id]=True
                    rel_obj.save(using=using_dest)
                    #print "-> saved related object %s" % (rel_obj,)
            #now ready to insert obj: 
            if model not in extracted:
                extracted[model]={}
            if obj is not None and obj.id not in extracted[model]: 
                extracted[model][obj.id]=True
                try: 
                    obj.save(using=using_dest)            
                except IntegrityError as e: 
                    if model in special_handlers: 
                        special_handlers[model](obj, using_dest)
                        obj.save(using=using_dest)
                        
                    else: 
                        raise e
                #print "saved obj %s" %(obj,)
        print "%s done TOTAL objects written: %s " % (model.__name__, sum([len(extracted[i]) for i in extracted]))        <commit_msg>progress on export a group<commit_after>"""
jobs.py - base notification routines

Author 
    <NAME> <<EMAIL>>

License
    Copyright (c) 2010 Massachusetts Institute of Technology.
    MIT License (cf. MIT-LICENSE.txt or http://www.opensource.org/licenses/mit-license.php)
"""
import sys,os
import datetime
if "." not in sys.path: 
    sys.path.append(".")
if "DJANGO_SETTINGS_MODULE" not in os.environ or __name__=="__main__": 
    os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
from django.conf import settings
import base.utils as utils, base.models as M
from django.template.loader import render_to_string
from django.core.mail import EmailMessage
from django.db.models import Max
from django.db.models.deletion import Collector
from django.db.utils import IntegrityError

VISIBILITY = {1: "Myself", 2: "Staff", 3: "Class"}

pending_inserts = []
def extract_obj(o, from_class, cut_at):
    #inspired from from  http://stackoverflow.com/a/2315053/768104
    extracted = {}
    print "pulling objects related to %s" % (o,)
    links = [rel.get_accessor_name() for rel in o._meta.get_all_related_objects()]
    for link in links:
        rel_objects = getattr(o, link).all()
        for ro in rel_objects:
            classname = ro.__class__.__name__
            if classname not in extracted: 
                extracted[classname]={}
            if ro.id not in extracted[classname]:
                extracted[classname][ro.id]=1 
                extract_obj(ro, classname, cut_at)
from django.db.models.fields.related import ForeignKey
def duplicate(objs, using_src, using_dest, special_handlers):
    #adapted from http://stackoverflow.com/a/6064096/768104    
    collector = Collector(using_src)
    collector.collect(objs)
    collector.sort()
    related_models = collector.data.keys()
    duplicate_order = reversed(related_models)
    extracted = {}
    for model in duplicate_order:
        # Find all FKs on model that point to a related_model.
        fks = []
        for f in model._meta.fields:
            if isinstance(f, ForeignKey) and f.rel.to not in related_models:
                fks.append(f)
        # Replace each `sub_obj` with a duplicate.
        if model not in collector.data:
            continue
        sub_objects = collector.data[model]
        for obj in sub_objects:
            for fk in fks:
                rel_obj = getattr(obj, fk.name)
                rel_cls = rel_obj.__class__
                if rel_cls not in extracted:
                    extracted[rel_cls]={}
                if rel_obj is not None and rel_obj.id not in extracted[rel_cls]: 
                    extracted[rel_cls][rel_obj.id]=True
                    rel_obj.save(using=using_dest)
                    #print "-> saved related object %s" % (rel_obj,)
            #now ready to insert obj: 
            if model not in extracted:
                extracted[model]={}
            if obj is not None and obj.id not in extracted[model]: 
                extracted[model][obj.id]=True
                try: 
                    obj.save(using=using_dest)            
                except IntegrityError as e: 
                    pending_inserts.append(obj)
        print "%s done TOTAL objects written: %s " % (model.__name__, sum([len(extracted[i]) for i in extracted]))        
    do_pending_inserts(using_dest)

def do_pending_inserts(using):
    global pending_inserts
    new_pending = []
    for o in pending_inserts: 
        try: 
            o.save(using=using)
        except IntegrityError as e: 
            new_pending.append(o)
                            
{'fraction_non_alphanumeric': 0.09143502187103146, 'fraction_numerical': 0.009595033159305771, 'mean_word_length': 2.1628737170905845, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6671896', 'n_tokens_mistral': 2145, 'n_tokens_neox': 2020, 'n_words': 575}