<commit_before>import logging
import datetime
import json
from urlparse import urlparse

from google.appengine.ext import ndb
from google.appengine.ext import deferred
from google.appengine.api import memcache

from gae_django.auth.models import User
from july import settings
from july.live.models import Message

class Commit(ndb.Model):
    """
    Commit record for the profile, the parent is the profile
    that way we can update the commit count and last commit timestamp
    in the same transaction.
    """
    hash = ndb.StringProperty()
    author = ndb.StringProperty(indexed=False)
    name = ndb.StringProperty(indexed=False)
    email = ndb.StringProperty()
    message = ndb.StringProperty(indexed=False)
    url = ndb.StringProperty()
    project = ndb.StringProperty()
    project_slug = ndb.ComputedProperty(lambda self: Project.parse_project_name(self.project))
    timestamp = ndb.DateTimeProperty()
    created_on = ndb.DateTimeProperty(auto_now_add=True)
    
    @classmethod
    def make_key(cls, commit_hash, user=None):
        """
        Return a ndb.Key for this commit, if the user is passed
        make the user the parent of the commit.
        """
        if user and isinstance(user, ndb.Model):
            return ndb.Key('User', user.key.id(), 'Commit', commit_hash)
        elif user and isinstance(user, ndb.Key):
            return ndb.Key('User', user.id(), 'Commit', commit_hash)
        else:
            return ndb.Key('Commit', commit_hash)
    
    @classmethod
    def create_by_email(cls, email, commits, project=None):
        """Create a commit by email address"""
        return cls.create_by_auth_id('email:%s' % email, commits, project=project)
    
    @classmethod
    def create_by_auth_id(cls, auth_id, commits, project=None):
        user = User.get_by_auth_id(auth_id)
        if user:
            return cls.create_by_user(user, commits, project=project)
        return cls.create_orphan(commits, project=project)
    
    @classmethod
    def create_by_user(cls, user, commits, project=None):
        """Create a commit with parent user, updating users points."""
        if not isinstance(commits, (list, tuple)):
            commits = [commits]
        user_key = user.key
        username = 'own:%s' % user.username
        project = project
        logging.info(project)
        
        @ndb.transactional
        def txn():
            to_put = []
            for c in commits:
                commit_hash = c.get('hash')
                if commit_hash is None:
                    logging.info("Commit hash missing in create.")
                    continue
                commit_key = cls.make_key(commit_hash, user=user_key)
                commit = commit_key.get()
                if commit is None:
                    c['project'] = project
                    commit = Commit(key=commit_key, **c)
                    to_put.append(commit)
                    
            
            # Check if there are no new commits and return
            if len(to_put) == 0:
                return []
            
            user = user_key.get()
            count = getattr(user, 'total', 0)
            
            if project is not None:
                # get the list of existing projects and make it a set 
                # to filter uniques, if this project is new add it and 
                # update the users total points.
                projects = set(getattr(user, 'projects', []))
                if project not in projects:
                    logging.info('Adding project to user: %s', user.username)
                    projects.add(project)
                    count += 10
                    user.projects = list(projects)

            points = len(to_put)
            user.total = count + points
            to_put.append(user)
            keys = ndb.put_multi(to_put)
            return keys
        
        commits = txn()
        committed = filter(lambda x: x.kind() == 'Commit', commits)
        
        # defer updating the users location if they have one.
        if user.location:
            logging.info("deferring add points to location: %s", user.location_slug)
            deferred.defer(Location.add_points, user.location_slug, len(committed), project)
            
        if getattr(user, 'team_slug', None):
            logging.info("deferring add points to team: %s", user.team_slug)
            deferred.defer(Team.add_points, user.team_slug, len(committed), project)
        
        # Tally up some points, we do this here after the transaction
        # was successful, since the timestamps could be different days
        # we need to pull each one and then record it :( 
        for commit in committed:
            # Mark the commits in the history
            deferred.defer(Accumulator.add_count, username, commit.urlsafe())
            deferred.defer(Accumulator.add_count, 'global', commit.urlsafe())
            deferred.defer(Message.add_commit, commit.urlsafe(), _countdown=15, _queue="live")
        
        return committed
    
    @classmethod
    def create_orphan(cls, commits, project=None):
        """Create a commit with no parent."""
        if not isinstance(commits, (list, tuple)):
            commits = [commits]
        
        to_put = []
        for c in commits:
            commit_hash = c.get('hash')
            if commit_hash is None:
                logging.info("Commit hash missing in create.")
                continue
            commit_key = cls.make_key(commit_hash)
            commit = commit_key.get()
            if commit is None:
                c['project'] = project
                commit = Commit(key=commit_key, **c)
                to_put.append(commit)
                # Defer adding points to global counter 
                deferred.defer(Accumulator.add_count, 'global', c['timestamp'])
        
        commits = ndb.put_multi(to_put)
        return commits
    
class Project(ndb.Model):
    """<commit_msg>Wait less time to send message<commit_after>import logging
import datetime
import json
from urlparse import urlparse

from google.appengine.ext import ndb
from google.appengine.ext import deferred
from google.appengine.api import memcache

from gae_django.auth.models import User
from july import settings
from july.live.models import Message

class Commit(ndb.Model):
    """
    Commit record for the profile, the parent is the profile
    that way we can update the commit count and last commit timestamp
    in the same transaction.
    """
    hash = ndb.StringProperty()
    author = ndb.StringProperty(indexed=False)
    name = ndb.StringProperty(indexed=False)
    email = ndb.StringProperty()
    message = ndb.StringProperty(indexed=False)
    url = ndb.StringProperty()
    project = ndb.StringProperty()
    project_slug = ndb.ComputedProperty(lambda self: Project.parse_project_name(self.project))
    timestamp = ndb.DateTimeProperty()
    created_on = ndb.DateTimeProperty(auto_now_add=True)
    
    @classmethod
    def make_key(cls, commit_hash, user=None):
        """
        Return a ndb.Key for this commit, if the user is passed
        make the user the parent of the commit.
        """
        if user and isinstance(user, ndb.Model):
            return ndb.Key('User', user.key.id(), 'Commit', commit_hash)
        elif user and isinstance(user, ndb.Key):
            return ndb.Key('User', user.id(), 'Commit', commit_hash)
        else:
            return ndb.Key('Commit', commit_hash)
    
    @classmethod
    def create_by_email(cls, email, commits, project=None):
        """Create a commit by email address"""
        return cls.create_by_auth_id('email:%s' % email, commits, project=project)
    
    @classmethod
    def create_by_auth_id(cls, auth_id, commits, project=None):
        user = User.get_by_auth_id(auth_id)
        if user:
            return cls.create_by_user(user, commits, project=project)
        return cls.create_orphan(commits, project=project)
    
    @classmethod
    def create_by_user(cls, user, commits, project=None):
        """Create a commit with parent user, updating users points."""
        if not isinstance(commits, (list, tuple)):
            commits = [commits]
        user_key = user.key
        username = 'own:%s' % user.username
        project = project
        logging.info(project)
        
        @ndb.transactional
        def txn():
            to_put = []
            for c in commits:
                commit_hash = c.get('hash')
                if commit_hash is None:
                    logging.info("Commit hash missing in create.")
                    continue
                commit_key = cls.make_key(commit_hash, user=user_key)
                commit = commit_key.get()
                if commit is None:
                    c['project'] = project
                    commit = Commit(key=commit_key, **c)
                    to_put.append(commit)
                    
            
            # Check if there are no new commits and return
            if len(to_put) == 0:
                return []
            
            user = user_key.get()
            count = getattr(user, 'total', 0)
            
            if project is not None:
                # get the list of existing projects and make it a set 
                # to filter uniques, if this project is new add it and 
                # update the users total points.
                projects = set(getattr(user, 'projects', []))
                if project not in projects:
                    logging.info('Adding project to user: %s', user.username)
                    projects.add(project)
                    count += 10
                    user.projects = list(projects)

            points = len(to_put)
            user.total = count + points
            to_put.append(user)
            keys = ndb.put_multi(to_put)
            return keys
        
        commits = txn()
        committed = filter(lambda x: x.kind() == 'Commit', commits)
        
        # defer updating the users location if they have one.
        if user.location:
            logging.info("deferring add points to location: %s", user.location_slug)
            deferred.defer(Location.add_points, user.location_slug, len(committed), project)
            
        if getattr(user, 'team_slug', None):
            logging.info("deferring add points to team: %s", user.team_slug)
            deferred.defer(Team.add_points, user.team_slug, len(committed), project)
        
        # Tally up some points, we do this here after the transaction
        # was successful, since the timestamps could be different days
        # we need to pull each one and then record it :( 
        for commit in committed:
            # Mark the commits in the history
            deferred.defer(Accumulator.add_count, username, commit.urlsafe())
            deferred.defer(Accumulator.add_count, 'global', commit.urlsafe())
            deferred.defer(Message.add_commit, commit.urlsafe(), _countdown=5, _queue="live")
        
        return committed
    
    @classmethod
    def create_orphan(cls, commits, project=None):
        """Create a commit with no parent."""
        if not isinstance(commits, (list, tuple)):
            commits = [commits]
        
        to_put = []
        for c in commits:
            commit_hash = c.get('hash')
            if commit_hash is None:
                logging.info("Commit hash missing in create.")
                continue
            commit_key = cls.make_key(commit_hash)
            commit = commit_key.get()
            if commit is None:
                c['project'] = project
                commit = Commit(key=commit_key, **c)
                to_put.append(commit)
                # Defer adding points to global counter 
                deferred.defer(Accumulator.add_count, 'global', c['timestamp'])
        
        commits = ndb.put_multi(to_put)
        return commits
    
class Project(ndb.Model):
    """
{'fraction_non_alphanumeric': 0.08919325204609989, 'fraction_numerical': 0.000918657090362452, 'mean_word_length': 1.9422604422604424, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1164475', 'n_tokens_mistral': 3277, 'n_tokens_neox': 3213, 'n_words': 949}