<commit_before>import os, datetime
from subprocess import Popen, PIPE
from speedcenter import settings


path = settings.BASEDIR + '/repos/'
    
def updaterepo(repo):
    repodir = path + repo.split('/')[-1] + "/"
    if os.path.exists(repodir):
        cmd = "hg pull -u"
        p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE, cwd=repodir)
        stdout, stderr = p.communicate()
        if stderr:
            return [{'error': True, 'message': stderr}]
        else:
            return [{'error': False}]
    else:
        #os.mkdir(path)
        cmd = "hg clone %s" % repo
        p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE, cwd=path)
        stdout, stderr = p.communicate()
        if stderr:
            return [{'error': True, 'message': stderr}]
        else:
            return [{'error': False}]

def getlogs(endrev, startrev):
    repodir = path + endrev.project.repo_path.split('/')[-1] + "/"
    if not os.path.exists(repodir):
        updaterepo(endrev.project.repo_path)
    
    cmd = "hg log -r %s:%s -b default --template '{rev}:{node|short}\n{author|person} / {author|user}\n{date}\n{desc}\n=newlog=\n'" % (endrev.commitid, startrev.commitid)
    p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE, cwd=repodir)
    stdout, stderr = p.communicate()
    if stderr:
        return [{'error': True, 'message': stderr}]
    else:
        logs = []
        for log in stdout.split("=newlog=\n"):
            commitid, author, date, message = ("","","","")
            elements = []
            elements = log.split('\n')[:-1]
            if not len(elements) or len(elements) < 4:
                continue
            else:
                commitid = elements.pop(0)
                author = elements.pop(0)
                date = elements.pop(0)
                message = '\n'.join(elements)
                
                # Parse date
                date = date.split('-')[0]
                date = datetime.datetime.fromtimestamp(float(date)).strftime("%Y-%m-%d %H:%M:%S")
                
                # Add changeset info
                logs.append({'date': date, 'author': author, 'message': message,
                'commitid': commitid})
    return logs<commit_msg>Don't show again last log message<commit_after>import os, datetime
from subprocess import Popen, PIPE
from speedcenter import settings


path = settings.BASEDIR + '/repos/'
    
def updaterepo(repo):
    repodir = path + repo.split('/')[-1] + "/"
    if os.path.exists(repodir):
        cmd = "hg pull -u"
        p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE, cwd=repodir)
        stdout, stderr = p.communicate()
        if stderr:
            return [{'error': True, 'message': stderr}]
        else:
            return [{'error': False}]
    else:
        #os.mkdir(path)
        cmd = "hg clone %s" % repo
        p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE, cwd=path)
        stdout, stderr = p.communicate()
        if stderr:
            return [{'error': True, 'message': stderr}]
        else:
            return [{'error': False}]

def getlogs(endrev, startrev):
    repodir = path + endrev.project.repo_path.split('/')[-1] + "/"
    if not os.path.exists(repodir):
        updaterepo(endrev.project.repo_path)
    
    cmd = "hg log -r %s:%s -b default --template '{rev}:{node|short}\n{author|person} / {author|user}\n{date}\n{desc}\n=newlog=\n'" % (endrev.commitid, startrev.commitid)
    p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE, cwd=repodir)
    stdout, stderr = p.communicate()
    if stderr:
        return [{'error': True, 'message': stderr}]
    else:
        logs = []
        for log in stdout.split("=newlog=\n"):
            commitid, author, date, message = ("","","","")
            elements = []
            elements = log.split('\n')[:-1]
            if len(elements) < 4:
                # Don't save "malformed" log
                continue
            else:
                commitid = elements.pop(0)
                author = elements.pop(0)
                date = elements.pop(0)
                # All other newlines should belong to the description text. Join.
                message = '\n'.join(elements)
                
                # Parse date
                date = date.split('-')[0]
                date = datetime.datetime.fromtimestamp(float(date)).strftime("%Y-%m-%d %H:%M:%S")
                
                # Add changeset info
                logs.append({'date': date, 'author': author, 'message': message,
                'commitid': commitid})
    # Remove last log because the startrev log shouldn't be shown
    if len(logs) > 1:
        print logs.pop()
    return logs
{'fraction_non_alphanumeric': 0.1506966773847803, 'fraction_numerical': 0.0036441586280814577, 'mean_word_length': 1.9550348321722608, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '228610', 'n_tokens_mistral': 1494, 'n_tokens_neox': 1441, 'n_words': 350}