<commit_before>#!/usr/bin/python
# Filename: changelist.py


"""
Usage: $shell changelist.py [OPTIONS]

              OPTIONS:
                  --make  : Make the change list out
                  --show  : Show the change list out
"""

__author__ = '<EMAIL>'


import commands
import re
import os, sys

from config import Config
from formatters.log import Log

# try:
#     import xml.etree.cElementTree as ET
# except ImportError:
#     import xml.etree.ElementTree as ET

from lxml import etree as ET

TAG="changelist"

class ChangeList:


    def __init__(self, olderRoot, newerRoot, patchXML):
        ChangeList.OLDER_ROOT = olderRoot
        ChangeList.NEWER_ROOT = newerRoot
        ChangeList.PATCH_XML  = patchXML

    def make(self, force=True):
        """ Generate the change list into XML.
            Set force as False not to generate again if exists.  
        """

        if not force and os.path.exists(ChangeList.PATCH_XML):
            Log.d(TAG, "Using the existing %s" % ChangeList.PATCH_XML)
            return True

        cmd = "diff -rq %s %s" % (commands.mkarg(ChangeList.OLDER_ROOT), commands.mkarg(ChangeList.NEWER_ROOT))
        output = commands.getoutput(cmd)
        Log.d(TAG, output)
        hasChange = ChangeList.xmlFrom(output)
        return hasChange

    @staticmethod
    def xmlFrom(diffout):
        root = ET.Element('features')
        feature = ET.Element('feature', 
                   {'description' : 'These files are diff from %s and %s'%(ChangeList.OLDER_ROOT, ChangeList.NEWER_ROOT)})
        root.append(feature)

        # Named group REGEX of differ 
        differRE = re.compile("Files (?P<older>.*) and (?P<newer>.*) differ")
        # Named group REGEX of newer
        onlyRE   = re.compile("Only in (?P<path>.*): (?P<name>.*)")

        hasChange = False

        lines = diffout.split("\n")
        for line in lines:
            match = differRE.search(line)
            if match != None:
                target = os.path.relpath(match.group("newer"), ChangeList.NEWER_ROOT)

                revise = ET.Element('revise',
                           {'action' : 'MERGE',
                            'target' : target})
                feature.append(revise)

                hasChange = True
                continue

            match = onlyRE.search(line)
            if match != None:
                fullpath= os.path.join(match.group("path"), match.group("name"))
                if fullpath.startswith(ChangeList.NEWER_ROOT):
                    action = "ADD"
                    target = os.path.relpath(fullpath, ChangeList.NEWER_ROOT)
                elif fullpath.startswith(ChangeList.OLDER_ROOT):
                    action = "DELETE"
                    target = os.path.relpath(fullpath, ChangeList.OLDER_ROOT)

                revise = ET.Element('revise',
                           {'action' : action,
                            'target' : target})
                feature.append(revise)

                hasChange = True
                continue

        tree = ET.ElementTree(root)
        # Using pretty print to format XML
        tree.write(ChangeList.PATCH_XML, pretty_print=True,
               xml_declaration=True, encoding='utf-8')

        Log.i(TAG, "%s is generated" % ChangeList.PATCH_XML)<commit_msg>update tools in coron<commit_after>#!/usr/bin/python
# Filename: changelist.py


"""
Usage: $shell changelist.py [OPTIONS]

              OPTIONS:
                  --make  : Make the change list out
                  --show  : Show the change list out
"""

__author__ = '<EMAIL>'


import commands
import re
import os, sys

from config import Config
from formatters.log import Log

try:
    import xml.etree.cElementTree as ET
except ImportError:
    import xml.etree.ElementTree as ET

#from lxml import etree as ET

TAG="changelist"

class ChangeList:


    def __init__(self, olderRoot, newerRoot, patchXML):
        ChangeList.OLDER_ROOT = olderRoot
        ChangeList.NEWER_ROOT = newerRoot
        ChangeList.PATCH_XML  = patchXML

    def make(self, force=True):
        """ Generate the change list into XML.
            Set force as False not to generate again if exists.  
        """

        if not force and os.path.exists(ChangeList.PATCH_XML):
            Log.d(TAG, "Using the existing %s" % ChangeList.PATCH_XML)
            return True

        cmd = "diff -rq %s %s" % (commands.mkarg(ChangeList.OLDER_ROOT), commands.mkarg(ChangeList.NEWER_ROOT))
        output = commands.getoutput(cmd)
        Log.d(TAG, output)
        hasChange = ChangeList.xmlFrom(output)
        return hasChange

    @staticmethod
    def xmlFrom(diffout):
        root = ET.Element('features')
        feature = ET.Element('feature', 
                   {'description' : 'These files are diff from %s and %s'%(ChangeList.OLDER_ROOT, ChangeList.NEWER_ROOT)})
        root.append(feature)

        # Named group REGEX of differ 
        differRE = re.compile("Files (?P<older>.*) and (?P<newer>.*) differ")
        # Named group REGEX of newer
        onlyRE   = re.compile("Only in (?P<path>.*): (?P<name>.*)")

        hasChange = False

        lines = diffout.split("\n")
        for line in lines:
            match = differRE.search(line)
            if match != None:
                target = os.path.relpath(match.group("newer"), ChangeList.NEWER_ROOT)

                revise = ET.Element('revise',
                           {'action' : 'MERGE',
                            'target' : target})
                feature.append(revise)

                hasChange = True
                continue

            match = onlyRE.search(line)
            if match != None:
                fullpath= os.path.join(match.group("path"), match.group("name"))
                if fullpath.startswith(ChangeList.NEWER_ROOT):
                    action = "ADD"
                    target = os.path.relpath(fullpath, ChangeList.NEWER_ROOT)
                elif fullpath.startswith(ChangeList.OLDER_ROOT):
                    action = "DELETE"
                    target = os.path.relpath(fullpath, ChangeList.OLDER_ROOT)

                revise = ET.Element('revise',
                           {'action' : action,
                            'target' : target})
                feature.append(revise)

                hasChange = True
                continue

        tree = ET.ElementTree(root)

        # Using pretty print to format XML
        tree.write(ChangeList.PATCH_XML, #pretty_print=True,
               xml_declaration=True, encoding='utf-8')


        Log.i(TAG, "%s is generated" % ChangeList.PATCH_XML)
{'fraction_non_alphanumeric': 0.10505600968816228, 'fraction_numerical': 0.0003027550711474417, 'mean_word_length': 1.8404987102321582, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3064094', 'n_tokens_mistral': 1935, 'n_tokens_neox': 1838, 'n_words': 460}