<commit_before>import ConfigParser
import os
import StringIO
import xml.etree.ElementTree as ET

# The standard library XML parser does not give access to comments, which we
# need.  The following extended parser remedies this.  # Code taken from
# https://stackoverflow.com/a/27333347, which was in turn based on <NAME>'s
# code at http://effbot.org/zone/element-pi.htm

_generated_str = "Generated by BEASTling"
_config_file_str = "Original config file:"
_proggen_str = "Configuration built programmatically"
_data_file_str = "BEASTling embedded data file"

class CommentParser(ET.XMLTreeBuilder):

   def __init__(self):
       ET.XMLTreeBuilder.__init__(self)
       # assumes ElementTree 1.2.X
       self._parser.CommentHandler = self.handle_comment

   def handle_comment(self, data):
       self._target.start(ET.Comment, {})
       self._target.data(data)
       self._target.end(ET.Comment)

def extract(filename, overwrite=False):

    messages = []

    # Parse XML file
    parser = CommentParser()
    fp = open(filename, "r")
    parser.feed(fp.read())
    fp.close()
    xml = parser.close()

    comments = [e for e in xml if e.tag == ET.Comment]
    beastling_confs = [c for c in comments if c.text.startswith(_generated_str)]
    if not len(beastling_confs) == 1:
        # Zero or several embedded configs - is this one of our files?!
        raise ValueError("%s doesn't look like a BEASTling-generated XML file" % filename)
    messages.append(write_config(beastling_confs[0].text, overwrite))
    
    data_files = [c for c in comments if c.text.startswith(_data_file_str)]
    for data_file in data_files:
        messages.append(write_data_file(data_file.text, overwrite))

    return [msg for msg in messages if msg]

def write_config(comment_text, overwrite):
    lines = comment_text.split("\n")
    assert lines[1] in (_config_file_str, _proggen_str)
    if lines[1] == _proggen_str:
        return "Original configuration was generated programmatically, no configuration to extract."
    config_text = "\n".join(lines[2:])
    p = ConfigParser.SafeConfigParser()
    p.readfp(StringIO.StringIO(config_text))
    if p.has_option("admin", "basename"):
        filename = "%s.conf" % p.get("admin", "basename")
    else:
        filename = "beastling.conf"
    if os.path.exists(filename) and not overwrite:
        return "BEASTling configuration file %s already exists!  Run beastling with the --overwrite option if you wish to overwrite it.\n" % filename
    directory = os.path.dirname(filename)
    if directory and not os.path.exists(directory):
        os.makedirs(directory)
    fp = open(filename, "w")
    fp.write(config_text)
    fp.close()
    return "Wrote BEASTling configuration file %s.\n" % filename

def write_data_file(comment_text, overwrite):
    lines = comment_text.split("\n")
    filename = lines[0].split(":",1)[1].strip()
    if os.path.exists(filename) and not overwrite:
        return "Embedded data file %s already exists!  Run beastling with the --overwrite option if you wish to overwrite it.\n" % filename
    directory = os.path.dirname(filename)
    if directory and not os.path.exists(directory):
        os.makedirs(directory)
    fp = open(filename, "w")
    fp.write("\n".join(lines[1:]))
    fp.close()
    return "Wrote embedded data file %s.\n" % filename<commit_msg>Partial conversion to py2+3 compatibility.  XML parsing is still problematic.<commit_after>import os
import xml.etree.ElementTree as ET

from clldutils.inifile import INI
from six import StringIO

# The standard library XML parser does not give access to comments, which we
# need.  The following extended parser remedies this.  # Code taken from
# https://stackoverflow.com/a/27333347, which was in turn based on <NAME>'s
# code at http://effbot.org/zone/element-pi.htm

_generated_str = "Generated by BEASTling"
_config_file_str = "Original config file:"
_proggen_str = "Configuration built programmatically"
_data_file_str = "BEASTling embedded data file"

class CommentParser(ET.TreeBuilder):

   def __init__(self):
       ET.TreeBuilder.__init__(self)
       # assumes ElementTree 1.2.X
       self._parser.CommentHandler = self.handle_comment

   def handle_comment(self, data):
       self._target.start(ET.Comment, {})
       self._target.data(data)
       self._target.end(ET.Comment)

def extract(filename, overwrite=False):

    messages = []

    # Parse XML file
    parser = CommentParser()
    fp = open(filename, "r")
    parser.feed(fp.read())
    fp.close()
    xml = parser.close()

    comments = [e for e in xml if e.tag == ET.Comment]
    beastling_confs = [c for c in comments if c.text.startswith(_generated_str)]
    if not len(beastling_confs) == 1:
        # Zero or several embedded configs - is this one of our files?!
        raise ValueError("%s doesn't look like a BEASTling-generated XML file" % filename)
    messages.append(write_config(beastling_confs[0].text, overwrite))
    
    data_files = [c for c in comments if c.text.startswith(_data_file_str)]
    for data_file in data_files:
        messages.append(write_data_file(data_file.text, overwrite))

    return [msg for msg in messages if msg]

def write_config(comment_text, overwrite):
    lines = comment_text.split("\n")
    assert lines[1] in (_config_file_str, _proggen_str)
    if lines[1] == _proggen_str:
        return "Original configuration was generated programmatically, no configuration to extract."
    config_text = "\n".join(lines[2:])
    p = INI()
    p.readfp(StringIO(config_text))
    if p.has_option("admin", "basename"):
        filename = "%s.conf" % p.get("admin", "basename")
    else:
        filename = "beastling.conf"
    if os.path.exists(filename) and not overwrite:
        return "BEASTling configuration file %s already exists!  Run beastling with the --overwrite option if you wish to overwrite it.\n" % filename
    directory = os.path.dirname(filename)
    if directory and not os.path.exists(directory):
        os.makedirs(directory)
    fp = open(filename, "w")
    fp.write(config_text)
    fp.close()
    return "Wrote BEASTling configuration file %s.\n" % filename

def write_data_file(comment_text, overwrite):
    lines = comment_text.split("\n")
    filename = lines[0].split(":",1)[1].strip()
    if os.path.exists(filename) and not overwrite:
        return "Embedded data file %s already exists!  Run beastling with the --overwrite option if you wish to overwrite it.\n" % filename
    directory = os.path.dirname(filename)
    if directory and not os.path.exists(directory):
        os.makedirs(directory)
    fp = open(filename, "w")
    fp.write("\n".join(lines[1:]))
    fp.close()
    return "Wrote embedded data file %s.\n" % filename
{'fraction_non_alphanumeric': 0.11006990926669641, 'fraction_numerical': 0.005949724825226833, 'mean_word_length': 3.534052596089009, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1268546', 'n_tokens_mistral': 2089, 'n_tokens_neox': 2038, 'n_words': 600}