<commit_before>//
// $Id$
//
// Narya library - tools for developing networked games
// Copyright (C) 2002-2011 Three Rings Design, Inc., All Rights Reserved
// http://code.google.com/p/narya/
//
// This library is free software; you can redistribute it and/or modify it
// under the terms of the GNU Lesser General Public License as published
// by the Free Software Foundation; either version 2.1 of the License, or
// (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

package com.threerings.presents.tools;

import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;

/**
 * Merges updates to a generated source file into a previously generated version of that source
 * while leaving changes outside marked sections alone.
 */
public class GeneratedSourceMerger
{
    /**
     * Returns <code>previouslyGenerated</code> with marked sections updated from the same marked
     * sections in <code>newlyGenerated</code>. Everything outside these sections in
     * <code>previouslyGenerated</code> is returned as is. A marked section starts with <code>//
     * GENERATED {name} START</code> and ends with <code>// GENERATED {name} END</code><p>
     *
     * If <code>previouslyGenerated</code> has a generated section replaced with <code>//
     * GENERATED {name} DISABLED</code>, that section will no longer be updated.
     */
    public String merge (String newlyGenerated, String previouslyGenerated)
        throws Exception
    {
        // Extract the generated section names from the output and make sure they're all matched
        Map<String, Section> sections = Maps.newLinkedHashMap();
        Matcher m = _sectionDelimiter.matcher(newlyGenerated);
        while (m.find()) {
            Section section = extractGeneratedSection(m, newlyGenerated);
            Preconditions.checkArgument(!sections.containsKey(section.name),
                "Section '%s' used more than once", section.name);
            sections.put(section.name, section);
        }

        // Merge with the previously generated source
        StringBuilder merged = new StringBuilder();
        m = _sectionDelimiter.matcher(previouslyGenerated);
        int currentStart = 0;
        while (m.find()) {
            merged.append(previouslyGenerated.substring(currentStart, m.start()));
            Section existingSection = extractGeneratedSection(m, previouslyGenerated);
            Section newSection = sections.remove(existingSection.name);
            if (newSection == null) {
                // Allow generated sections to be dropped in the template, but warn in case
                // something odd's happening
                System.err.println("Dropping previously generated section '" + m.group(1)
                    + "' that's no longer generated by the template");
            } else  if (existingSection.disabled) {
                // If the existing code disables this generation, add that disabled comment
                merged.append(existingSection.contents);
            } else {
                // Otherwise pop in the newly generated code in the place of what was there before
                merged.append(newSection.contents);
            }
            currentStart = m.end();
        }
        // Add generated sections that weren't present in the old output before the last
        // non-generated code.  It's a 50-50 shot, so warn when this happens
        for (Section newSection : sections.values()) {
            System.err.println("Adding previously missing generated section '"
                + newSection.name + "' before the last non-generated text");
            merged.append(newSection.contents);
        }
        // Add any text past the last previously generated section
        merged.append(previouslyGenerated.substring(currentStart));

        return merged.toString();
    }

    /**
     * Returns a section name and its contents from the given matcher pointing to the start of a
     * section. <code>m</code> is at the end of the section when this returns.
     */
    protected Section extractGeneratedSection (Matcher m, String input)
    {
        int startIdx = m.start();
        String name = m.group(1);
        if (m.group(2).equals("DISABLED")) {
            return new Section(name, input.substring(startIdx, m.end()), true);
        }
        Preconditions.checkArgument(m.group(2).equals("START"), "'%s' END without START",
            name);
        Preconditions.checkArgument(m.find(), "'%s' START without END", name);
        String endName = m.group(1);
        Preconditions.checkArgument(m.group(2).equals("END"),
            "'%s' START after '%s' START", endName, name);
        Preconditions.checkArgument(endName.equals(name),
            "'%s' END after '%s' START", endName, name);
        return new Section(name, input.substring(startIdx, m.end()), false);
    }

    protected static class Section {
        public final String name, contents;
        public final boolean disabled;

        public Section (String name, String contents, boolean disabled) {
            this.name = name;
            this.contents = contents;
            this.disabled = disabled;
        }
    }

    protected final Pattern _sectionDelimiter =
        Pattern.compile(" *// GENERATED (\\w+) (START|END|DISABLED)\n");
}<commit_msg>Fixed conflict in generated source merge when using native eol-styles by allowing an optional CR before the LF ($ here did not work for some reason)<commit_after>//
// $Id$
//
// Narya library - tools for developing networked games
// Copyright (C) 2002-2011 Three Rings Design, Inc., All Rights Reserved
// http://code.google.com/p/narya/
//
// This library is free software; you can redistribute it and/or modify it
// under the terms of the GNU Lesser General Public License as published
// by the Free Software Foundation; either version 2.1 of the License, or
// (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

package com.threerings.presents.tools;

import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;

/**
 * Merges updates to a generated source file into a previously generated version of that source
 * while leaving changes outside marked sections alone.
 */
public class GeneratedSourceMerger
{
    /**
     * Returns <code>previouslyGenerated</code> with marked sections updated from the same marked
     * sections in <code>newlyGenerated</code>. Everything outside these sections in
     * <code>previouslyGenerated</code> is returned as is. A marked section starts with <code>//
     * GENERATED {name} START</code> and ends with <code>// GENERATED {name} END</code><p>
     *
     * If <code>previouslyGenerated</code> has a generated section replaced with <code>//
     * GENERATED {name} DISABLED</code>, that section will no longer be updated.
     */
    public String merge (String newlyGenerated, String previouslyGenerated)
        throws Exception
    {
        // Extract the generated section names from the output and make sure they're all matched
        Map<String, Section> sections = Maps.newLinkedHashMap();
        Matcher m = _sectionDelimiter.matcher(newlyGenerated);
        while (m.find()) {
            Section section = extractGeneratedSection(m, newlyGenerated);
            Preconditions.checkArgument(!sections.containsKey(section.name),
                "Section '%s' used more than once", section.name);
            sections.put(section.name, section);
        }

        // Merge with the previously generated source
        StringBuilder merged = new StringBuilder();
        m = _sectionDelimiter.matcher(previouslyGenerated);
        int currentStart = 0;
        while (m.find()) {
            merged.append(previouslyGenerated.substring(currentStart, m.start()));
            Section existingSection = extractGeneratedSection(m, previouslyGenerated);
            Section newSection = sections.remove(existingSection.name);
            if (newSection == null) {
                // Allow generated sections to be dropped in the template, but warn in case
                // something odd's happening
                System.err.println("Dropping previously generated section '" + m.group(1)
                    + "' that's no longer generated by the template");
            } else  if (existingSection.disabled) {
                // If the existing code disables this generation, add that disabled comment
                merged.append(existingSection.contents);
            } else {
                // Otherwise pop in the newly generated code in the place of what was there before
                merged.append(newSection.contents);
            }
            currentStart = m.end();
        }
        // Add generated sections that weren't present in the old output before the last
        // non-generated code.  It's a 50-50 shot, so warn when this happens
        for (Section newSection : sections.values()) {
            System.err.println("Adding previously missing generated section '"
                + newSection.name + "' before the last non-generated text");
            merged.append(newSection.contents);
        }
        // Add any text past the last previously generated section
        merged.append(previouslyGenerated.substring(currentStart));

        return merged.toString();
    }

    /**
     * Returns a section name and its contents from the given matcher pointing to the start of a
     * section. <code>m</code> is at the end of the section when this returns.
     */
    protected Section extractGeneratedSection (Matcher m, String input)
    {
        int startIdx = m.start();
        String name = m.group(1);
        if (m.group(2).equals("DISABLED")) {
            return new Section(name, input.substring(startIdx, m.end()), true);
        }
        Preconditions.checkArgument(m.group(2).equals("START"), "'%s' END without START",
            name);
        Preconditions.checkArgument(m.find(), "'%s' START without END", name);
        String endName = m.group(1);
        Preconditions.checkArgument(m.group(2).equals("END"),
            "'%s' START after '%s' START", endName, name);
        Preconditions.checkArgument(endName.equals(name),
            "'%s' END after '%s' START", endName, name);
        return new Section(name, input.substring(startIdx, m.end()), false);
    }

    protected static class Section {
        public final String name, contents;
        public final boolean disabled;

        public Section (String name, String contents, boolean disabled) {
            this.name = name;
            this.contents = contents;
            this.disabled = disabled;
        }
    }

    protected final Pattern _sectionDelimiter =
        Pattern.compile(" *// GENERATED (\\w+) (START|END|DISABLED)\r?\n");
}
{'fraction_non_alphanumeric': 0.09667622743377763, 'fraction_numerical': 0.0059051796861818795, 'mean_word_length': 2.638735420503376, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5397289', 'n_tokens_mistral': 2996, 'n_tokens_neox': 2878, 'n_words': 1108}