<commit_before>/*---------------------------------------------------------------------------*\
  $Id$
  ---------------------------------------------------------------------------
  This software is released under a Berkeley-style license:

  Copyright (c) 2004-2006 <NAME>. All rights reserved.

  Redistribution and use in source and binary forms are permitted provided
  that: (1) source distributions retain this entire copyright notice and
  comment; and (2) modifications made to the software are prominently
  mentioned, and a copy of the original software (or a pointer to its
  location) are included. The name of the author may not be used to endorse
  or promote products derived from this software without specific prior
  written permission.

  THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
  WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.

  Effectively, this means you can do what you want with the software except
  remove this notice or take advantage of the author's name. If you modify
  the software and redistribute your modified version, you must indicate that
  your version is a modification of the original, and you must provide either
  a pointer to or a copy of the original.
\*---------------------------------------------------------------------------*/

package org.clapper.curn.plugins;

import org.clapper.curn.Constants;
import org.clapper.curn.Curn;
import org.clapper.curn.CurnConfig;
import org.clapper.curn.CurnException;
import org.clapper.curn.FeedInfo;
import org.clapper.curn.MainConfigItemPlugIn;
import org.clapper.curn.FeedConfigItemPlugIn;
import org.clapper.curn.PostFeedDownloadPlugIn;
import org.clapper.curn.Util;

import org.clapper.util.classutil.ClassUtil;
import org.clapper.util.config.ConfigurationException;
import org.clapper.util.logging.Logger;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.InputStreamReader;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * <p>The <tt>CommonXMLFixupsPlugIn</tt> attempts to fix some common errors
 * in the downloaded, pre-parsed XML in any feed for which it is enabled.
 * There is some XML badness that is surprisingly common across feeds,
 * including (but not limited to):</p>
 *
 * <ul>
 *   <li>Using a "naked" ampersand (&amp;) without escaping it.
 *   <li>Use of nonexistent entities (e.g., &amp;ouml;, &amp;nbsp;)
 *   <li>Improperly formatted entity escapes
 * </ul>
 *
 * <p>This plug-in attempts to fix those problems.</p>
 *
 * <p>This plug-in intercepts the following configuration parameters:</p>
 *
 * <table border="1">
 *   <tr valign="bottom" align="left">
 *     <th>Section</th>
 *     <th>Parameter</th>
 *     <th>Legal Values</th>
 *     <th>Meaning</th>
 *   </tr>
 *   <tr valign="top">
 *     <td><tt>[curn]</tt></td>
 *     <td><tt>CommonXMLFixups</tt></td>
 *     <td><tt>true</tt>, <tt>false</tt></td>
 *     <td>The global setting, which can be used to enable or disable
 *         this plug-in for all feeds (though the plug-in can still be
 *         disabled or enabled on a per-feed basis). If not specified, this
 *         parameter defaults to <tt>false</tt>.</td>
 *   </tr>
 *   <tr valign="top">
 *     <td><tt>[Feed<i>xxx</i>]</tt></td>
 *     <td><tt>CommonXMLFixups</tt></td>
 *     <td><tt>true</tt>, <tt>false</tt></td>
 *     <td>Enables or disables this plug-in for a specific feed. If not
 *         specified, this parameter defaults to the global setting.</td>
 *   </tr>
 * </table>
 *
 * @version <tt>$Revision$</tt>
 */
public class CommonXMLFixupsPlugIn
    extends AbstractXMLEditPlugIn
    implements MainConfigItemPlugIn,
               FeedConfigItemPlugIn,
               PostFeedDownloadPlugIn
{
    /*----------------------------------------------------------------------*\
                             Private Constants
    \*----------------------------------------------------------------------*/

    private static final String VAR_COMMON_XML_FIXUPS = "CommonXMLFixups";

    /**
     * The table of edit commands.
     */
    private static final String[] EDITS = new String[]
    {
        // Various forms of unescaped ampersands.

        "s/ & / \\&amp; /g",
        "s/&$/\\&amp;/g",
        "s/&ouml;/\\&#246;/g",
        "s/ &amp;amp; / \\&amp; /g",
        "s/&([^;]+)(\\s)/\\&amp;$1$2/g",
 
        // Remove "&nbsp;" and "nbsp;". The first is legal HTML, but not
        // legal XML. The second is illegal.

        "s/&nbsp;/ /g",
        "s/([^&])nbsp;/$1 /g",

        "s/&mdash;/\\&#8212;/g" 
    };

    /*----------------------------------------------------------------------*\
                            Private Data Items
    \*----------------------------------------------------------------------*/

    /**
     * Feed save data, by feed
     */
    private Map<FeedInfo,Boolean> perFeedEnabledFlag =
        new HashMap<FeedInfo,Boolean>();

    /**
     * Whether globally enabled or not.
     */
    private boolean globallyEnabled = false;

    /**
     * Saved reference to the configuration
     */
    private CurnConfig config = null;

    /**<commit_msg>Added some more fixups (e.g., AT&T)<commit_after>/*---------------------------------------------------------------------------*\
  $Id$
  ---------------------------------------------------------------------------
  This software is released under a Berkeley-style license:

  Copyright (c) 2004-2006 <NAME>. All rights reserved.

  Redistribution and use in source and binary forms are permitted provided
  that: (1) source distributions retain this entire copyright notice and
  comment; and (2) modifications made to the software are prominently
  mentioned, and a copy of the original software (or a pointer to its
  location) are included. The name of the author may not be used to endorse
  or promote products derived from this software without specific prior
  written permission.

  THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED
  WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.

  Effectively, this means you can do what you want with the software except
  remove this notice or take advantage of the author's name. If you modify
  the software and redistribute your modified version, you must indicate that
  your version is a modification of the original, and you must provide either
  a pointer to or a copy of the original.
\*---------------------------------------------------------------------------*/

package org.clapper.curn.plugins;

import org.clapper.curn.Constants;
import org.clapper.curn.Curn;
import org.clapper.curn.CurnConfig;
import org.clapper.curn.CurnException;
import org.clapper.curn.FeedInfo;
import org.clapper.curn.MainConfigItemPlugIn;
import org.clapper.curn.FeedConfigItemPlugIn;
import org.clapper.curn.PostFeedDownloadPlugIn;
import org.clapper.curn.Util;

import org.clapper.util.classutil.ClassUtil;
import org.clapper.util.config.ConfigurationException;
import org.clapper.util.logging.Logger;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.InputStreamReader;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * <p>The <tt>CommonXMLFixupsPlugIn</tt> attempts to fix some common errors
 * in the downloaded, pre-parsed XML in any feed for which it is enabled.
 * There is some XML badness that is surprisingly common across feeds,
 * including (but not limited to):</p>
 *
 * <ul>
 *   <li>Using a "naked" ampersand (&amp;) without escaping it.
 *   <li>Use of nonexistent entities (e.g., &amp;ouml;, &amp;nbsp;)
 *   <li>Improperly formatted entity escapes
 * </ul>
 *
 * <p>This plug-in attempts to fix those problems.</p>
 *
 * <p>This plug-in intercepts the following configuration parameters:</p>
 *
 * <table border="1">
 *   <tr valign="bottom" align="left">
 *     <th>Section</th>
 *     <th>Parameter</th>
 *     <th>Legal Values</th>
 *     <th>Meaning</th>
 *   </tr>
 *   <tr valign="top">
 *     <td><tt>[curn]</tt></td>
 *     <td><tt>CommonXMLFixups</tt></td>
 *     <td><tt>true</tt>, <tt>false</tt></td>
 *     <td>The global setting, which can be used to enable or disable
 *         this plug-in for all feeds (though the plug-in can still be
 *         disabled or enabled on a per-feed basis). If not specified, this
 *         parameter defaults to <tt>false</tt>.</td>
 *   </tr>
 *   <tr valign="top">
 *     <td><tt>[Feed<i>xxx</i>]</tt></td>
 *     <td><tt>CommonXMLFixups</tt></td>
 *     <td><tt>true</tt>, <tt>false</tt></td>
 *     <td>Enables or disables this plug-in for a specific feed. If not
 *         specified, this parameter defaults to the global setting.</td>
 *   </tr>
 * </table>
 *
 * @version <tt>$Revision$</tt>
 */
public class CommonXMLFixupsPlugIn
    extends AbstractXMLEditPlugIn
    implements MainConfigItemPlugIn,
               FeedConfigItemPlugIn,
               PostFeedDownloadPlugIn
{
    /*----------------------------------------------------------------------*\
                             Private Constants
    \*----------------------------------------------------------------------*/

    private static final String VAR_COMMON_XML_FIXUPS = "CommonXMLFixups";

    /**
     * The table of edit commands.
     */
    private static final String[] EDITS = new String[]
    {
        // Various forms of unescaped ampersands.

        "s/ & / \\&amp; /g",
        "s/&$/\\&amp;/g",
        "s/ &amp;amp; / \\&amp; /g",
        "s/&([^;]+)(\\s)/\\&amp;$1$2/g",

        // Remove "&nbsp;" and "nbsp;". The first is legal HTML, but not
        // legal XML. The second is illegal.

        "s/&nbsp;/ /g",
        "s/([^&])nbsp;/$1 /g",

        // Non-existent XML entities

        "s/&ouml;/\\&#246;/g",
        "s/&mdash;/\\&#8212;/g",
        
        // For some reason, no one seems to escape "AT&T" properly...
        
        "s/AT&T/AT\\&amp;T/g"
    };

    /*----------------------------------------------------------------------*\
                            Private Data Items
    \*----------------------------------------------------------------------*/

    /**
     * Feed save data, by feed
     */
    private Map<FeedInfo,Boolean> perFeedEnabledFlag =
        new HashMap<FeedInfo,Boolean>();

    /**
     * Whether globally enabled or not.
     */
    private boolean globallyEnabled = false;

    /**
     * Saved reference to the configuration
     */
    private CurnConfig config = null;

    /**
{'fraction_non_alphanumeric': 0.20615495365787817, 'fraction_numerical': 0.003779357509223432, 'mean_word_length': 2.0889382990550307, 'pattern_counts': {'":': 0, '<': 143, '<?xml version=': 0, '>': 143, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4040148', 'n_tokens_mistral': 3239, 'n_tokens_neox': 3093, 'n_words': 983}