<commit_before>/*
 * Copyright 1998-2009 Linux.org.ru
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

/* (C) <NAME>,
       <NAME>
*/

package ru.org.linux.util;

import java.net.URLEncoder;
import java.util.Iterator;
import java.util.StringTokenizer;

import gnu.regexp.RE;
import gnu.regexp.REException;
import gnu.regexp.REMatch;
import gnu.regexp.REMatchEnumeration;

public class HTMLFormatter {
  private final String text;
  private String nl = " ";
  private int maxlength = 80;
  private boolean urlHighlight = false;
  private boolean Preformat = false;
  private boolean NewLine = false;
  private boolean texNewLine = false;
  private boolean quoting = false;
  private String delim = " \n";

  public HTMLFormatter(String atext) {
    text = atext;
  }

  private static final RE nlRE;
  private static final RE texnlRE;

  static {
    try {
      nlRE = new RE("\n");
      texnlRE = new RE("\n\r?\n\r?");
    } catch (REException e) {
      throw new RuntimeException(e);
    }
  }

  public String process() {
    StringTokenizer st = new StringTokenizer(htmlSpecialChars(text), delim, true);

    StringBuffer sb = new StringBuffer();

    while (st.hasMoreTokens()) {
      sb.append(formatHTMLLine(st.nextToken()));
    }

    String res = sb.toString();

    if (NewLine) {      
      res = nl2br(res, quoting);
    }
    if (texNewLine) {
      res = texnl2br(res, quoting);
    }

    if (Preformat) {
      res = "<pre>" + res + "</pre>";
    }

    return res;
  }

  public void setMaxLength(int value) {
    maxlength = value;
  }

  public void enableUrlHighLightMode() {
    urlHighlight = true;
  }

  public void enablePreformatMode() {
    Preformat = true;
    nl = "\n";
    delim = " \n";
  }

  public void enableNewLineMode() {
    NewLine = true;
    texNewLine = false;
  }

  public void enableTexNewLineMode() {
    NewLine = false;
    texNewLine = true;
  }

  public void enableQuoting() {
    quoting = true;
  }

  private static String URLEncoder(String str) {
    try {
      StringBuffer buf = new StringBuffer();
      for (int i = 0; i < str.length(); i++) {
        char c = str.charAt(i);
        if (c > ' ' && c <= 'z') {
          buf.append(c);
        } else {
          buf.append(URLEncoder.encode("" + c, "UTF-8"));
        }
      }
      return buf.toString();
    } catch (Exception e) {
      return str;
    }
  }

  private static final RE urlRE;

  static {
    try {
//      urlRE = new RE("(?:(?:(?:(?:https?://)|(?:ftp://)|(?:www\\.))|(?:ftp\\.))[a-z0-9.-]+\\.[a-z]+(?::[0-9]+)?(?:/(?:[\\w=?:+/\\(\\)\\[\\]~&%;,._#-]*[\\w=?+/~&%-])?)?)|(?:mailto: ?[a-z0-9+]+@[a-z0-9.-]+.[a-z]+)", RE.REG_ICASE);
        /*urlRE = new RE("(?:(?:(?:(?:https?://)|(?:ftp://)|(?:www\\.))|(?:ftp\\.))[a-z0-9.-]+\\.[a-z]+(?::[" +
          "0-9]+)?(?:/(?:([\\w=?:+/\\[\\]~&%;,._#-]|(\\([^\\)]*\\)))*([\\w=?+/~&%-]|(\\([^\\)]*\\))))" +
          "?)?)|(?:mailto: ?[a-z0-9+]+@[a-z0-9.-]+.[a-z]+)", RE.REG_ICASE);*/
      //fix #73: allow only &amp; entity in url
        urlRE = new RE("(?:(?:(?:(?:https?://)|(?:ftp://)|(?:www\\.))|(?:ftp\\.))[a-z0-9.-]+\\.[a-z]+(?::[" +
          "0-9]+)?(?:/(?:([\\w=?:+/\\[\\]~%;,._\\#-]|(\\([^\\)]*\\)))*([\\w=?+/~%-]" +
          "|(&(amp;)?(([\\w=?:+/\\[\\]~%;,._\\#-]+=[\\w=?:+/\\[\\]~%;,._\\#-]+)|([\\#0-9;]+)))+" +
          "|(\\([^\\)]*\\))))?)?)|(?:mailto: ?[a-z0-9+]+@[a-z0-9.-]+.[a-z]+)",
        RE.REG_ICASE);
    } catch (REException e) {
      throw new RuntimeException(e);
    }
  }
<commit_msg>Allow almost any characters in parameter names and values<commit_after>/*
 * Copyright 1998-2009 Linux.org.ru
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *        http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

/* (C) <NAME>,
       <NAME>
*/

package ru.org.linux.util;

import java.net.URLEncoder;
import java.util.Iterator;
import java.util.StringTokenizer;

import gnu.regexp.RE;
import gnu.regexp.REException;
import gnu.regexp.REMatch;
import gnu.regexp.REMatchEnumeration;

public class HTMLFormatter {
  private final String text;
  private String nl = " ";
  private int maxlength = 80;
  private boolean urlHighlight = false;
  private boolean Preformat = false;
  private boolean NewLine = false;
  private boolean texNewLine = false;
  private boolean quoting = false;
  private String delim = " \n";

  public HTMLFormatter(String atext) {
    text = atext;
  }

  private static final RE nlRE;
  private static final RE texnlRE;

  static {
    try {
      nlRE = new RE("\n");
      texnlRE = new RE("\n\r?\n\r?");
    } catch (REException e) {
      throw new RuntimeException(e);
    }
  }

  public String process() {
    StringTokenizer st = new StringTokenizer(htmlSpecialChars(text), delim, true);

    StringBuffer sb = new StringBuffer();

    while (st.hasMoreTokens()) {
      sb.append(formatHTMLLine(st.nextToken()));
    }

    String res = sb.toString();

    if (NewLine) {      
      res = nl2br(res, quoting);
    }
    if (texNewLine) {
      res = texnl2br(res, quoting);
    }

    if (Preformat) {
      res = "<pre>" + res + "</pre>";
    }

    return res;
  }

  public void setMaxLength(int value) {
    maxlength = value;
  }

  public void enableUrlHighLightMode() {
    urlHighlight = true;
  }

  public void enablePreformatMode() {
    Preformat = true;
    nl = "\n";
    delim = " \n";
  }

  public void enableNewLineMode() {
    NewLine = true;
    texNewLine = false;
  }

  public void enableTexNewLineMode() {
    NewLine = false;
    texNewLine = true;
  }

  public void enableQuoting() {
    quoting = true;
  }

  private static String URLEncoder(String str) {
    try {
      StringBuffer buf = new StringBuffer();
      for (int i = 0; i < str.length(); i++) {
        char c = str.charAt(i);
        if (c > ' ' && c <= 'z') {
          buf.append(c);
        } else {
          buf.append(URLEncoder.encode("" + c, "UTF-8"));
        }
      }
      return buf.toString();
    } catch (Exception e) {
      return str;
    }
  }

  private static final RE urlRE;

  static {
    try {
//      urlRE = new RE("(?:(?:(?:(?:https?://)|(?:ftp://)|(?:www\\.))|(?:ftp\\.))[a-z0-9.-]+\\.[a-z]+(?::[0-9]+)?(?:/(?:[\\w=?:+/\\(\\)\\[\\]~&%;,._#-]*[\\w=?+/~&%-])?)?)|(?:mailto: ?[a-z0-9+]+@[a-z0-9.-]+.[a-z]+)", RE.REG_ICASE);
        /*urlRE = new RE("(?:(?:(?:(?:https?://)|(?:ftp://)|(?:www\\.))|(?:ftp\\.))[a-z0-9.-]+\\.[a-z]+(?::[" +
          "0-9]+)?(?:/(?:([\\w=?:+/\\[\\]~&%;,._#-]|(\\([^\\)]*\\)))*([\\w=?+/~&%-]|(\\([^\\)]*\\))))" +
          "?)?)|(?:mailto: ?[a-z0-9+]+@[a-z0-9.-]+.[a-z]+)", RE.REG_ICASE);*/
      //fix #73: allow only &amp; entity in url
        urlRE = new RE("(?:(?:(?:(?:https?://)|(?:ftp://)|(?:www\\.))|(?:ftp\\.))[a-z0-9.-]+\\.[a-z]+(?::[" +
          "0-9]+)?(?:/(?:([\\w=?:+/\\[\\]~%;,._\\#-]|(\\([^\\)]*\\)))*([\\w=?+/~%-]" +
          "|(&(amp;)?(([\\w=?:+/\\[\\]~%;,._\\#-$]+=[\\w=?:+/\\[\\]~%;,._\\#-$]+)|([\\#0-9;]+)))+" +
          "|(\\([^\\)]*\\))))?)?)|(?:mailto: ?[a-z0-9+]+@[a-z0-9.-]+.[a-z]+)",
        RE.REG_ICASE);
    } catch (REException e) {
      throw new RuntimeException(e);
    }
  }

{'fraction_non_alphanumeric': 0.24039408866995074, 'fraction_numerical': 0.011330049261083743, 'mean_word_length': 2.451338716532087, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3093193', 'n_tokens_mistral': 3053, 'n_tokens_neox': 2970, 'n_words': 650}