<commit_before>/* LanguageTool, a natural language style checker
 * Copyright (C) 2020 <NAME> (http://www.danielnaber.de)
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
 * USA
 */
package org.languagetool.dev.diff;

import org.jetbrains.annotations.NotNull;
import org.languagetool.JLanguageTool;
import org.languagetool.Language;
import org.languagetool.Languages;
import org.languagetool.rules.Rule;

import java.io.FileNotFoundException;
import java.io.FileReader;
import java.util.*;

/**
 * Converts plain text results of Main or SentenceSourceChecker to HTML, sorted by rule id.
 */
class ResultToHtml {

  private final Map<String, String> ruleIdToCategoryId = new HashMap<>();
  
  ResultToHtml(Language lang) {
    JLanguageTool lt = new JLanguageTool(lang);
    for (Rule rule : lt.getAllRules()) {
      ruleIdToCategoryId.put(rule.getId(), rule.getCategory().getId().toString());
    }
  }

  private void run(String filename) throws FileNotFoundException {
    LightRuleMatchParser parser = new LightRuleMatchParser();
    List<LightRuleMatch> matches = parser.parse(new FileReader(filename));
    matches.sort((k, v) -> {
      String catIdK = getCategoryId(k);
      String catIdV = getCategoryId(v);
      if (catIdK.equals(catIdV)) {
        return k.getRuleId().compareTo(v.getRuleId());
      } else {
        return catIdK.compareTo(catIdV);
      }
    }
    );
    printHtml(filename, matches);
  }

  private void printHtml(String filename, List<LightRuleMatch> matches) {
    print("<!-- generated by " + ResultToHtml.class.getSimpleName() + " on " + new Date() + " -->");
    print("<html>");
    print("<head>");
    print("  <title>Sorted " + filename + "</title>");
    print("  <style>");
    print("    .sentence { color: #000; }");
    print("    .message { color: #777; }");
    print("    .marker { text-decoration: underline; background-color: #ffe8e8 }");
    print("    li { margin-bottom: 8px; }");
    print("  </style>");
    print("</head>");
    print("<body>");
    print(matches.size() + " total matches<br>");
    printToc(matches);
    String prevRuleId = "";
    String prevCategoryId = "";
    boolean listStarted = false;
    for (LightRuleMatch match : matches) {
      String categoryId = getCategoryId(match);
      if (!match.getRuleId().equals(prevRuleId)) {
        if (listStarted) {
          print("</ol>");
        }
        if (!categoryId.equals(prevCategoryId)) {
          print("<h1>Category " + categoryId + "</h1>");
        }
        String tempOff = match.isTempOff() ? "[temp_off]" : "";
        print("<a name='" + match.getRuleId() + "'></a><h3>" + match.getRuleId() + " " + tempOff + "</h3>");
        print("Source: " + match.getRuleSource() + "<br><br>");
        print("<ol>");
        listStarted = true;
      }
      print("<li>");
      print("  <span class='message'>" + match.getMessage() + "</span><br>");
      print("  <span class='sentence'>" + match.getContext() + "</span><br>");
      print("</li>");
      prevRuleId = match.getRuleId();
      prevCategoryId = categoryId;
    }
    print("</ol>");
    print("</body>");
    print("</html>");
  }

  @NotNull
  private String getCategoryId(LightRuleMatch match) {
    String categoryId = ruleIdToCategoryId.get(match.getRuleId());
    if (categoryId == null) {
      categoryId = "unknown";  // some rules cannot be mapped, as the rule ids might have changes since the input was generated
    }
    return categoryId;
  }

  private void printToc(List<LightRuleMatch> matches) {
    Map<String, Integer> catToCount = getCatToCount(matches);
    String prevRuleId = "";
    String prevCategoryId = "";
    print("<h1>TOC</h1>");
    for (LightRuleMatch match : matches) {
      String ruleId = match.getRuleId();
      String categoryId = getCategoryId(match);
      if (!ruleId.equals(prevRuleId)) {
        if (!categoryId.equals(prevCategoryId)) {
          print("<h3>Category " + categoryId + "</h3>");
        }
        print("<a href='#" + ruleId + "'>" + ruleId + " (" + catToCount.get(ruleId) + ")</a><br>");
      }
      prevRuleId = ruleId;
      prevCategoryId = categoryId;
    }
    print("<br>");
  }

  private Map<String, Integer> getCatToCount(List<LightRuleMatch> matches) {
    Map<String, Integer> catToCount = new HashMap<>();
    for (LightRuleMatch match : matches) {
      String id = match.getRuleId();
      if (catToCount.containsKey(id)) {
        catToCount.put(id, catToCount.get(id) + 1);
      } else {
        catToCount.put(id, 1);
      }
    }
    return catToCount;
  }

  private void print(String s) {
    System.out.println(s);
  }

  public static void main(String[] args) throws FileNotFoundException {
    if (args.length != 2) {
      System.out.println("Usage: " + ResultToHtml.class.getSimpleName() + " <langCode> <plainTextResult>");
      System.out.println("  <plainTextResult> is the result of e.g. Main or SentenceSourceChecker");
      System.exit(1);
    }
    ResultToHtml prg = new ResultToHtml(Languages.getLanguageForShortCode(args[0]));
    prg.run(args[1]);
  }
}
<commit_msg>set encoding for HTML<commit_after>/* LanguageTool, a natural language style checker
 * Copyright (C) 2020 <NAME> (http://www.danielnaber.de)
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
 * USA
 */
package org.languagetool.dev.diff;

import org.jetbrains.annotations.NotNull;
import org.languagetool.JLanguageTool;
import org.languagetool.Language;
import org.languagetool.Languages;
import org.languagetool.rules.Rule;

import java.io.FileNotFoundException;
import java.io.FileReader;
import java.util.*;

/**
 * Converts plain text results of Main or SentenceSourceChecker to HTML, sorted by rule id.
 */
class ResultToHtml {

  private final Map<String, String> ruleIdToCategoryId = new HashMap<>();
  
  ResultToHtml(Language lang) {
    JLanguageTool lt = new JLanguageTool(lang);
    for (Rule rule : lt.getAllRules()) {
      ruleIdToCategoryId.put(rule.getId(), rule.getCategory().getId().toString());
    }
  }

  private void run(String filename) throws FileNotFoundException {
    LightRuleMatchParser parser = new LightRuleMatchParser();
    List<LightRuleMatch> matches = parser.parse(new FileReader(filename));
    matches.sort((k, v) -> {
      String catIdK = getCategoryId(k);
      String catIdV = getCategoryId(v);
      if (catIdK.equals(catIdV)) {
        return k.getRuleId().compareTo(v.getRuleId());
      } else {
        return catIdK.compareTo(catIdV);
      }
    }
    );
    printHtml(filename, matches);
  }

  private void printHtml(String filename, List<LightRuleMatch> matches) {
    print("<!doctype html>");
    print("<!-- generated by " + ResultToHtml.class.getSimpleName() + " on " + new Date() + " -->");
    print("<html>");
    print("<head>");
    print("  <title>Sorted " + filename + "</title>");
    print("  <meta http-equiv=\"content-type\" content=\"charset=utf-8\">");
    print("  <style>");
    print("    .sentence { color: #000; }");
    print("    .message { color: #777; }");
    print("    .marker { text-decoration: underline; background-color: #ffe8e8 }");
    print("    li { margin-bottom: 8px; }");
    print("  </style>");
    print("</head>");
    print("<body>");
    print(matches.size() + " total matches<br>");
    printToc(matches);
    String prevRuleId = "";
    String prevCategoryId = "";
    boolean listStarted = false;
    for (LightRuleMatch match : matches) {
      String categoryId = getCategoryId(match);
      if (!match.getRuleId().equals(prevRuleId)) {
        if (listStarted) {
          print("</ol>");
        }
        if (!categoryId.equals(prevCategoryId)) {
          print("<h1>Category " + categoryId + "</h1>");
        }
        String tempOff = match.isTempOff() ? "[temp_off]" : "";
        print("<a name='" + match.getRuleId() + "'></a><h3>" + match.getRuleId() + " " + tempOff + "</h3>");
        print("Source: " + match.getRuleSource() + "<br><br>");
        print("<ol>");
        listStarted = true;
      }
      print("<li>");
      print("  <span class='message'>" + match.getMessage() + "</span><br>");
      print("  <span class='sentence'>" + match.getContext() + "</span><br>");
      print("</li>");
      prevRuleId = match.getRuleId();
      prevCategoryId = categoryId;
    }
    print("</ol>");
    print("</body>");
    print("</html>");
  }

  @NotNull
  private String getCategoryId(LightRuleMatch match) {
    String categoryId = ruleIdToCategoryId.get(match.getRuleId());
    if (categoryId == null) {
      categoryId = "unknown";  // some rules cannot be mapped, as the rule ids might have changes since the input was generated
    }
    return categoryId;
  }

  private void printToc(List<LightRuleMatch> matches) {
    Map<String, Integer> catToCount = getCatToCount(matches);
    String prevRuleId = "";
    String prevCategoryId = "";
    print("<h1>TOC</h1>");
    for (LightRuleMatch match : matches) {
      String ruleId = match.getRuleId();
      String categoryId = getCategoryId(match);
      if (!ruleId.equals(prevRuleId)) {
        if (!categoryId.equals(prevCategoryId)) {
          print("<h3>Category " + categoryId + "</h3>");
        }
        print("<a href='#" + ruleId + "'>" + ruleId + " (" + catToCount.get(ruleId) + ")</a><br>");
      }
      prevRuleId = ruleId;
      prevCategoryId = categoryId;
    }
    print("<br>");
  }

  private Map<String, Integer> getCatToCount(List<LightRuleMatch> matches) {
    Map<String, Integer> catToCount = new HashMap<>();
    for (LightRuleMatch match : matches) {
      String id = match.getRuleId();
      if (catToCount.containsKey(id)) {
        catToCount.put(id, catToCount.get(id) + 1);
      } else {
        catToCount.put(id, 1);
      }
    }
    return catToCount;
  }

  private void print(String s) {
    System.out.println(s);
  }

  public static void main(String[] args) throws FileNotFoundException {
    if (args.length != 2) {
      System.out.println("Usage: " + ResultToHtml.class.getSimpleName() + " <langCode> <plainTextResult>");
      System.out.println("  <plainTextResult> is the result of e.g. Main or SentenceSourceChecker");
      System.exit(1);
    }
    ResultToHtml prg = new ResultToHtml(Languages.getLanguageForShortCode(args[0]));
    prg.run(args[1]);
  }
}

{'fraction_non_alphanumeric': 0.1488959532605894, 'fraction_numerical': 0.0069593607698255865, 'mean_word_length': 3.220449601160261, 'pattern_counts': {'":': 0, '<': 111, '<?xml version=': 0, '>': 113, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7152735', 'n_tokens_mistral': 3626, 'n_tokens_neox': 3560, 'n_words': 919}