<commit_before>/* LanguageTool, a natural language style checker 
 * Copyright (C) 2007 <NAME> (http://www.danielnaber.de)
 * 
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
 * USA
 */

package org.languagetool.tagging.disambiguation;

import gnu.trove.THashMap;
import org.languagetool.AnalyzedSentence;
import org.languagetool.AnalyzedToken;
import org.languagetool.AnalyzedTokenReadings;
import org.languagetool.JLanguageTool;
import org.languagetool.tools.StringTools;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;

/**
 * Multiword tagger-chunker.
 *
 * @author <NAME>
 */
public class MultiWordChunker extends AbstractDisambiguator {

  private final String filename;
  private final boolean allowFirstCapitalized;
  private final boolean allowAllUppercase;

  private Map<String, Integer> mStartSpace;
  private Map<String, Integer> mStartNoSpace;
  private Map<String, AnalyzedToken> mFull;

  /**
   * @param filename file text with multiwords and tags
   */
  public MultiWordChunker(String filename) {
    this(filename, false, false);
  }

  /**
   * @param filename              file text with multiwords and tags
   * @param allowFirstCapitalized if set to {@code true}, first word of the
   *                              multiword can be capitalized
   * @param allowAllUppercase     if set to {@code true}, the all uppercase
   *                              version of the multiword is allowed
   */
  public MultiWordChunker(String filename, boolean allowFirstCapitalized, boolean allowAllUppercase) {
    this.filename = filename;
    this.allowFirstCapitalized = allowFirstCapitalized;
    this.allowAllUppercase = allowAllUppercase;
  }

  /*
   * Lazy init, thanks to <NAME>
   */
  private void lazyInit() {

    if (mStartSpace != null) {
      return;
    }

    Map<String, String> interner = new HashMap<>();

    THashMap<String, Integer> mStartSpace = new THashMap<>();
    THashMap<String, Integer> mStartNoSpace = new THashMap<>();
    THashMap<String, AnalyzedToken> mFull = new THashMap<>();

    try (InputStream stream = JLanguageTool.getDataBroker().getFromResourceDirAsStream(filename)) {
      List<String> posTokens = loadWords(stream);
      for (String posToken : posTokens) {
        String[] tokenAndTag = posToken.split("\t");
        if (tokenAndTag.length != 2) {
          throw new RuntimeException(
              "Invalid format in " + filename + ": '" + posToken + "', expected two tab-separated parts");
        }
        List<String> tokens = new ArrayList<>();
        String originalToken = interner.computeIfAbsent(tokenAndTag[0], Function.identity());
        String tag = interner.computeIfAbsent(tokenAndTag[1], Function.identity());
        tokens.add(originalToken);
        if (allowFirstCapitalized) {
          String tokenFirstCapitalized = StringTools.uppercaseFirstChar(originalToken);
          if (!mFull.containsKey(tokenFirstCapitalized) && !originalToken.equals(tokenFirstCapitalized)) {
            tokens.add(tokenFirstCapitalized);
          }
        }
        if (allowAllUppercase) {
          String tokenAllUppercase = originalToken.toUpperCase();
          if (!mFull.containsKey(tokenAllUppercase) && !originalToken.equals(tokenAllUppercase)) {
            tokens.add(tokenAllUppercase);
          }
        }
        for (String token : tokens) {
          boolean containsSpace = token.indexOf(' ') > 0;
          String firstToken;
          String[] firstTokens;
          if (!containsSpace) {
            firstTokens = new String[tokenAndTag[0].length()];
            firstToken = token.substring(0, 1);
            for (int i = 1; i < token.length(); i++) {
              firstTokens[i] = token.substring(i - 1, i);
            }
            if (mStartNoSpace.containsKey(firstToken)) {
              if (mStartNoSpace.get(firstToken) < firstTokens.length) {
                mStartNoSpace.put(firstToken, firstTokens.length);
              }
            } else {
              mStartNoSpace.put(firstToken, firstTokens.length);
            }
          } else {
            firstTokens = token.split(" ");
            firstToken = firstTokens[0];

            if (mStartSpace.containsKey(firstToken)) {
              if (mStartSpace.get(firstToken) < firstTokens.length) {
                mStartSpace.put(firstToken, firstTokens.length);
              }
            } else {
              mStartSpace.put(firstToken, firstTokens.length);
            }
          }
          mFull.put(token, new AnalyzedToken(token, tag, originalToken));
        }
      }
    } catch (IOException e) {
      throw new RuntimeException(e);
    }

    mStartSpace.trimToSize();
    mStartNoSpace.trimToSize();
    mFull.trimToSize();

    this.mStartSpace = mStartSpace;
    this.mStartNoSpace = mStartNoSpace;
    this.mFull = mFull;
  }

  /**
   * Implements multiword POS tags, e.g., &lt;ELLIPSIS&gt; for ellipsis (...)<commit_msg>MultiWordChunker: make sure the map is fully filled before its publishing #5224<commit_after>/* LanguageTool, a natural language style checker 
 * Copyright (C) 2007 <NAME> (http://www.danielnaber.de)
 * 
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
 * USA
 */

package org.languagetool.tagging.disambiguation;

import gnu.trove.THashMap;
import org.languagetool.AnalyzedSentence;
import org.languagetool.AnalyzedToken;
import org.languagetool.AnalyzedTokenReadings;
import org.languagetool.JLanguageTool;
import org.languagetool.tools.StringTools;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;

/**
 * Multiword tagger-chunker.
 *
 * @author <NAME>
 */
public class MultiWordChunker extends AbstractDisambiguator {

  private final String filename;
  private final boolean allowFirstCapitalized;
  private final boolean allowAllUppercase;

  private volatile boolean initialized;
  private Map<String, Integer> mStartSpace;
  private Map<String, Integer> mStartNoSpace;
  private Map<String, AnalyzedToken> mFull;

  /**
   * @param filename file text with multiwords and tags
   */
  public MultiWordChunker(String filename) {
    this(filename, false, false);
  }

  /**
   * @param filename              file text with multiwords and tags
   * @param allowFirstCapitalized if set to {@code true}, first word of the
   *                              multiword can be capitalized
   * @param allowAllUppercase     if set to {@code true}, the all uppercase
   *                              version of the multiword is allowed
   */
  public MultiWordChunker(String filename, boolean allowFirstCapitalized, boolean allowAllUppercase) {
    this.filename = filename;
    this.allowFirstCapitalized = allowFirstCapitalized;
    this.allowAllUppercase = allowAllUppercase;
  }

  /*
   * Lazy init, thanks to <NAME>
   */
  private void lazyInit() {
    if (initialized) {
      return;
    }

    synchronized (this) {
      if (initialized) return;

      THashMap<String, Integer> mStartSpace = new THashMap<>();
      THashMap<String, Integer> mStartNoSpace = new THashMap<>();
      THashMap<String, AnalyzedToken> mFull = new THashMap<>();

      fillMaps(mStartSpace, mStartNoSpace, mFull);

      mStartSpace.trimToSize();
      mStartNoSpace.trimToSize();
      mFull.trimToSize();

      this.mStartSpace = mStartSpace;
      this.mStartNoSpace = mStartNoSpace;
      this.mFull = mFull;
      initialized = true;
    }
  }

  private void fillMaps(Map<String, Integer> mStartSpace, Map<String, Integer> mStartNoSpace, Map<String, AnalyzedToken> mFull) {
    Map<String, String> interner = new HashMap<>();
    try (InputStream stream = JLanguageTool.getDataBroker().getFromResourceDirAsStream(filename)) {
      List<String> posTokens = loadWords(stream);
      for (String posToken : posTokens) {
        String[] tokenAndTag = posToken.split("\t");
        if (tokenAndTag.length != 2) {
          throw new RuntimeException(
              "Invalid format in " + filename + ": '" + posToken + "', expected two tab-separated parts");
        }
        List<String> tokens = new ArrayList<>();
        String originalToken = interner.computeIfAbsent(tokenAndTag[0], Function.identity());
        String tag = interner.computeIfAbsent(tokenAndTag[1], Function.identity());
        tokens.add(originalToken);
        if (allowFirstCapitalized) {
          String tokenFirstCapitalized = StringTools.uppercaseFirstChar(originalToken);
          if (!mFull.containsKey(tokenFirstCapitalized) && !originalToken.equals(tokenFirstCapitalized)) {
            tokens.add(tokenFirstCapitalized);
          }
        }
        if (allowAllUppercase) {
          String tokenAllUppercase = originalToken.toUpperCase();
          if (!mFull.containsKey(tokenAllUppercase) && !originalToken.equals(tokenAllUppercase)) {
            tokens.add(tokenAllUppercase);
          }
        }
        for (String token : tokens) {
          boolean containsSpace = token.indexOf(' ') > 0;
          String firstToken;
          String[] firstTokens;
          if (!containsSpace) {
            firstTokens = new String[tokenAndTag[0].length()];
            firstToken = token.substring(0, 1);
            for (int i = 1; i < token.length(); i++) {
              firstTokens[i] = token.substring(i - 1, i);
            }
            if (mStartNoSpace.containsKey(firstToken)) {
              if (mStartNoSpace.get(firstToken) < firstTokens.length) {
                mStartNoSpace.put(firstToken, firstTokens.length);
              }
            } else {
              mStartNoSpace.put(firstToken, firstTokens.length);
            }
          } else {
            firstTokens = token.split(" ");
            firstToken = firstTokens[0];

            if (mStartSpace.containsKey(firstToken)) {
              if (mStartSpace.get(firstToken) < firstTokens.length) {
                mStartSpace.put(firstToken, firstTokens.length);
              }
            } else {
              mStartSpace.put(firstToken, firstTokens.length);
            }
          }
          mFull.put(token, new AnalyzedToken(token, tag, originalToken));
        }
      }
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }

  /**
   * Implements multiword POS tags, e.g., &lt;ELLIPSIS&gt; for ellipsis (...)
{'fraction_non_alphanumeric': 0.09821132825438887, 'fraction_numerical': 0.004802914872474329, 'mean_word_length': 2.8437301082113304, 'pattern_counts': {'":': 2, '<': 46, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1570845', 'n_tokens_mistral': 3412, 'n_tokens_neox': 3377, 'n_words': 909}