<commit_before>package org.ihtsdo.otf.mapping.mojo;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.PushbackInputStream;
import java.io.Reader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Stack;
import java.util.StringTokenizer;

import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.KeywordAnalyzer;
import org.apache.lucene.analysis.PerFieldAnalyzerWrapper;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.Field.TermVector;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.SimpleFSDirectory;
import org.apache.lucene.util.Version;
import org.apache.maven.plugin.AbstractMojo;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugin.MojoFailureException;
import org.ihtsdo.otf.mapping.services.helpers.ConfigUtility;
import org.xml.sax.Attributes;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;

/**
 * Converts XML index file to HTML pages.
 * 
 * See admin/indexes/pom.xml for a sample execution.
 * 
 * @goal convert-to-lucene
 */
public class IndexXmlToLuceneMojo extends AbstractMojo {

  /** The writer. */
  IndexWriter writer = null;

  /** The out. */
  private PrintWriter out = null;

  /** The words. */
  Set<String> words = null;

  /** Delimiter string - not part of a word. */
  private static final String DELIM = " \t-({[)}]_!@#%&\\:;\"',.?/~+=|<>$`^";

  /**
   * The input directory.
   * 
   * @parameter
   * @required
   */
  private String inputDir;

  /**
   * The input XML file.
   * 
   * @parameter
   */
  private String inputFile;

  /**
   * The terminology.
   * 
   * @parameter
   */
  String terminology;

  /**
   * The terminology version.
   * 
   * @parameter
   */
  String terminologyVersion;

  /**
   * The domain.
   * 
   * @parameter
   */
  String domain;

  /**
   * The dictionary file.
   * 
   */
  private String dictionaryFile;

  /** The output dir. */
  private String outputDir;

  /**
   * Executes the plugin.
   * 
   * @throws MojoExecutionException the mojo execution exception
   */
  @Override
  public void execute() throws MojoExecutionException {
    getLog().info("Starting to convert index XML to Lucene");
    getLog().info("  inputDir = " + inputDir);
    getLog().info("  inputFile = " + inputFile);
    getLog().info("  terminology = " + terminology);
    getLog().info("  terminologyVersion = " + terminologyVersion);
    getLog().info("  domain = " + domain);

    // Use XML parser to go through extracted XML files
    // and produce HTLM files

    try {

      // set the input directory
      String baseDir = inputDir + "/" + terminology + "/" + terminologyVersion;
      inputDir = baseDir + "/xml";

      // set the lucene output directory
      String indexBaseDir =
          ConfigUtility.getConfigProperties().getProperty(
              "hibernate.search.default.indexBase");
      outputDir =
          indexBaseDir + "/" + terminology + "/" + terminologyVersion + "/"
              + domain;
      String dictionaryDir = outputDir + "/config/dict";
      dictionaryFile = "words.txt";

      getLog().info("  outputDir: " + outputDir);
      getLog().info("  dictionaryDir: " + dictionaryDir);

      // Set up index/dictionary directories and files
      new File(outputDir).mkdirs();
      new File(dictionaryDir, dictionaryFile).getParentFile().mkdirs();
      dictionaryFile =
          new File(dictionaryDir, dictionaryFile).getAbsolutePath();
      out = new PrintWriter(new FileWriter(new File(dictionaryFile)));

      // initialize state
      words = new HashSet<>();

      Map<String, Analyzer> fieldAnalyzers = new HashMap<>();<commit_msg>ADMIN: script for building ICD10 index viewer data<commit_after>package org.ihtsdo.otf.mapping.mojo;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.PushbackInputStream;
import java.io.Reader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Stack;
import java.util.StringTokenizer;

import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.KeywordAnalyzer;
import org.apache.lucene.analysis.PerFieldAnalyzerWrapper;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.Field.TermVector;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.SimpleFSDirectory;
import org.apache.lucene.util.Version;
import org.apache.maven.plugin.AbstractMojo;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugin.MojoFailureException;
import org.ihtsdo.otf.mapping.services.helpers.ConfigUtility;
import org.xml.sax.Attributes;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;

/**
 * Converts XML index file to HTML pages.
 * 
 * See admin/indexes/pom.xml for a sample execution.
 * 
 * @goal convert-to-lucene
 */
public class IndexXmlToLuceneMojo extends AbstractMojo {

  /** The writer. */
  IndexWriter writer = null;

  /** The out. */
  private PrintWriter out = null;

  /** The words. */
  Set<String> words = null;

  /** Delimiter string - not part of a word. */
  private static final String DELIM = " \t-({[)}]_!@#%&\\:;\"',.?/~+=|<>$`^";

  /**
   * The input directory.
   * 
   * @parameter
   * @required
   */
  private String inputDir;

  /**
   * The input XML file.
   * 
   * @parameter
   */
  private String inputFile;

  /**
   * The terminology.
   * 
   * @parameter
   */
  String terminology;

  /**
   * The terminology version.
   * 
   * @parameter
   */
  String terminologyVersion;

  /**
   * The domain.
   * 
   * @parameter
   */
  String domain;

  /**
   * The dictionary file.
   * 
   */
  private String dictionaryFile;

  /** The output dir. */
  private String outputDir;

  /**
   * Executes the plugin.
   * 
   * @throws MojoExecutionException the mojo execution exception
   */
  @Override
  public void execute() throws MojoExecutionException {
    getLog().info("Starting to convert index XML to Lucene");
    getLog().info("  inputDir = " + inputDir);
    getLog().info("  inputFile = " + inputFile);
    getLog().info("  terminology = " + terminology);
    getLog().info("  terminologyVersion = " + terminologyVersion);
    getLog().info("  domain = " + domain);

    // Use XML parser to go through extracted XML files
    // and produce HTLM files

    try {

      // set the input directory
      String baseDir = inputDir + "/" + terminology + "/" + terminologyVersion;
      inputDir = baseDir + "/xml";

      // set the lucene output directory
      // TODO: put this into "lucene" directory of baseDir, not hibernate setting!
      String indexBaseDir =
          ConfigUtility.getConfigProperties().getProperty(
              "hibernate.search.default.indexBase");
      outputDir =
          indexBaseDir + "/" + terminology + "/" + terminologyVersion + "/"
              + domain;
      String dictionaryDir = outputDir + "/config/dict";
      dictionaryFile = "words.txt";

      getLog().info("  outputDir: " + outputDir);
      getLog().info("  dictionaryDir: " + dictionaryDir);

      // Set up index/dictionary directories and files
      new File(outputDir).mkdirs();
      new File(dictionaryDir, dictionaryFile).getParentFile().mkdirs();
      dictionaryFile =
          new File(dictionaryDir, dictionaryFile).getAbsolutePath();
      out = new PrintWriter(new FileWriter(new File(dictionaryFile)));

      // initialize state
      words = new HashSet<>();

      Map<String, Analyzer> fieldAnalyzers = new HashMap<>();
{'fraction_non_alphanumeric': 0.12505475251861586, 'fraction_numerical': 0.0002190100744634253, 'mean_word_length': 3.365678776290631, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1893627', 'n_tokens_mistral': 2850, 'n_tokens_neox': 2708, 'n_words': 621}