<commit_before>///////////////////////////////////////////////////////////////////////////////
//  corpusutil.scala
//
//  Copyright (C) 2012 <NAME>, The University of Texas at Austin
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
///////////////////////////////////////////////////////////////////////////////

package opennlp.textgrounder.util

import collection.mutable

import java.io.PrintStream

import printutil.{errprint, warning}
import ioutil._

package object corpusutil {
  /**
   * A text-file processor where each line is made up of a fixed number
   * of fields, separated by some sort of separator (by default a tab
   * character).  No implementation is provided for `process_lines`,
   * the driver function.  This function in general should loop over the
   * lines, calling `parse_row` on each one.
   *
   * @param split_re Regular expression used to split one field from another.
   *   By default a tab character.
   */
  abstract class FieldTextFileProcessor(
    split_re: String = "\t"
  ) extends TextFileProcessor {
    var all_num_processed = 0
    var all_num_bad = 0
    var num_processed = 0
    var num_bad = 0

    var fieldnames: Seq[String] = _

    /**
     * Set the field names used for processing rows.
     */
    def set_fieldnames(fieldnames: Seq[String]) {
      this.fieldnames = fieldnames
    }

    override def begin_process_file(filehand: FileHandler, file: String) {
      num_processed = 0
      num_bad = 0
      super.begin_process_file(filehand, file)
    }

    override def end_process_file(filehand: FileHandler, file: String) {
      all_num_processed += num_processed
      all_num_bad += num_bad
      super.end_process_file(filehand, file)
    }

    /**
     * Parse a given row into fields.  Call either #process_row or
     * #handle_bad_row.
     *
     * @param line Raw text of line describing the row
     * @return True if processing should continue, false if it should stop.
     */
    def parse_row(line: String) = {
      // println("[%s]" format line)
      val fieldvals = line.split(split_re, -1)
      if (fieldvals.length != fieldnames.length) {
        handle_bad_row(line, fieldvals)
        num_processed += 1
        true
      } else {
        val (good, keep_going) = process_row(fieldvals)
        if (!good)
          handle_bad_row(line, fieldvals)
        num_processed += 1
        keep_going
      }
    }

    /*********************** MUST BE IMPLEMENTED *************************/

    /**
     * Called when a "good" row is seen (good solely in that it has the
     * correct number of fields).
     *
     * @param fieldvals List of the string values for each field.
     *
     * @return Tuple `(good, keep_going)` where `good` indicates whether
     *   the given row was truly "good" (and hence processed, rather than
     *   skipped), and `keep_going` indicates whether processing of further
     *   rows should continue or stop.  If the return value indicates that
     *   the row isn't actually good, `handle_bad_row` will be called.
     */
    def process_row(fieldvals: Seq[String]): (Boolean, Boolean)

    /* Also,

    def process_lines(lines: Iterator[String],
        filehand: FileHandler, file: String,
        compression: String, realname: String): Boolean

       A simple implementation simply loops over all the lines and calls
       parse_row() on each one.
    */

    /******************* MAY BE IMPLEMENTED (OVERRIDDEN) *******************/

    /**
     * Called when a bad row is seen.  By default, output a warning.
     *
     * @param line Text of the row.
     * @param fieldvals Field values parsed from the row.
     */
    def handle_bad_row(line: String, fieldvals: Seq[String]) {
      val lineno = num_processed + 1
      if (fieldnames.length != fieldvals.length) {
        warning(
          """Line %s: Bad record, expected %s fields, saw %s fields;
          skipping line=%s""", lineno, fieldnames.length, fieldvals.length,
          line)
      } else {
        warning("""Line %s: Bad record; skipping line=%s""", lineno, line)
      }
      num_bad += 1
    }

    /* Also, the same "may-be-implemented" functions from the superclass
       TextFileProcessor. */
  }

  case class Schema(
    fieldnames: Seq[String],
    fixed_values: Map[String, String]
  ) {

    val field_indices = fieldnames.zipWithIndex.toMap

    def get_field(fieldvals: Seq[String], key: String,
        error_if_missing: Boolean = true) =
      get_field_or_else(fieldvals, key, error_if_missing = error_if_missing)

    def get_field_or_else(fieldvals: Seq[String], key: String,
        default: String = null, error_if_missing: Boolean = false): String = {
      if (fieldvals.length != fieldnames.length)
        throw FileFormatException(
          "Wrong-length line, expected %d fields, found %d: %s" format (
            fieldnames.length, fieldvals.length, fieldvals))
      if (field_indices contains key)
        fieldvals(field_indices(key))
      else
        get_fixed_field(key, default, error_if_missing)
    }

    def get_fixed_field(key: String, default: String = null,
        error_if_missing: Boolean = false) = {
      if (fixed_values contains key)<commit_msg>Document Schema and allow fixed fields to be optional<commit_after>///////////////////////////////////////////////////////////////////////////////
//  corpusutil.scala
//
//  Copyright (C) 2012 <NAME>, The University of Texas at Austin
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
///////////////////////////////////////////////////////////////////////////////

package opennlp.textgrounder.util

import collection.mutable

import java.io.PrintStream

import printutil.{errprint, warning}
import ioutil._

package object corpusutil {
  /**
   * A text-file processor where each line is made up of a fixed number
   * of fields, separated by some sort of separator (by default a tab
   * character).  No implementation is provided for `process_lines`,
   * the driver function.  This function in general should loop over the
   * lines, calling `parse_row` on each one.
   *
   * @param split_re Regular expression used to split one field from another.
   *   By default a tab character.
   */
  abstract class FieldTextFileProcessor(
    split_re: String = "\t"
  ) extends TextFileProcessor {
    var all_num_processed = 0
    var all_num_bad = 0
    var num_processed = 0
    var num_bad = 0

    var fieldnames: Seq[String] = _

    /**
     * Set the field names used for processing rows.
     */
    def set_fieldnames(fieldnames: Seq[String]) {
      this.fieldnames = fieldnames
    }

    override def begin_process_file(filehand: FileHandler, file: String) {
      num_processed = 0
      num_bad = 0
      super.begin_process_file(filehand, file)
    }

    override def end_process_file(filehand: FileHandler, file: String) {
      all_num_processed += num_processed
      all_num_bad += num_bad
      super.end_process_file(filehand, file)
    }

    /**
     * Parse a given row into fields.  Call either #process_row or
     * #handle_bad_row.
     *
     * @param line Raw text of line describing the row
     * @return True if processing should continue, false if it should stop.
     */
    def parse_row(line: String) = {
      // println("[%s]" format line)
      val fieldvals = line.split(split_re, -1)
      if (fieldvals.length != fieldnames.length) {
        handle_bad_row(line, fieldvals)
        num_processed += 1
        true
      } else {
        val (good, keep_going) = process_row(fieldvals)
        if (!good)
          handle_bad_row(line, fieldvals)
        num_processed += 1
        keep_going
      }
    }

    /*********************** MUST BE IMPLEMENTED *************************/

    /**
     * Called when a "good" row is seen (good solely in that it has the
     * correct number of fields).
     *
     * @param fieldvals List of the string values for each field.
     *
     * @return Tuple `(good, keep_going)` where `good` indicates whether
     *   the given row was truly "good" (and hence processed, rather than
     *   skipped), and `keep_going` indicates whether processing of further
     *   rows should continue or stop.  If the return value indicates that
     *   the row isn't actually good, `handle_bad_row` will be called.
     */
    def process_row(fieldvals: Seq[String]): (Boolean, Boolean)

    /* Also,

    def process_lines(lines: Iterator[String],
        filehand: FileHandler, file: String,
        compression: String, realname: String): Boolean

       A simple implementation simply loops over all the lines and calls
       parse_row() on each one.
    */

    /******************* MAY BE IMPLEMENTED (OVERRIDDEN) *******************/

    /**
     * Called when a bad row is seen.  By default, output a warning.
     *
     * @param line Text of the row.
     * @param fieldvals Field values parsed from the row.
     */
    def handle_bad_row(line: String, fieldvals: Seq[String]) {
      val lineno = num_processed + 1
      if (fieldnames.length != fieldvals.length) {
        warning(
          """Line %s: Bad record, expected %s fields, saw %s fields;
          skipping line=%s""", lineno, fieldnames.length, fieldvals.length,
          line)
      } else {
        warning("""Line %s: Bad record; skipping line=%s""", lineno, line)
      }
      num_bad += 1
    }

    /* Also, the same "may-be-implemented" functions from the superclass
       TextFileProcessor. */
  }

  /**
   * An object describing a corpus schema, i.e. a description of each if the
   * fields in a corpus, along with "fixed fields" containing the same
   * value for every row.
   *
   * @param fieldnames List of the name of each field
   * @param fixed_values Map specifying additional fields possessing the
   *   same value for every row.  This is optional, but usually at least
   *   the "corpus" field should be given, with the name of the corpus
   *   (currently used purely for identification purposes).
   */
  case class Schema(
    fieldnames: Seq[String],
    fixed_values: Map[String, String] = Map[String, String]()
  ) {

    val field_indices = fieldnames.zipWithIndex.toMap

    def get_field(fieldvals: Seq[String], key: String,
        error_if_missing: Boolean = true) =
      get_field_or_else(fieldvals, key, error_if_missing = error_if_missing)

    def get_field_or_else(fieldvals: Seq[String], key: String,
        default: String = null, error_if_missing: Boolean = false): String = {
      if (fieldvals.length != fieldnames.length)
        throw FileFormatException(
          "Wrong-length line, expected %d fields, found %d: %s" format (
            fieldnames.length, fieldvals.length, fieldvals))
      if (field_indices contains key)
        fieldvals(field_indices(key))
      else
        get_fixed_field(key, default, error_if_missing)
    }

    def get_fixed_field(key: String, default: String = null,
        error_if_missing: Boolean = false) = {
      if (fixed_values contains key)
{'fraction_non_alphanumeric': 0.12554004652708542, 'fraction_numerical': 0.0031571950814223994, 'mean_word_length': 2.5052417006406524, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2649842', 'n_tokens_mistral': 3450, 'n_tokens_neox': 3342, 'n_words': 1192}