<commit_before>/*
 * Copyright 1998 by <NAME>.  You can use this software freely so long as you preserve
 * the copyright notice and this restriction, and label your changes.
 */
package edu.brandeis.cs.steele.wn;

import java.util.logging.*;
import java.util.EnumSet;
import java.util.Arrays;
import java.util.Iterator;

/**
 * A <code>Word</code> represents a line of a WordNet <code>index.<em>pos</em></code> file.
 * A <code>Word</code> is retrieved via {@link DictionaryDatabase#lookupWord},
 * and has a <i>lemma</i>, a <i>part of speech (POS)</i>, and a set of <i>senses</i>, which are of type {@link Synset}.
 * 
 * XXX debatable what the type of each sense is - Steele said Synset, i'd say WordSense
 *
 * @see Synset
 * @see WordSense
 * @see Pointer
 * @author <NAME>, <EMAIL>
 * @version 1.0
 */
public class Word implements Iterable<Synset> {
  private static final Logger log = Logger.getLogger(Word.class.getName());
  
  /** offset in <var>pos</var><code>.index</code> file */
  private final int offset;
  /** No case "lemma". Each {@link WordSense} has at least 1 true case lemma
   * (could vary by POS). 
   */
  private final String lemma; 
  // number of senses with counts in sense tagged corpora
  private final int taggedSenseCount;
  /** Synsets are initially stored as offsets, and paged in on demand
   * of the first call of {@link #getSynsets()}. 
   */
  private Object synsets;

  private EnumSet<PointerType> ptrTypes;
  private final byte posOrdinal;
  //
  // Constructor
  //
  Word(final CharSequence line, final int offset) {
    try {
      log.log(Level.FINEST, "parsing line: {0}", line);
      final CharSequenceTokenizer tokenizer = new CharSequenceTokenizer(line, " ");
      this.lemma = tokenizer.nextToken().toString().replace('_', ' ');
      this.posOrdinal = (byte) POS.lookup(tokenizer.nextToken()).ordinal();

      tokenizer.skipNextToken(); // poly_cnt
      //final int poly_cnt = tokenizer.nextInt(); // poly_cnt
      final int pointerCount = tokenizer.nextInt();
      //this.ptrTypes = EnumSet.noneOf(PointerType.class);
      for (int i = 0; i < pointerCount; i++) {
        //XXX each of these tokens is a pointertype, although it may be may be
        //incorrect - see getPointerTypes() comments)
        tokenizer.skipNextToken(); 
        //  try {
        //    ptrTypes.add(PointerType.parseKey(tokenizer.nextToken()));
        //  } catch (final java.util.NoSuchElementException exc) {
        //    log.log(Level.SEVERE, "Word() got PointerType.parseKey() error:", exc);
        //  }
      }

      this.offset = offset;
      final int senseCount = tokenizer.nextInt();
      // this is redundant information
      //assert senseCount == poly_cnt;
      this.taggedSenseCount = tokenizer.nextInt();
      final int[] synsetOffsets = new int[senseCount];
      for (int i = 0; i < senseCount; i++) {
        synsetOffsets[i] = tokenizer.nextInt();
      }
      this.synsets = synsetOffsets;
      //final EnumSet<PointerType> actualPtrTypes = EnumSet.noneOf(PointerType.class);
      //for (final Synset synset : getSynsets()) {
      //  for (final Pointer pointer : synset.getPointers()) {
      //    final PointerType ptrType = pointer.getType();
      //    actualPtrTypes.add(ptrType);
      //  }
      //}
      //// in actualPtrTypes, NOT ptrTypes
      //final EnumSet<PointerType> missing = EnumSet.copyOf(actualPtrTypes); missing.removeAll(ptrTypes);
      //// in ptrTypes, NOT actualPtrTypes
      //final EnumSet<PointerType> extra = EnumSet.copyOf(ptrTypes); extra.removeAll(actualPtrTypes);
      //if(false == missing.isEmpty()) {
      //  //log.log(Level.SEVERE, "missing: {0}", missing);
      //}
      //if(false == extra.isEmpty()) {
      //  //log.log(Level.SEVERE, "extra: {0}", extra);
      //}
    } catch (final RuntimeException e) {
      log.log(Level.SEVERE, "Word parse error on offset: {0} line:\n\"{1}\"", 
          new Object[]{ offset, line });
      log.log(Level.SEVERE, "",  e);
      throw e;
    }
  }

  //
  // Object methods
  //
  @Override public boolean equals(final Object object) {
    return (object instanceof Word)
      && ((Word) object).posOrdinal == posOrdinal
      && ((Word) object).offset == offset;
  }

  @Override public int hashCode() {
    // times 10 shifts left by 1 decimal place
    return ((int) offset * 10) + getPOS().hashCode();
  }

  @Override public String toString() {
    return new StringBuilder("[Word ").
      append(offset).
      append("@").
      append(getPOS().getLabel()).
      append(": \"").
      append(getLemma()).
      append("\"]").toString();
  }

  //
  // Accessors
  //
  public POS getPOS() {
    return POS.fromOrdinal(posOrdinal);
  }

  /**
   * The pointer types available for this indexed word.  May not apply to all
   * senses of the word.
   */
  public EnumSet<PointerType> getPointerTypes() {
    if (ptrTypes == null) {
      // these are not always correct
      // PointerType.INSTANCE_HYPERNYM
      // PointerType.HYPERNYM
      // PointerType.INSTANCE_HYPONYM
      // PointerType.HYPONYM
      final EnumSet<PointerType> localPtrTypes = EnumSet.noneOf(PointerType.class);
      for (final Synset synset : getSynsets()) {
        for (final Pointer pointer : synset.getPointers()) {
          final PointerType ptrType = pointer.getType();
          localPtrTypes.add(ptrType);
        }
      }
      this.ptrTypes = localPtrTypes;
    }
    return ptrTypes;
  }

  /** Return the word's lowercased <i>lemma</i>.  Its lemma is its orthographic
   * representation, for example <code>"dog"</code> or <code>"get up"</code>
   * or <code>"u.s."</code>.
   */
  public String getLemma() {
    return lemma; 
  }

  /**
   * Number of "words" (aka "tokens") in this <tt>Word</tt>'s lemma.
   */
  public int getWordCount() {
    // Morphy.counts() default implementation already counts 
    // space (' ') and underscore ('_') separated words -
    // the given argument is just a formality
    return Morphy.countWords(lemma, ' ');
  }

  /**
   * @return true if this <tt>Word</tt>'s <code>{@link #getWordCount()} &lt; 1</code>.
   */
  public boolean isCollocation() {
    return getWordCount() > 1;
  }

  public int getTaggedSenseCount() {
    return taggedSenseCount;
  }

  public Iterator<Synset> iterator() {
    return Arrays.asList(getSynsets()).iterator();
  }

  public Synset[] getSynsets() {
    // careful with this.synsets
    synchronized(this) {
      if (synsets instanceof int[]) {
        final FileBackedDictionary dictionary = FileBackedDictionary.getInstance();
        final int[] synsetOffsets = (int[])synsets;
        // This memory optimization allows this.synsets is an int[] until this
        // method is called to avoid needing to store both the offset and synset
        // arrays <commit_msg>count words of colloc by space dash<commit_after>/*
 * Copyright 1998 by <NAME>.  You can use this software freely so long as you preserve
 * the copyright notice and this restriction, and label your changes.
 */
package edu.brandeis.cs.steele.wn;

import java.util.logging.*;
import java.util.EnumSet;
import java.util.Arrays;
import java.util.Iterator;

/**
 * A <code>Word</code> represents a line of a WordNet <code>index.<em>pos</em></code> file.
 * A <code>Word</code> is retrieved via {@link DictionaryDatabase#lookupWord},
 * and has a <i>lemma</i>, a <i>part of speech (POS)</i>, and a set of <i>senses</i>, which are of type {@link Synset}.
 * 
 * XXX debatable what the type of each sense is - Steele said Synset, i'd say WordSense
 *
 * @see Synset
 * @see WordSense
 * @see Pointer
 * @author <NAME>, <EMAIL>
 * @version 1.0
 */
public class Word implements Iterable<Synset> {
  private static final Logger log = Logger.getLogger(Word.class.getName());
  
  /** offset in <var>pos</var><code>.index</code> file */
  private final int offset;
  /** No case "lemma". Each {@link WordSense} has at least 1 true case lemma
   * (could vary by POS). 
   */
  private final String lemma; 
  // number of senses with counts in sense tagged corpora
  private final int taggedSenseCount;
  /** Synsets are initially stored as offsets, and paged in on demand
   * of the first call of {@link #getSynsets()}. 
   */
  private Object synsets;

  private EnumSet<PointerType> ptrTypes;
  private final byte posOrdinal;
  //
  // Constructor
  //
  Word(final CharSequence line, final int offset) {
    try {
      log.log(Level.FINEST, "parsing line: {0}", line);
      final CharSequenceTokenizer tokenizer = new CharSequenceTokenizer(line, " ");
      this.lemma = tokenizer.nextToken().toString().replace('_', ' ');
      this.posOrdinal = (byte) POS.lookup(tokenizer.nextToken()).ordinal();

      tokenizer.skipNextToken(); // poly_cnt
      //final int poly_cnt = tokenizer.nextInt(); // poly_cnt
      final int pointerCount = tokenizer.nextInt();
      //this.ptrTypes = EnumSet.noneOf(PointerType.class);
      for (int i = 0; i < pointerCount; i++) {
        //XXX each of these tokens is a pointertype, although it may be may be
        //incorrect - see getPointerTypes() comments)
        tokenizer.skipNextToken(); 
        //  try {
        //    ptrTypes.add(PointerType.parseKey(tokenizer.nextToken()));
        //  } catch (final java.util.NoSuchElementException exc) {
        //    log.log(Level.SEVERE, "Word() got PointerType.parseKey() error:", exc);
        //  }
      }

      this.offset = offset;
      final int senseCount = tokenizer.nextInt();
      // this is redundant information
      //assert senseCount == poly_cnt;
      this.taggedSenseCount = tokenizer.nextInt();
      final int[] synsetOffsets = new int[senseCount];
      for (int i = 0; i < senseCount; i++) {
        synsetOffsets[i] = tokenizer.nextInt();
      }
      this.synsets = synsetOffsets;
      //final EnumSet<PointerType> actualPtrTypes = EnumSet.noneOf(PointerType.class);
      //for (final Synset synset : getSynsets()) {
      //  for (final Pointer pointer : synset.getPointers()) {
      //    final PointerType ptrType = pointer.getType();
      //    actualPtrTypes.add(ptrType);
      //  }
      //}
      //// in actualPtrTypes, NOT ptrTypes
      //final EnumSet<PointerType> missing = EnumSet.copyOf(actualPtrTypes); missing.removeAll(ptrTypes);
      //// in ptrTypes, NOT actualPtrTypes
      //final EnumSet<PointerType> extra = EnumSet.copyOf(ptrTypes); extra.removeAll(actualPtrTypes);
      //if(false == missing.isEmpty()) {
      //  //log.log(Level.SEVERE, "missing: {0}", missing);
      //}
      //if(false == extra.isEmpty()) {
      //  //log.log(Level.SEVERE, "extra: {0}", extra);
      //}
    } catch (final RuntimeException e) {
      log.log(Level.SEVERE, "Word parse error on offset: {0} line:\n\"{1}\"", 
          new Object[]{ offset, line });
      log.log(Level.SEVERE, "",  e);
      throw e;
    }
  }

  //
  // Object methods
  //
  @Override public boolean equals(final Object object) {
    return (object instanceof Word)
      && ((Word) object).posOrdinal == posOrdinal
      && ((Word) object).offset == offset;
  }

  @Override public int hashCode() {
    // times 10 shifts left by 1 decimal place
    return ((int) offset * 10) + getPOS().hashCode();
  }

  @Override public String toString() {
    return new StringBuilder("[Word ").
      append(offset).
      append("@").
      append(getPOS().getLabel()).
      append(": \"").
      append(getLemma()).
      append("\"]").toString();
  }

  //
  // Accessors
  //
  public POS getPOS() {
    return POS.fromOrdinal(posOrdinal);
  }

  /**
   * The pointer types available for this indexed word.  May not apply to all
   * senses of the word.
   */
  public EnumSet<PointerType> getPointerTypes() {
    if (ptrTypes == null) {
      // these are not always correct
      // PointerType.INSTANCE_HYPERNYM
      // PointerType.HYPERNYM
      // PointerType.INSTANCE_HYPONYM
      // PointerType.HYPONYM
      final EnumSet<PointerType> localPtrTypes = EnumSet.noneOf(PointerType.class);
      for (final Synset synset : getSynsets()) {
        for (final Pointer pointer : synset.getPointers()) {
          final PointerType ptrType = pointer.getType();
          localPtrTypes.add(ptrType);
        }
      }
      this.ptrTypes = localPtrTypes;
    }
    return ptrTypes;
  }

  /** Return the word's lowercased <i>lemma</i>.  Its lemma is its orthographic
   * representation, for example <code>"dog"</code> or <code>"get up"</code>
   * or <code>"u.s."</code>.
   */
  public String getLemma() {
    return lemma; 
  }

  /**
   * Number of "words" (aka "tokens") in this <tt>Word</tt>'s lemma.
   */
  public int getWordCount() {
    // Morphy.counts() default implementation already counts 
    // space (' ') and underscore ('_') separated words
    return Morphy.countWords(lemma, '-');
  }

  /**
   * @return true if this <tt>Word</tt>'s <code>{@link #getWordCount()} &lt; 1</code>.
   */
  public boolean isCollocation() {
    return getWordCount() > 1;
  }

  public int getTaggedSenseCount() {
    return taggedSenseCount;
  }

  public Iterator<Synset> iterator() {
    return Arrays.asList(getSynsets()).iterator();
  }

  public Synset[] getSynsets() {
    // careful with this.synsets
    synchronized(this) {
      if (synsets instanceof int[]) {
        final FileBackedDictionary dictionary = FileBackedDictionary.getInstance();
        final int[] synsetOffsets = (int[])synsets;
        // This memory optimization allows this.synsets is an int[] until this
        // method is called to avoid needing to store both the offset and synset
        // arrays 
{'fraction_non_alphanumeric': 0.13767264178665883, 'fraction_numerical': 0.003085512782838672, 'mean_word_length': 2.7367554213560252, 'pattern_counts': {'":': 2, '<': 93, '<?xml version=': 0, '>': 91, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7385075', 'n_tokens_mistral': 4230, 'n_tokens_neox': 4091, 'n_words': 1171}