<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreemnets.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


package opennlp.tools.util.model;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;

import opennlp.tools.util.InvalidFormatException;
import opennlp.tools.util.Version;

/**
 * This model is a common based which can be used by the components
 * model classes.
 * 
 * TODO:
 * Provide sub classes access to serializers already in constructor
 */
public abstract class BaseModel {

  protected static final String MANIFEST_ENTRY = "manifest.properties";
  
  private static final String MANIFEST_VERSION_PROPERTY = "Manifest-Version";
  private static final String COMPONENT_NAME_PROPERTY = "Component-Name";
  private static final String VERSION_PROPERTY = "OpenNLP-Version";
  private static final String TIMESTAMP_PROPERTY = "Timestamp";
  private static final String LANGUAGE_PROPERTY = "Language";
  
  public static final String TRAINING_CUTOFF_PROPERTY = "Training-Cutoff";
  public static final String TRAINING_ITERATIONS_PROPERTY = "Training-Iterations";
  public static final String TRAINING_EVENTHASH_PROPERTY = "Training-Eventhash";
  
  private Map<String, ArtifactSerializer> artifactSerializers =
      new HashMap<String, ArtifactSerializer>();

  protected final Map<String, Object> artifactMap;

  private final String componentName;
  
  /**
   * Initializes the current instance.
   *
   * @param languageCode
   * @param manifestInfoEntries additional information in the manifest
   */
  protected BaseModel(String componentName, String languageCode, Map<String, String> manifestInfoEntries) {

    if (componentName == null)
        throw new IllegalArgumentException("componentName must not be null!");
    
    if (languageCode == null)
        throw new IllegalArgumentException("languageCode must not be null!");

    this.componentName = componentName;
    
    artifactMap = new HashMap<String, Object>();
    
    createArtifactSerializers(artifactSerializers);
    
    Properties manifest = new Properties();
    manifest.setProperty(MANIFEST_VERSION_PROPERTY, "1.0");
    manifest.setProperty(LANGUAGE_PROPERTY, languageCode);
    manifest.setProperty(VERSION_PROPERTY, Version.currentVersion().toString());
    manifest.setProperty(TIMESTAMP_PROPERTY, 
        Long.toString(System.currentTimeMillis()));
    manifest.setProperty(COMPONENT_NAME_PROPERTY, componentName);
    
    if (manifestInfoEntries != null) {
      for (Map.Entry<String, String> entry : manifestInfoEntries.entrySet()) {
        manifest.setProperty(entry.getKey(), entry.getValue());
      }
    }
      
    artifactMap.put(MANIFEST_ENTRY, manifest);
  }

  /**
   * Initializes the current instance.
   *
   * @param in
   *
   * @throws IOException
   * @throws InvalidFormatException
   */
  protected BaseModel(String componentName, InputStream in) throws IOException, InvalidFormatException {

    if (componentName == null)
      throw new IllegalArgumentException("componentName must not be null!");
    
    if (in == null)
        throw new IllegalArgumentException("in must not be null!");

    this.componentName = componentName;
    
    Map<String, Object> artifactMap = new HashMap<String, Object>();
    
    createArtifactSerializers(artifactSerializers);

    final ZipInputStream zip = new ZipInputStream(in);

    ZipEntry entry;
    while((entry = zip.getNextEntry()) != null ) {

      String extension = getEntryExtension(entry.getName());

      ArtifactSerializer factory = artifactSerializers.get(extension);

      if (factory == null) {
        throw new InvalidFormatException("Unkown artifact format: " + extension);
      }

      artifactMap.put(entry.getName(), factory.create(zip));

      zip.closeEntry();
    }

    this.artifactMap = Collections.unmodifiableMap(artifactMap);

    validateArtifactMap();
  }

  /**
   * Extracts the "." extension from an entry name.
   *
   * @param entry the entry name which contains the extension
   *
   * @return the extension
   *
   * @throws InvalidFormatException if no extension can be extracted
   */
  private String getEntryExtension(String entry) throws InvalidFormatException {
    int extensionIndex = entry.lastIndexOf('.') + 1;

    if (extensionIndex == -1 || extensionIndex >= entry.length())
        throw new InvalidFormatException("Entry name must have type extension: " + entry);

    return entry.substring(extensionIndex);
  }

  protected ArtifactSerializer getArtifactSerializer(String resoruceName) {
    String extension = null;
    try {
      extension = getEntryExtension(resoruceName);
    } catch (InvalidFormatException e) {
      throw new IllegalStateException(e);
    }

    return artifactSerializers.get(extension);  
  }
  
  /**
   * Registers all {@link ArtifactSerializer} for their artifact file name extensions.
   * The registered {@link ArtifactSerializer} are used to create and serialize
   * resources in the model package.
   * 
   * Override this method to register custom {@link ArtifactSerializer}s.
   *
   * Note:
   * Subclasses should generally invoke super.createArtifactSerializers at the beginning
   * of this method.
   * 
   * This method is called during construction.
   *
   * @param serializers the key of the map is the file extension used to lookup
   *     the {@link ArtifactSerializer}.
   */
  protected void createArtifactSerializers(
      Map<String, ArtifactSerializer> serializers) {
    GenericModelSerializer.register(serializers);
    PropertiesSerializer.register(serializers);
    DictionarySerializer.register(serializers);
  }
<commit_msg>OPENNLP-17 Added static method to create serializers<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreemnets.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


package opennlp.tools.util.model;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;

import opennlp.tools.util.InvalidFormatException;
import opennlp.tools.util.Version;

/**
 * This model is a common based which can be used by the components
 * model classes.
 * 
 * TODO:
 * Provide sub classes access to serializers already in constructor
 */
public abstract class BaseModel {

  protected static final String MANIFEST_ENTRY = "manifest.properties";
  
  private static final String MANIFEST_VERSION_PROPERTY = "Manifest-Version";
  private static final String COMPONENT_NAME_PROPERTY = "Component-Name";
  private static final String VERSION_PROPERTY = "OpenNLP-Version";
  private static final String TIMESTAMP_PROPERTY = "Timestamp";
  private static final String LANGUAGE_PROPERTY = "Language";
  
  public static final String TRAINING_CUTOFF_PROPERTY = "Training-Cutoff";
  public static final String TRAINING_ITERATIONS_PROPERTY = "Training-Iterations";
  public static final String TRAINING_EVENTHASH_PROPERTY = "Training-Eventhash";
  
  private Map<String, ArtifactSerializer> artifactSerializers =
      new HashMap<String, ArtifactSerializer>();

  protected final Map<String, Object> artifactMap;

  private final String componentName;
  
  /**
   * Initializes the current instance.
   *
   * @param languageCode
   * @param manifestInfoEntries additional information in the manifest
   */
  protected BaseModel(String componentName, String languageCode, Map<String, String> manifestInfoEntries) {

    if (componentName == null)
        throw new IllegalArgumentException("componentName must not be null!");
    
    if (languageCode == null)
        throw new IllegalArgumentException("languageCode must not be null!");

    this.componentName = componentName;
    
    artifactMap = new HashMap<String, Object>();
    
    createArtifactSerializers(artifactSerializers);
    
    Properties manifest = new Properties();
    manifest.setProperty(MANIFEST_VERSION_PROPERTY, "1.0");
    manifest.setProperty(LANGUAGE_PROPERTY, languageCode);
    manifest.setProperty(VERSION_PROPERTY, Version.currentVersion().toString());
    manifest.setProperty(TIMESTAMP_PROPERTY, 
        Long.toString(System.currentTimeMillis()));
    manifest.setProperty(COMPONENT_NAME_PROPERTY, componentName);
    
    if (manifestInfoEntries != null) {
      for (Map.Entry<String, String> entry : manifestInfoEntries.entrySet()) {
        manifest.setProperty(entry.getKey(), entry.getValue());
      }
    }
      
    artifactMap.put(MANIFEST_ENTRY, manifest);
  }

  /**
   * Initializes the current instance.
   *
   * @param in
   *
   * @throws IOException
   * @throws InvalidFormatException
   */
  protected BaseModel(String componentName, InputStream in) throws IOException, InvalidFormatException {

    if (componentName == null)
      throw new IllegalArgumentException("componentName must not be null!");
    
    if (in == null)
        throw new IllegalArgumentException("in must not be null!");

    this.componentName = componentName;
    
    Map<String, Object> artifactMap = new HashMap<String, Object>();
    
    createArtifactSerializers(artifactSerializers);

    final ZipInputStream zip = new ZipInputStream(in);

    ZipEntry entry;
    while((entry = zip.getNextEntry()) != null ) {

      String extension = getEntryExtension(entry.getName());

      ArtifactSerializer factory = artifactSerializers.get(extension);

      if (factory == null) {
        throw new InvalidFormatException("Unkown artifact format: " + extension);
      }

      artifactMap.put(entry.getName(), factory.create(zip));

      zip.closeEntry();
    }

    this.artifactMap = Collections.unmodifiableMap(artifactMap);

    validateArtifactMap();
  }

  /**
   * Extracts the "." extension from an entry name.
   *
   * @param entry the entry name which contains the extension
   *
   * @return the extension
   *
   * @throws InvalidFormatException if no extension can be extracted
   */
  private String getEntryExtension(String entry) throws InvalidFormatException {
    int extensionIndex = entry.lastIndexOf('.') + 1;

    if (extensionIndex == -1 || extensionIndex >= entry.length())
        throw new InvalidFormatException("Entry name must have type extension: " + entry);

    return entry.substring(extensionIndex);
  }

  protected ArtifactSerializer getArtifactSerializer(String resoruceName) {
    String extension = null;
    try {
      extension = getEntryExtension(resoruceName);
    } catch (InvalidFormatException e) {
      throw new IllegalStateException(e);
    }

    return artifactSerializers.get(extension);  
  }
  
  protected static Map<String, ArtifactSerializer> createArtifactSerializers() {
    Map<String, ArtifactSerializer> serializers = new HashMap<String, ArtifactSerializer>();
    
    GenericModelSerializer.register(serializers);
    PropertiesSerializer.register(serializers);
    DictionarySerializer.register(serializers);
    
    return serializers;
  }
  
  /**
   * Registers all {@link ArtifactSerializer} for their artifact file name extensions.
   * The registered {@link ArtifactSerializer} are used to create and serialize
   * resources in the model package.
   * 
   * Override this method to register custom {@link ArtifactSerializer}s.
   *
   * Note:
   * Subclasses should generally invoke super.createArtifactSerializers at the beginning
   * of this method.
   * 
   * This method is called during construction.
   *
   * @param serializers the key of the map is the file extension used to lookup
   *     the {@link ArtifactSerializer}.
   */
  protected void createArtifactSerializers(
      Map<String, ArtifactSerializer> serializers) {
    serializers.putAll(createArtifactSerializers());
  }

{'fraction_non_alphanumeric': 0.08467771912868932, 'fraction_numerical': 0.0013381904691101034, 'mean_word_length': 3.932893289328933, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7250191', 'n_tokens_mistral': 3534, 'n_tokens_neox': 3443, 'n_words': 1096}