<commit_before>/* $Id: IRepositoryConnector.java 996524 2010-09-13 13:38:01Z kwright $ */

/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.manifoldcf.crawler.interfaces;

import org.apache.manifoldcf.core.interfaces.*;
import org.apache.manifoldcf.agents.interfaces.*;

import java.io.*;
import java.util.*;

/** This interface describes an instance of a connection between a repository and ManifoldCF's
* standard "pull" ingestion agent.
*
* Each instance of this interface is used in only one thread at a time.  Connection Pooling
* on these kinds of objects is performed by the factory which instantiates repository connectors
* from symbolic names and config parameters, and is pooled by these parameters.  That is, a pooled connector
* handle is used only if all the connection parameters for the handle match.
*
* Implementers of this interface should provide a default constructor which has this signature:
*
* xxx();
*
* Connectors are either configured or not.  If configured, they will persist in a pool, and be
* reused multiple times.  Certain methods of a connector may be called before the connector is
* configured.  This includes basically all methods that permit inspection of the connector's
* capabilities.  The complete list is:
*
*
* The purpose of the repository connector is to allow documents to be fetched from the repository.
*
* Each repository connector describes a set of documents that are known only to that connector.
* It therefore establishes a space of document identifiers.  Each connector will only ever be
* asked to deal with identifiers that have in some way originated from the connector.
*
* Documents are fetched by ManifoldCF in two stages.  First, the addSeedDocuments() method is called in the connector
* implementation.  This method is meant to add a set of document identifiers to the queue.  When ManifoldCF is ready
* to process a document, the document identifier is used to build a version string for the document and check whether
* the document needs to be indexed, and index it if needed (the second stage).  The second stage
* consists of the processDocuments() method.
*
* All of these methods interact with ManifoldCF by means of an "activity" interface.
*/
public interface IRepositoryConnector extends IConnector
{
  public static final String _rcsid = "@(#)$Id: IRepositoryConnector.java 996524 2010-09-13 13:38:01Z kwright $";

  // Connector models.
  // These values describe what the connector returns for the getDocumentIdentifiers() method.  The framework
  // uses these to figure out how to most efficiently use the connector.  It is desirable to pick a model that
  // is the most restrictive that is still accurate.  For example, if MODEL_ADD_CHANGE_DELETE applies, you would
  // return that value rather than MODEL_ADD.

  // For the CHAINED models, what the connector is describing are the documents that will be processed IF the seeded
  // documents are followed to their leaves.  For instance, imagine a hierarchy where the root document is the only one ever
  // seeded, but if that document is processed, and its discovered changed children are processed as well, then all documents
  // that have been added, changed, or deleted will eventually be discovered.  In that case, model
  // MODEL_CHAINED_ADD_CHANGE_DELETE would be appropriate.  But, if a changed node can only discover child
  // additions and changes, then MODEL_CHAINED_ADD_CHANGE would be the right choice.

  /** This is the legacy ManifoldCF catch-all crawling model.  All existing documents will be rechecked when a crawl
  * is done, every time.  This model was typically used for models where seeds were essentially fixed and all
  * real documents were discovered during crawling. */
  public static final int MODEL_ALL = 0;
  /** This indicates that the seeds are never complete; the previous seeds are lost and cannot be retrieved. */
  public static final int MODEL_PARTIAL = 4;

  /** Supply at least the documents that have been added since the specified start time.  Connector is
  * aware of the start time and end time of the request, and supplies at least the documents that have been
  * added within the specified time range. */
  public static final int MODEL_ADD = 1;
  /** Supply at least the documents that have been added or changed within the specified time range. */
  public static final int MODEL_ADD_CHANGE = 2;
  /** Supply at least the documents that have been added, changed, or deleted within the specified time range. */
  public static final int MODEL_ADD_CHANGE_DELETE = 3;

  /** Like MODEL_ADD, except considering document discovery */
  public static final int MODEL_CHAINED_ADD = 9;
  /** Like MODEL_ADD_CHANGE, except considering document discovery */
  public static final int MODEL_CHAINED_ADD_CHANGE = 10;
  /** Like MODEL_ADD_CHANGE_DELETE, except considering document discovery */<commit_msg>Update comment to make CHAINED model reference extraction requirements clear<commit_after>/* $Id: IRepositoryConnector.java 996524 2010-09-13 13:38:01Z kwright $ */

/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.manifoldcf.crawler.interfaces;

import org.apache.manifoldcf.core.interfaces.*;
import org.apache.manifoldcf.agents.interfaces.*;

import java.io.*;
import java.util.*;

/** This interface describes an instance of a connection between a repository and ManifoldCF's
* standard "pull" ingestion agent.
*
* Each instance of this interface is used in only one thread at a time.  Connection Pooling
* on these kinds of objects is performed by the factory which instantiates repository connectors
* from symbolic names and config parameters, and is pooled by these parameters.  That is, a pooled connector
* handle is used only if all the connection parameters for the handle match.
*
* Implementers of this interface should provide a default constructor which has this signature:
*
* xxx();
*
* Connectors are either configured or not.  If configured, they will persist in a pool, and be
* reused multiple times.  Certain methods of a connector may be called before the connector is
* configured.  This includes basically all methods that permit inspection of the connector's
* capabilities.  The complete list is:
*
*
* The purpose of the repository connector is to allow documents to be fetched from the repository.
*
* Each repository connector describes a set of documents that are known only to that connector.
* It therefore establishes a space of document identifiers.  Each connector will only ever be
* asked to deal with identifiers that have in some way originated from the connector.
*
* Documents are fetched by ManifoldCF in two stages.  First, the addSeedDocuments() method is called in the connector
* implementation.  This method is meant to add a set of document identifiers to the queue.  When ManifoldCF is ready
* to process a document, the document identifier is used to build a version string for the document and check whether
* the document needs to be indexed, and index it if needed (the second stage).  The second stage
* consists of the processDocuments() method.
*
* All of these methods interact with ManifoldCF by means of an "activity" interface.
*
* A note on connector models:
*
* These values describe what the connector returns for the addSeedDocuments() method.  The framework
* uses these to figure out how to most efficiently use the connector.  It is desirable to pick a model that
* is the most restrictive that is still accurate.  For example, if MODEL_ADD_CHANGE_DELETE applies, you would
* return that value rather than MODEL_ADD.
*
* For the CHAINED models, what the connector is describing are the documents that will be processed IF the seeded
* documents are followed to their leaves.  For instance, imagine a hierarchy where the root document is the only one ever
* seeded, but if that document is processed, and its discovered changed children are processed as well, then all documents
* that have been added, changed, or deleted will eventually be discovered.  In that case, model
* MODEL_CHAINED_ADD_CHANGE_DELETE would be appropriate.  But, if a changed node can only discover child
* additions and changes, then MODEL_CHAINED_ADD_CHANGE would be the right choice.
*	
* A CHAINED model also requires cooperation on the part of the connector for processing.  Specifically,
* a document may be unchanged but its references are expected to still be extracted in order for a CHAINED
* model to do the right thing.  For non-CHAINED models, re-extraction of references if there are no reference changes
* for a document is NOT required.
*/
public interface IRepositoryConnector extends IConnector
{
  public static final String _rcsid = "@(#)$Id: IRepositoryConnector.java 996524 2010-09-13 13:38:01Z kwright $";

  /** This is the legacy ManifoldCF catch-all crawling model.  All existing documents will be rechecked when a crawl
  * is done, every time.  This model was typically used for models where seeds were essentially fixed and all
  * real documents were discovered during crawling. */
  public static final int MODEL_ALL = 0;
  /** This indicates that the seeds are never complete; the previous seeds are lost and cannot be retrieved. */
  public static final int MODEL_PARTIAL = 4;

  /** Supply at least the documents that have been added since the specified start time.  Connector is
  * aware of the start time and end time of the request, and supplies at least the documents that have been
  * added within the specified time range. */
  public static final int MODEL_ADD = 1;
  /** Supply at least the documents that have been added or changed within the specified time range. */
  public static final int MODEL_ADD_CHANGE = 2;
  /** Supply at least the documents that have been added, changed, or deleted within the specified time range. */
  public static final int MODEL_ADD_CHANGE_DELETE = 3;

  /** Like MODEL_ADD, except considering document discovery */
  public static final int MODEL_CHAINED_ADD = 9;
  /** Like MODEL_ADD_CHANGE, except considering document discovery */
  public static final int MODEL_CHAINED_ADD_CHANGE = 10;
  /** Like MODEL_ADD_CHANGE_DELETE, except considering document discovery */
{'fraction_non_alphanumeric': 0.05224391915039397, 'fraction_numerical': 0.008907159986296678, 'mean_word_length': 4.317395264116575, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3026520', 'n_tokens_mistral': 3000, 'n_tokens_neox': 2796, 'n_words': 1630}