<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.nutch.indexer.links;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.crawl.Inlink;
import org.apache.nutch.crawl.Inlinks;
import org.apache.nutch.indexer.IndexingException;
import org.apache.nutch.indexer.IndexingFilter;
import org.apache.nutch.indexer.NutchDocument;
import org.apache.nutch.parse.Outlink;
import org.apache.nutch.parse.Parse;
import org.slf4j.LoggerFactory;

import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;

/**
 * An {@link org.apache.nutch.indexer.IndexingFilter} that adds
 * <code>outlinks</code> and <code>inlinks</code> field(s) to the document.
 *
 * In case that you want to ignore the outlinks that point to the same host
 * as the URL being indexed use the following settings in your configuration
 * file:
 *
 * <property>
 *   <name>outlinks.host.ignore</name>
 *   <value>true</value>
 * </property>
 *
 * The same configuration is available for inlinks:
 *
 * <property>
 *   <name>inlinks.host.ignore</name>
 *   <value>true</value>
 * </property>
 *
 * To store only the host portion of each inlink URL or outlink URL add the
 * following to your configuration file.
 *
 * <property>
 *   <name>links.hosts.only</name>
 *   <value>false</value>
 * </property>
 *
 */
public class LinksIndexingFilter implements IndexingFilter {

  public final static String LINKS_OUTLINKS_HOST = "index.links.outlinks.host.ignore";
  public final static String LINKS_INLINKS_HOST = "index.links.inlinks.host.ignore";
  public final static String LINKS_ONLY_HOSTS = "index.links.hosts.only";

  public final static org.slf4j.Logger LOG = LoggerFactory
      .getLogger(LinksIndexingFilter.class);

  private Configuration conf;
  private boolean filterOutlinks;
  private boolean filterInlinks;
  private boolean indexHost;

  @Override
  public NutchDocument filter(NutchDocument doc, Parse parse, Text url,
      CrawlDatum datum, Inlinks inlinks) throws IndexingException {

    // Add the outlinks
    Outlink[] outlinks = parse.getData().getOutlinks();

    if (outlinks != null) {
      Set<String> hosts = new HashSet<String>();

      for (Outlink outlink : outlinks) {
        try {<commit_msg>Updating javadoc for the index-links plugin, this closes #78<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.nutch.indexer.links;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.crawl.Inlink;
import org.apache.nutch.crawl.Inlinks;
import org.apache.nutch.indexer.IndexingException;
import org.apache.nutch.indexer.IndexingFilter;
import org.apache.nutch.indexer.NutchDocument;
import org.apache.nutch.parse.Outlink;
import org.apache.nutch.parse.Parse;
import org.slf4j.LoggerFactory;

import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;

/**
 * An {@link org.apache.nutch.indexer.IndexingFilter} that adds
 * <code>outlinks</code> and <code>inlinks</code> field(s) to the document.
 *
 * In case that you want to ignore the outlinks that point to the same host
 * as the URL being indexed use the following settings in your configuration
 * file:
 *
 * <property>
 *   <name>index.links.outlinks.host.ignore</name>
 *   <value>true</value>
 * </property>
 *
 * The same configuration is available for inlinks:
 *
 * <property>
 *   <name>index.links.inlinks.host.ignore</name>
 *   <value>true</value>
 * </property>
 *
 * To store only the host portion of each inlink URL or outlink URL add the
 * following to your configuration file.
 *
 * <property>
 *   <name>index.links.hosts.only</name>
 *   <value>false</value>
 * </property>
 *
 */
public class LinksIndexingFilter implements IndexingFilter {

  public final static String LINKS_OUTLINKS_HOST = "index.links.outlinks.host.ignore";
  public final static String LINKS_INLINKS_HOST = "index.links.inlinks.host.ignore";
  public final static String LINKS_ONLY_HOSTS = "index.links.hosts.only";

  public final static org.slf4j.Logger LOG = LoggerFactory
      .getLogger(LinksIndexingFilter.class);

  private Configuration conf;
  private boolean filterOutlinks;
  private boolean filterInlinks;
  private boolean indexHost;

  @Override
  public NutchDocument filter(NutchDocument doc, Parse parse, Text url,
      CrawlDatum datum, Inlinks inlinks) throws IndexingException {

    // Add the outlinks
    Outlink[] outlinks = parse.getData().getOutlinks();

    if (outlinks != null) {
      Set<String> hosts = new HashSet<String>();

      for (Outlink outlink : outlinks) {
        try {
{'fraction_non_alphanumeric': 0.0965257035057381, 'fraction_numerical': 0.0022009118063197612, 'mean_word_length': 4.159772911597729, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4674286', 'n_tokens_mistral': 1877, 'n_tokens_neox': 1836, 'n_words': 610}