<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements. See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations under the License.
 */
package org.apache.shindig.gadgets.rewrite;

import org.apache.shindig.common.uri.Uri;
import org.apache.shindig.common.util.Utf8UrlCoder;
import org.apache.shindig.common.xml.DomUtil;
import org.apache.shindig.gadgets.Gadget;
import org.apache.shindig.gadgets.http.HttpRequest;
import org.apache.shindig.gadgets.http.HttpResponse;
import org.apache.shindig.gadgets.servlet.ProxyBase;
import org.apache.shindig.gadgets.spec.View;

import com.google.common.base.Nullable;
import com.google.common.base.Predicate;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.inject.Inject;
import com.google.inject.name.Named;

import org.w3c.dom.Element;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;

import java.io.StringReader;
import java.io.StringWriter;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
 * Perform rewriting of HTML content including
 * - Concatenating & proxying of referred script content
 * - Concatenating & proxying of stylesheet links
 * - Moving all style into head and converting @imports into links
 * - Proxying referred content of images and embeds
 */
public class HTMLContentRewriter  implements ContentRewriter {
  private final static int MAX_URL_LENGTH = 1500;
  private static final String DEFAULT_CONCAT_URL_BASE = "/gadgets/concat?";

  public final static Set<String> TAGS =
      Sets.newHashSet("img", "embed", "link", "script", "style");
                                                                                       
  private final static Map<String, Set<String>> LINKING_TAG_ATTRS = Maps.newHashMap();

  static {
    LINKING_TAG_ATTRS.put("img", Sets.newHashSet("src"));
    LINKING_TAG_ATTRS.put("embed", Sets.newHashSet("src"));
  }

  private final ContentRewriterFeatureFactory rewriterFeatureFactory;
  private final String proxyBaseNoGadget;
  private final String concatBaseNoGadget;

  @Inject
  public HTMLContentRewriter(ContentRewriterFeatureFactory rewriterFeatureFactory,
      @Named("shindig.content-rewrite.proxy-url")String proxyBaseNoGadget,
      @Named("shindig.content-rewrite.concat-url")String concatBaseNoGadget) {
    this.rewriterFeatureFactory = rewriterFeatureFactory;
    if (concatBaseNoGadget != null) {
      this.concatBaseNoGadget = concatBaseNoGadget;
    } else {
      this.concatBaseNoGadget = DEFAULT_CONCAT_URL_BASE;
    }
    this.proxyBaseNoGadget = proxyBaseNoGadget;
  }

  public RewriterResults rewrite(HttpRequest request, HttpResponse original,
      MutableContent content) {
    if (RewriterUtils.isHtml(request, original)) {
      ContentRewriterFeature feature = rewriterFeatureFactory.get(request);
      return rewriteImpl(feature, request.getGadget(), request.getUri(), content);
    }
    return null;
  }

  public RewriterResults rewrite(Gadget gadget, MutableContent content) {
    ContentRewriterFeature feature = rewriterFeatureFactory.get(gadget.getSpec());
    Uri contentBase = gadget.getSpec().getUrl();
    View view = gadget.getCurrentView();
    if (view != null && view.getHref() != null) {
      contentBase = view.getHref();
    }
    return rewriteImpl(feature, gadget.getSpec().getUrl(), contentBase, content);
  }

  protected RewriterResults rewriteImpl(ContentRewriterFeature feature, Uri gadgetUri,
                                        Uri contentBase, MutableContent content) {
    if (!feature.isRewriteEnabled() || content.getDocument() == null) {
      return null;
    }

    // Get ALL interesting tags
    List<Element> tagList =
        DomUtil.getElementsByTagNameCaseInsensitive(content.getDocument(), TAGS);<commit_msg>Removed useless null check. Only @Optional bindings can ever be null in Guice-provided objects.<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements. See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations under the License.
 */
package org.apache.shindig.gadgets.rewrite;

import org.apache.shindig.common.uri.Uri;
import org.apache.shindig.common.util.Utf8UrlCoder;
import org.apache.shindig.common.xml.DomUtil;
import org.apache.shindig.gadgets.Gadget;
import org.apache.shindig.gadgets.http.HttpRequest;
import org.apache.shindig.gadgets.http.HttpResponse;
import org.apache.shindig.gadgets.servlet.ProxyBase;
import org.apache.shindig.gadgets.spec.View;

import com.google.common.base.Nullable;
import com.google.common.base.Predicate;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.inject.Inject;
import com.google.inject.name.Named;

import org.w3c.dom.Element;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;

import java.io.StringReader;
import java.io.StringWriter;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
 * Perform rewriting of HTML content including
 * - Concatenating & proxying of referred script content
 * - Concatenating & proxying of stylesheet links
 * - Moving all style into head and converting @imports into links
 * - Proxying referred content of images and embeds
 */
public class HTMLContentRewriter  implements ContentRewriter {
  private final static int MAX_URL_LENGTH = 1500;

  public final static Set<String> TAGS =
      Sets.newHashSet("img", "embed", "link", "script", "style");

  private final static Map<String, Set<String>> LINKING_TAG_ATTRS = Maps.newHashMap();

  static {
    LINKING_TAG_ATTRS.put("img", Sets.newHashSet("src"));
    LINKING_TAG_ATTRS.put("embed", Sets.newHashSet("src"));
  }

  private final ContentRewriterFeatureFactory rewriterFeatureFactory;
  private final String proxyBaseNoGadget;
  private final String concatBaseNoGadget;

  @Inject
  public HTMLContentRewriter(ContentRewriterFeatureFactory rewriterFeatureFactory,
      @Named("shindig.content-rewrite.proxy-url")String proxyBaseNoGadget,
      @Named("shindig.content-rewrite.concat-url")String concatBaseNoGadget) {
    this.rewriterFeatureFactory = rewriterFeatureFactory;
    this.concatBaseNoGadget = concatBaseNoGadget;
    this.proxyBaseNoGadget = proxyBaseNoGadget;
  }

  public RewriterResults rewrite(HttpRequest request, HttpResponse original,
      MutableContent content) {
    if (RewriterUtils.isHtml(request, original)) {
      ContentRewriterFeature feature = rewriterFeatureFactory.get(request);
      return rewriteImpl(feature, request.getGadget(), request.getUri(), content);
    }
    return null;
  }

  public RewriterResults rewrite(Gadget gadget, MutableContent content) {
    ContentRewriterFeature feature = rewriterFeatureFactory.get(gadget.getSpec());
    Uri contentBase = gadget.getSpec().getUrl();
    View view = gadget.getCurrentView();
    if (view != null && view.getHref() != null) {
      contentBase = view.getHref();
    }
    return rewriteImpl(feature, gadget.getSpec().getUrl(), contentBase, content);
  }

  protected RewriterResults rewriteImpl(ContentRewriterFeature feature, Uri gadgetUri,
                                        Uri contentBase, MutableContent content) {
    if (!feature.isRewriteEnabled() || content.getDocument() == null) {
      return null;
    }

    // Get ALL interesting tags
    List<Element> tagList =
        DomUtil.getElementsByTagNameCaseInsensitive(content.getDocument(), TAGS);
{'fraction_non_alphanumeric': 0.09416603528520402, 'fraction_numerical': 0.0025976837320056284, 'mean_word_length': 4.273972602739726, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5567340', 'n_tokens_mistral': 2716, 'n_tokens_neox': 2654, 'n_words': 704}