<commit_before>// Copyright (C) 2014 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.googlesource.gerrit.plugins.xdocs;

import static java.nio.charset.StandardCharsets.UTF_8;

import com.google.common.base.MoreObjects;
import com.google.common.base.Strings;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.Weigher;
import com.google.common.collect.Maps;
import com.google.gerrit.extensions.annotations.PluginName;
import com.google.gerrit.httpd.resources.Resource;
import com.google.gerrit.httpd.resources.SmallResource;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.cache.CacheModule;
import com.google.gerrit.server.config.CanonicalWebUrl;
import com.google.gerrit.server.config.PluginConfigFactory;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.Singleton;

import com.googlesource.gerrit.plugins.xdocs.formatter.Formatters;
import com.googlesource.gerrit.plugins.xdocs.formatter.Formatters.FormatterProvider;

import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.diff.RawText;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectLoader;
import org.eclipse.jgit.lib.ObjectReader;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevTree;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.treewalk.TreeWalk;
import org.eclipse.jgit.treewalk.filter.PathFilter;

import java.io.IOException;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@Singleton
public class XDocLoader extends CacheLoader<String, Resource> {
  private static final String DEFAULT_HOST = "review.example.com";

  private final GitRepositoryManager repoManager;
  private final Provider<String> webUrl;
  private final String pluginName;
  private final PluginConfigFactory cfgFactory;
  private final Formatters formatters;

  @Inject
  XDocLoader(GitRepositoryManager repoManager,
      @CanonicalWebUrl Provider<String> webUrl,
      @PluginName String pluginName,
      PluginConfigFactory cfgFactory,
      Formatters formatters) {
    this.repoManager = repoManager;
    this.webUrl = webUrl;
    this.pluginName = pluginName;
    this.cfgFactory = cfgFactory;
    this.formatters = formatters;
  }

  @Override
  public Resource load(String strKey) throws Exception {
    XDocResourceKey key = XDocResourceKey.fromString(strKey);
    XDocGlobalConfig cfg =
        new XDocGlobalConfig(cfgFactory.getGlobalPluginConfig(pluginName));
    FormatterProvider formatter = formatters.getByName(key.getFormatter());
    if (formatter == null) {
      return Resource.NOT_FOUND;
    }
    ConfigSection formatterCfg = cfg.getFormatterConfig(formatter.getName());
    Repository repo = repoManager.openRepository(key.getProject());
    try {
      RevWalk rw = new RevWalk(repo);
      try {
        ObjectId revId = key.getRevId();
        if (revId == null) {
          return Resource.NOT_FOUND;
        }
        RevCommit commit = rw.parseCommit(revId);
        RevTree tree = commit.getTree();
        TreeWalk tw = new TreeWalk(repo);
        try {
          tw.addTree(tree);
          tw.setRecursive(true);
          tw.setFilter(PathFilter.create(key.getResource()));
          if (!tw.next()) {
            return Resource.NOT_FOUND;
          }
          ObjectId objectId = tw.getObjectId(0);
          ObjectLoader loader = repo.open(objectId);
          byte[] bytes = loader.getBytes(Integer.MAX_VALUE);
          if (formatter.getName().equals(Formatters.RAW_FORMATTER)
              && RawText.isBinary(bytes)) {
            return Resources.METHOD_NOT_ALLOWED;
          }
          ObjectReader reader = repo.newObjectReader();
          try {
            String abbrRevId = reader.abbreviate(revId).name();
            String html =
                formatter.get().format(key.getProject().get(),
                    abbrRevId, formatterCfg,
                    replaceMacros(repo, key.getProject(), revId,
                        abbrRevId, bytes));
            return getAsHtmlResource(html, commit.getCommitTime());
          } finally {
            reader.release();
          }
        } finally {
          tw.release();
        }
      } finally {
        rw.release();
      }
    } finally {
      repo.close();
    }
  }

  private String replaceMacros(Repository repo, Project.NameKey project,
      ObjectId revId, String abbrRevId, byte[] raw) throws GitAPIException,
      IOException {
    Map<String, String> macros = Maps.newHashMap();

    String url = webUrl.get();
    if (Strings.isNullOrEmpty(url)) {
      url = "http://" + DEFAULT_HOST + "/";
    }
    macros.put("URL", url);

    macros.put("PROJECT", project.get());
    macros.put("PROJECT_URL", url + "#/admin/projects/" + project.get());
    macros.put("REVISION", abbrRevId);
    macros.put("GIT_DESCRIPTION", MoreObjects.firstNonNull(
        (new Git(repo)).describe().setTarget(revId).call(), abbrRevId));


    Matcher m = Pattern.compile("(\\\\)?@([A-Z_]+)@")
        .matcher(new String(raw, UTF_8));
    StringBuffer sb = new StringBuffer();
    while (m.find()) {
      String key = m.group(2);
      String val = macros.get(key);
      if (m.group(1) != null || val == null) {
        m.appendReplacement(sb, "@" + key + "@");
      } else {
        m.appendReplacement(sb, val);
      }
    }
    m.appendTail(sb);
    return sb.toString();
  }<commit_msg>Never replace macros in binary files<commit_after>// Copyright (C) 2014 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.googlesource.gerrit.plugins.xdocs;

import static java.nio.charset.StandardCharsets.UTF_8;

import com.google.common.base.MoreObjects;
import com.google.common.base.Strings;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.Weigher;
import com.google.common.collect.Maps;
import com.google.gerrit.extensions.annotations.PluginName;
import com.google.gerrit.httpd.resources.Resource;
import com.google.gerrit.httpd.resources.SmallResource;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.cache.CacheModule;
import com.google.gerrit.server.config.CanonicalWebUrl;
import com.google.gerrit.server.config.PluginConfigFactory;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.Singleton;

import com.googlesource.gerrit.plugins.xdocs.formatter.Formatters;
import com.googlesource.gerrit.plugins.xdocs.formatter.Formatters.FormatterProvider;

import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.diff.RawText;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectLoader;
import org.eclipse.jgit.lib.ObjectReader;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevTree;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.treewalk.TreeWalk;
import org.eclipse.jgit.treewalk.filter.PathFilter;

import java.io.IOException;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@Singleton
public class XDocLoader extends CacheLoader<String, Resource> {
  private static final String DEFAULT_HOST = "review.example.com";

  private final GitRepositoryManager repoManager;
  private final Provider<String> webUrl;
  private final String pluginName;
  private final PluginConfigFactory cfgFactory;
  private final Formatters formatters;

  @Inject
  XDocLoader(GitRepositoryManager repoManager,
      @CanonicalWebUrl Provider<String> webUrl,
      @PluginName String pluginName,
      PluginConfigFactory cfgFactory,
      Formatters formatters) {
    this.repoManager = repoManager;
    this.webUrl = webUrl;
    this.pluginName = pluginName;
    this.cfgFactory = cfgFactory;
    this.formatters = formatters;
  }

  @Override
  public Resource load(String strKey) throws Exception {
    XDocResourceKey key = XDocResourceKey.fromString(strKey);
    XDocGlobalConfig cfg =
        new XDocGlobalConfig(cfgFactory.getGlobalPluginConfig(pluginName));
    FormatterProvider formatter = formatters.getByName(key.getFormatter());
    if (formatter == null) {
      return Resource.NOT_FOUND;
    }
    ConfigSection formatterCfg = cfg.getFormatterConfig(formatter.getName());
    Repository repo = repoManager.openRepository(key.getProject());
    try {
      RevWalk rw = new RevWalk(repo);
      try {
        ObjectId revId = key.getRevId();
        if (revId == null) {
          return Resource.NOT_FOUND;
        }
        RevCommit commit = rw.parseCommit(revId);
        RevTree tree = commit.getTree();
        TreeWalk tw = new TreeWalk(repo);
        try {
          tw.addTree(tree);
          tw.setRecursive(true);
          tw.setFilter(PathFilter.create(key.getResource()));
          if (!tw.next()) {
            return Resource.NOT_FOUND;
          }
          ObjectId objectId = tw.getObjectId(0);
          ObjectLoader loader = repo.open(objectId);
          byte[] bytes = loader.getBytes(Integer.MAX_VALUE);
          boolean isBinary = RawText.isBinary(bytes);
          if (formatter.getName().equals(Formatters.RAW_FORMATTER) && isBinary) {
            return Resources.METHOD_NOT_ALLOWED;
          }
          ObjectReader reader = repo.newObjectReader();
          try {
            String abbrRevId = reader.abbreviate(revId).name();
            String raw = new String(bytes, UTF_8);
            if (!isBinary) {
              raw = replaceMacros(repo, key.getProject(), revId, abbrRevId, raw);
            }
            String html =
                formatter.get().format(key.getProject().get(),
                    abbrRevId, formatterCfg, raw);
            return getAsHtmlResource(html, commit.getCommitTime());
          } finally {
            reader.release();
          }
        } finally {
          tw.release();
        }
      } finally {
        rw.release();
      }
    } finally {
      repo.close();
    }
  }

  private String replaceMacros(Repository repo, Project.NameKey project,
      ObjectId revId, String abbrRevId, String raw) throws GitAPIException,
      IOException {
    Map<String, String> macros = Maps.newHashMap();

    String url = webUrl.get();
    if (Strings.isNullOrEmpty(url)) {
      url = "http://" + DEFAULT_HOST + "/";
    }
    macros.put("URL", url);

    macros.put("PROJECT", project.get());
    macros.put("PROJECT_URL", url + "#/admin/projects/" + project.get());
    macros.put("REVISION", abbrRevId);
    macros.put("GIT_DESCRIPTION", MoreObjects.firstNonNull(
        (new Git(repo)).describe().setTarget(revId).call(), abbrRevId));


    Matcher m = Pattern.compile("(\\\\)?@([A-Z_]+)@").matcher(raw);
    StringBuffer sb = new StringBuffer();
    while (m.find()) {
      String key = m.group(2);
      String val = macros.get(key);
      if (m.group(1) != null || val == null) {
        m.appendReplacement(sb, "@" + key + "@");
      } else {
        m.appendReplacement(sb, val);
      }
    }
    m.appendTail(sb);
    return sb.toString();
  }
{'fraction_non_alphanumeric': 0.11512706736587333, 'fraction_numerical': 0.0020976200080677692, 'mean_word_length': 3.182186234817814, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2143773', 'n_tokens_mistral': 3710, 'n_tokens_neox': 3611, 'n_words': 796}