<commit_before>/** Copyright 2015 TappingStone, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

package io.prediction.tools.console

import io.prediction.controller.Utils

import grizzled.slf4j.Logging
import org.apache.commons.io.FileUtils
import org.json4s._
import org.json4s.native.JsonMethods._
import org.json4s.native.Serialization.{ read, write }
import scalaj.http._

import scala.io.Source
import scala.sys.process._

import java.io.BufferedInputStream
import java.io.BufferedOutputStream
import java.io.File
import java.io.FileInputStream
import java.io.FileOutputStream
import java.net.ConnectException
import java.net.URI
import java.util.zip.ZipInputStream

case class TemplateArgs(
  directory: String = "",
  repository: String = "",
  name: Option[String] = None,
  packageName: Option[String] = None,
  email: Option[String] = None)

case class GitHubTag(
  name: String,
  zipball_url: String,
  tarball_url: String,
  commit: GitHubCommit)

case class GitHubCommit(
  sha: String,
  url: String)

case class GitHubCache(
  headers: Map[String, String],
  body: String)

case class TemplateEntry(
  repo: String)

object Template extends Logging {
  implicit val formats = Utils.json4sDefaultFormats

  /** Creates a wrapper that provides the functionality of scalaj.http.Http()
    * with automatic proxy settings handling. The proxy settings will first
    * come from "git" followed by system properties "http.proxyHost" and
    * "http.proxyPort".
    *
    * @param url URL to be connected
    * @return
    */
  def httpOptionalProxy(url: String): HttpRequest = {
    val gitProxy = try {
      Some(Process("git config --global http.proxy").lines.toList(0))
    } catch {
      case e: Throwable => None
    }

    val (host, port) = gitProxy map { p =>
      val proxyUri = new URI(p)
      (Option(proxyUri.getHost),
        if (proxyUri.getPort == -1) None else Some(proxyUri.getPort))
    } getOrElse {
      (sys.props.get("http.proxyHost"),
        sys.props.get("http.proxyPort").map { p =>
          try {
            Some(p.toInt)
          } catch {
            case e: NumberFormatException => None
          }
        } getOrElse None)
    }

    (host, port) match {
      case (Some(h), Some(p)) => Http(url).proxy(h, p)
      case _ => Http(url)
    }
  }

  def getGitHubRepos(
      repos: Seq[String],
      apiType: String,
      repoFilename: String): Map[String, GitHubCache] = {
    val reposCache = try {
      val cache =
        Source.fromFile(repoFilename)(scala.io.Codec.ISO8859).mkString
        read[Map[String, GitHubCache]](cache)
    } catch {
      case e: Throwable => Map[String, GitHubCache]()
    }
    val newReposCache = reposCache ++ (try {
      repos.map { repo =>
        val url = s"https://api.github.com/repos/$repo/$apiType"
        val http = httpOptionalProxy(url)
        val response = reposCache.get(repo).map { cache =>
          cache.headers.get("ETag").map { etag =>
            http.header("If-None-Match", etag).asString
          } getOrElse {
            http.asString
          }
        } getOrElse {
          http.asString
        }

        val body = if (response.code == 304) {
          reposCache(repo).body
        } else {
          response.body
        }

        repo -> GitHubCache(headers = response.headers, body = body)
      }.toMap
    } catch {
      case e: ConnectException =>
        githubConnectErrorMessage(e)
        Map()
    })
    FileUtils.writeStringToFile(
      new File(repoFilename),
      write(newReposCache),
      "ISO-8859-1")
    newReposCache
  }

  def sub(repo: String, name: String, email: String, org: String): Unit = {
    val data = Map(
      "repo" -> repo,
      "name" -> name,
      "email" -> email,
      "org" -> org)
    try {
      httpOptionalProxy("http://update.prediction.io/templates.subscribe").
        postData("json=" + write(data)).asString
    } catch {
      case e: Throwable => error("Unable to subscribe.")
    }
  }

  def meta(repo: String, name: String, org: String): Unit = {
    try {
      httpOptionalProxy(
        s"http://meta.prediction.io/templates/$repo/$org/$name").asString
    } catch {
      case e: Throwable => debug("Template metadata unavailable.")
    }
  }

  def list(ca: ConsoleArgs): Int = {
    val templatesUrl = "http://templates.prediction.io/index.json"
    try {
      val templatesJson = Source.fromURL(templatesUrl).mkString("")
      val templates = read[List[TemplateEntry]](templatesJson)
      println("The following is a list of template IDs officially recognized " +
        "by PredictionIO:")
      println()
      templates.foreach { template =><commit_msg>PDIO-608 Sort "pio template list" output<commit_after>/** Copyright 2015 TappingStone, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

package io.prediction.tools.console

import io.prediction.controller.Utils

import grizzled.slf4j.Logging
import org.apache.commons.io.FileUtils
import org.json4s._
import org.json4s.native.JsonMethods._
import org.json4s.native.Serialization.{ read, write }
import scalaj.http._

import scala.io.Source
import scala.sys.process._

import java.io.BufferedInputStream
import java.io.BufferedOutputStream
import java.io.File
import java.io.FileInputStream
import java.io.FileOutputStream
import java.net.ConnectException
import java.net.URI
import java.util.zip.ZipInputStream

case class TemplateArgs(
  directory: String = "",
  repository: String = "",
  name: Option[String] = None,
  packageName: Option[String] = None,
  email: Option[String] = None)

case class GitHubTag(
  name: String,
  zipball_url: String,
  tarball_url: String,
  commit: GitHubCommit)

case class GitHubCommit(
  sha: String,
  url: String)

case class GitHubCache(
  headers: Map[String, String],
  body: String)

case class TemplateEntry(
  repo: String)

object Template extends Logging {
  implicit val formats = Utils.json4sDefaultFormats

  /** Creates a wrapper that provides the functionality of scalaj.http.Http()
    * with automatic proxy settings handling. The proxy settings will first
    * come from "git" followed by system properties "http.proxyHost" and
    * "http.proxyPort".
    *
    * @param url URL to be connected
    * @return
    */
  def httpOptionalProxy(url: String): HttpRequest = {
    val gitProxy = try {
      Some(Process("git config --global http.proxy").lines.toList(0))
    } catch {
      case e: Throwable => None
    }

    val (host, port) = gitProxy map { p =>
      val proxyUri = new URI(p)
      (Option(proxyUri.getHost),
        if (proxyUri.getPort == -1) None else Some(proxyUri.getPort))
    } getOrElse {
      (sys.props.get("http.proxyHost"),
        sys.props.get("http.proxyPort").map { p =>
          try {
            Some(p.toInt)
          } catch {
            case e: NumberFormatException => None
          }
        } getOrElse None)
    }

    (host, port) match {
      case (Some(h), Some(p)) => Http(url).proxy(h, p)
      case _ => Http(url)
    }
  }

  def getGitHubRepos(
      repos: Seq[String],
      apiType: String,
      repoFilename: String): Map[String, GitHubCache] = {
    val reposCache = try {
      val cache =
        Source.fromFile(repoFilename)(scala.io.Codec.ISO8859).mkString
        read[Map[String, GitHubCache]](cache)
    } catch {
      case e: Throwable => Map[String, GitHubCache]()
    }
    val newReposCache = reposCache ++ (try {
      repos.map { repo =>
        val url = s"https://api.github.com/repos/$repo/$apiType"
        val http = httpOptionalProxy(url)
        val response = reposCache.get(repo).map { cache =>
          cache.headers.get("ETag").map { etag =>
            http.header("If-None-Match", etag).asString
          } getOrElse {
            http.asString
          }
        } getOrElse {
          http.asString
        }

        val body = if (response.code == 304) {
          reposCache(repo).body
        } else {
          response.body
        }

        repo -> GitHubCache(headers = response.headers, body = body)
      }.toMap
    } catch {
      case e: ConnectException =>
        githubConnectErrorMessage(e)
        Map()
    })
    FileUtils.writeStringToFile(
      new File(repoFilename),
      write(newReposCache),
      "ISO-8859-1")
    newReposCache
  }

  def sub(repo: String, name: String, email: String, org: String): Unit = {
    val data = Map(
      "repo" -> repo,
      "name" -> name,
      "email" -> email,
      "org" -> org)
    try {
      httpOptionalProxy("http://update.prediction.io/templates.subscribe").
        postData("json=" + write(data)).asString
    } catch {
      case e: Throwable => error("Unable to subscribe.")
    }
  }

  def meta(repo: String, name: String, org: String): Unit = {
    try {
      httpOptionalProxy(
        s"http://meta.prediction.io/templates/$repo/$org/$name").asString
    } catch {
      case e: Throwable => debug("Template metadata unavailable.")
    }
  }

  def list(ca: ConsoleArgs): Int = {
    val templatesUrl = "http://templates.prediction.io/index.json"
    try {
      val templatesJson = Source.fromURL(templatesUrl).mkString("")
      val templates = read[List[TemplateEntry]](templatesJson)
      println("The following is a list of template IDs registered on " +
        "PredictionIO Template Gallery:")
      println()
      templates.sortBy(_.repo.toLowerCase).foreach { template =>
{'fraction_non_alphanumeric': 0.11502392344497607, 'fraction_numerical': 0.005454545454545455, 'mean_word_length': 2.6890222379103426, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 41, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1791675', 'n_tokens_mistral': 3192, 'n_tokens_neox': 3094, 'n_words': 848}