<commit_before>/*
 * Copyright 2016 HM Revenue & Customs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package controllers

import java.io._

import com.typesafe.config.ConfigFactory
import play.api.libs.iteratee.Enumerator
import services.addressimporter.converter.{Extractor, CSVLine}
import services.addressimporter.ftpdownloader.{RealWorldIO, FtpDownloader}
import uk.gov.hmrc.play.microservice.controller.BaseController
import uk.gov.hmrc.play.http.logging.MdcLoggingExecutionContext._
import play.api.mvc._
import scala.concurrent.{ExecutionContext, Future}
import scala.util.{Failure, Success, Try}


object OrdnanceSurveyController extends OrdnanceSurveyController

trait OrdnanceSurveyController extends Controller {

  import services.addressimporter.converter.Extractor
  import scala.concurrent.ExecutionContext.Implicits.global


  def manualImport(): Action[AnyContent] = Action {

    val out: PipedOutputStream = new PipedOutputStream()

    val data: InputStream = new BufferedInputStream(new PipedInputStream(out))

    val dataContent: Enumerator[Array[Byte]] = Enumerator.fromStream(data).andThen(Enumerator.eof)


    Future {

      Try {
        val conf = ConfigFactory.load()

        // os server details
        val server = conf.getString("app.os.server")
        val port = conf.getString("app.os.port")
        val user = conf.getString("app.os.user")
        val pass = conf.getString("app.os.pass")
        val osHomeFolder = conf.getString("app.os.homeFolder")

        // temp folder details
        val tmpZipfilesHome = conf.getString("app.temp.folder")


        val ftpClient = new FtpDownloader with RealWorldIO
        ftpClient.FileCreationDelay = 1

        ftpClient.login(user, pass, server).map { result =>
          ftpClient.ftpDownload(osHomeFolder, tmpZipfilesHome) match {
            case Success((count, subFolder)) =>

              val csvOut = (line: CSVLine) => {
                // scalastyle:off
                out.write(line.toString.getBytes)
                out.write("\n".getBytes)
              }
              val result = Extractor.extract(new File(tmpZipfilesHome + "/" + subFolder), csvOut)
<commit_msg>Add an extra flush to fix not flushing at the end of run<commit_after>/*
 * Copyright 2016 HM Revenue & Customs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package controllers

import java.io._

import com.typesafe.config.ConfigFactory
import play.api.libs.iteratee.Enumerator
import services.addressimporter.converter.{Extractor, CSVLine}
import services.addressimporter.ftpdownloader.{RealWorldIO, FtpDownloader}
import uk.gov.hmrc.play.microservice.controller.BaseController
import uk.gov.hmrc.play.http.logging.MdcLoggingExecutionContext._
import play.api.mvc._
import scala.concurrent.{ExecutionContext, Future}
import scala.util.{Failure, Success, Try}


object OrdnanceSurveyController extends OrdnanceSurveyController

trait OrdnanceSurveyController extends Controller {

  import services.addressimporter.converter.Extractor
  import scala.concurrent.ExecutionContext.Implicits.global


  def manualImport(): Action[AnyContent] = Action {

    val out: PipedOutputStream = new PipedOutputStream()

    val data: InputStream = new BufferedInputStream(new PipedInputStream(out))

    val dataContent: Enumerator[Array[Byte]] = Enumerator.fromStream(data).andThen(Enumerator.eof)


    Future {

      Try {
        val conf = ConfigFactory.load()

        // os server details
        val server = conf.getString("app.os.server")
        val port = conf.getString("app.os.port")
        val user = conf.getString("app.os.user")
        val pass = conf.getString("app.os.pass")
        val osHomeFolder = conf.getString("app.os.homeFolder")

        // temp folder details
        val tmpZipfilesHome = conf.getString("app.temp.folder")


        val ftpClient = new FtpDownloader with RealWorldIO
        ftpClient.FileCreationDelay = 1

        ftpClient.login(user, pass, server).map { result =>
          ftpClient.ftpDownload(osHomeFolder, tmpZipfilesHome) match {
            case Success((count, subFolder)) =>

              val csvOut = (line: CSVLine) => {
                // scalastyle:off
                out.write(line.toString.getBytes)
                out.write("\n".getBytes)
                out.flush()
              }
              val result = Extractor.extract(new File(tmpZipfilesHome + "/" + subFolder), csvOut)

{'fraction_non_alphanumeric': 0.0916100605838076, 'fraction_numerical': 0.003304571323664402, 'mean_word_length': 3.2863886703383165, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7231492', 'n_tokens_mistral': 1547, 'n_tokens_neox': 1465, 'n_words': 417}