<commit_before>package uk.ac.ncl.openlab.intake24.sql.tools.food

import java.io.{File, FileWriter}

import com.opencsv.CSVWriter
import org.rogach.scallop.ScallopConf
import uk.ac.ncl.openlab.intake24.foodsql.admin.FoodsAdminImpl
import uk.ac.ncl.openlab.intake24.foodsql.foodindex.FoodIndexDataImpl
import uk.ac.ncl.openlab.intake24.foodsql.user.FoodDataServiceImpl
import uk.ac.ncl.openlab.intake24.sql.tools._

import scala.language.reflectiveCalls

object FoodListExport extends App with DatabaseConnection with WarningMessage {

  val options = new ScallopConf(args) {
    val dbConfigDir = opt[String](required = true)
    val outputFile = opt[String](required = true)
    val locale = opt[String](required = true)
  }

  options.verify()

  val databaseConfig = DatabaseConfigChooser.chooseDatabaseConfiguration(options.dbConfigDir())

  val dataSource = getDataSource(databaseConfig)

  val indexService = new FoodIndexDataImpl(dataSource)
  val foodsService = new FoodsAdminImpl(dataSource)

  val csvWriter = new CSVWriter(new FileWriter(new File(options.outputFile())))

  csvWriter.writeNext(Array("Intake24 code", "English description", "Local description", "Food composition table", "Food composition record ID", "Portion size estimation methods"))

  indexService.indexableFoods(options.locale()) match {
    case Right(foods) =>
      foods.sortBy(_.code).foreach {
        header =>

          println(header.code)

          foodsService.getFoodRecord(header.code, options.locale()) match {
            case Right(food) =>

              val compTables = food.local.nutrientTableCodes.toArray.flatMap(t => Array(t._1, t._2))


              val psm = food.local.portionSize.map {
                psm =>
                  (Seq(s"Method: ${psm.method}", s"conversion: ${psm.conversionFactor}") ++ psm.parameters.map( p => s"${p.name}: ${p.value}")).mkString(", ")
              }.mkString("\n")

              if (food.allowedInLocale(options.locale())) {
                val row = Array(food.main.code, food.main.englishDescription, food.local.localDescription.getOrElse("N/A")) ++ compTables :+ psm
                csvWriter.writeNext(row)
              }

            case Left(e) => e.exception.printStackTrace()
          }

      }
    case Left(e) => e.exception.printStackTrace()

  }

  csvWriter.close()

  println("Done!")
}<commit_msg>Add category list to food export<commit_after>package uk.ac.ncl.openlab.intake24.sql.tools.food

import java.io.{File, FileWriter}

import com.opencsv.CSVWriter
import org.rogach.scallop.ScallopConf
import uk.ac.ncl.openlab.intake24.foodsql.admin.FoodsAdminImpl
import uk.ac.ncl.openlab.intake24.foodsql.foodindex.FoodIndexDataImpl
import uk.ac.ncl.openlab.intake24.foodsql.user.{FoodBrowsingServiceImpl, FoodDataServiceImpl}
import uk.ac.ncl.openlab.intake24.sql.tools._

import scala.language.reflectiveCalls

object FoodListExport extends App with DatabaseConnection with WarningMessage {

  val options = new ScallopConf(args) {
    val dbConfigDir = opt[String](required = true)
    val outputFile = opt[String](required = true)
    val locale = opt[String](required = true)
  }

  options.verify()

  val databaseConfig = DatabaseConfigChooser.chooseDatabaseConfiguration(options.dbConfigDir())

  val dataSource = getDataSource(databaseConfig)

  val indexService = new FoodIndexDataImpl(dataSource)
  val foodsService = new FoodsAdminImpl(dataSource)

  val foodBrowsingService = new FoodBrowsingServiceImpl(dataSource)

  val csvWriter = new CSVWriter(new FileWriter(new File(options.outputFile())))

  csvWriter.writeNext(Array("Intake24 code", "English description", "Local description", "Food composition table", "Food composition record ID", "Portion size estimation methods", "Categories"))

  indexService.indexableFoods(options.locale()) match {
    case Right(foods) =>
      foods.sortBy(_.code).foreach {
        header =>

          println(header.code)

          foodsService.getFoodRecord(header.code, options.locale()) match {
            case Right(food) =>

              val (tableIds, recordIds) = food.local.nutrientTableCodes.toArray.unzip

              val categories = foodBrowsingService.getFoodAllCategories(header.code).right.get.toArray.sorted

              val psm = food.local.portionSize.map {
                psm =>
                  (Seq(s"Method: ${psm.method}", s"conversion: ${psm.conversionFactor}") ++ psm.parameters.map( p => s"${p.name}: ${p.value}")).mkString(", ")
              }.mkString("\n")

              if (food.allowedInLocale(options.locale())) {
                val row = Array(food.main.code, food.main.englishDescription, food.local.localDescription.getOrElse("N/A")) ++ Array(tableIds.mkString("\n"), recordIds.mkString("\n"), psm) ++ categories
                csvWriter.writeNext(row)
              }

            case Left(e) => e.exception.printStackTrace()
          }

      }
    case Left(e) => e.exception.printStackTrace()

  }

  csvWriter.close()

  println("Done!")
}
{'fraction_non_alphanumeric': 0.12295409181636727, 'fraction_numerical': 0.005189620758483034, 'mean_word_length': 3.474107142857143, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6004535', 'n_tokens_mistral': 1576, 'n_tokens_neox': 1472, 'n_words': 273}