<commit_before>package es.weso.wiFetcher.dao.poi

import java.io.InputStream

import scala.collection.immutable.List
import scala.collection.mutable.ListBuffer

import org.apache.log4j.Logger
import org.apache.poi.hssf.util.{ CellReference => CellRef }
import org.apache.poi.ss.usermodel.FormulaEvaluator
import org.apache.poi.ss.usermodel.Sheet
import org.apache.poi.ss.usermodel.Workbook
import org.apache.poi.ss.usermodel.WorkbookFactory

import es.weso.wiFetcher.configuration.Configuration
import es.weso.wiFetcher.dao.ObservationDAO
import es.weso.wiFetcher.entities.Area
import es.weso.wiFetcher.entities.Computation
import es.weso.wiFetcher.entities.Country
import es.weso.wiFetcher.entities.Dataset
import es.weso.wiFetcher.entities.Indicator
import es.weso.wiFetcher.entities.Observation
import es.weso.wiFetcher.entities.ObservationStatus
import es.weso.wiFetcher.fetchers.SpreadsheetsFetcher
import es.weso.wiFetcher.utils.POIUtils
import es.weso.wiFetcher.utils.IssueManagerUtils

/**
 * This class contains the implementation that allows extract all information
 * about the observations of the Web Index
 *
 * This implementation is temporary because Web Foundation or TAS still has not
 * give us the final structure of the sheet that contains the observations.
 * Maybe the implementation has to change.
 *
 */
class SecondaryObservationDAOImpl(
  is: InputStream)(implicit val sFetcher: SpreadsheetsFetcher)
  extends ObservationDAO with PoiDAO[Observation] {

  import SecondaryObservationDAOImpl._

  private val observations: ListBuffer[Observation] = ListBuffer.empty

  load(is)

  protected def load(is: InputStream) {

    val datasets = sFetcher.getDatasets

    if (datasets == null) {
      sFetcher.issueManager.addError(
        message = "The datasets are not loaded. It is mandatory to load " +
          "the datasets in order to process the Observations",
        path = XslxFile)
    } else {

      logger.info("Begin observations extraction")
      val workbook: Workbook = WorkbookFactory.create(is)
      checkSheets(workbook)
      val obs = for {
        dataset <- datasets
        sheet = obtainSheet(workbook, dataset.id)
      } yield {
        println(dataset.id)
        if (!sheet.isDefined) {
          sFetcher.issueManager.addError(
            message = s"The dataset ${dataset.id} are invalid or empty. It is " + 
        	  "mandatory to have data " +
              "within the datasets in order to process the Observations",
            path = XslxFile)
          List.empty
        } else parseData(workbook, sheet.get)
      }
      observations ++= obs.foldLeft(ListBuffer[Observation]())((a, b) => a ++= b)
      logger.info("Finish observations extraction")
    }
  }
  
  protected def obtainSheet(workbook : Workbook, datasetId : String) : Option[Sheet] = {<commit_msg>Add changes to avoid dataset comprobation before parsing data<commit_after>package es.weso.wiFetcher.dao.poi

import java.io.InputStream

import scala.collection.immutable.List
import scala.collection.mutable.ListBuffer

import org.apache.log4j.Logger
import org.apache.poi.hssf.util.{ CellReference => CellRef }
import org.apache.poi.ss.usermodel.FormulaEvaluator
import org.apache.poi.ss.usermodel.Sheet
import org.apache.poi.ss.usermodel.Workbook
import org.apache.poi.ss.usermodel.WorkbookFactory

import es.weso.wiFetcher.configuration.Configuration
import es.weso.wiFetcher.dao.ObservationDAO
import es.weso.wiFetcher.entities.Area
import es.weso.wiFetcher.entities.Computation
import es.weso.wiFetcher.entities.Country
import es.weso.wiFetcher.entities.Dataset
import es.weso.wiFetcher.entities.Indicator
import es.weso.wiFetcher.entities.Observation
import es.weso.wiFetcher.entities.ObservationStatus
import es.weso.wiFetcher.fetchers.SpreadsheetsFetcher
import es.weso.wiFetcher.utils.POIUtils
import es.weso.wiFetcher.utils.IssueManagerUtils

/**
 * This class contains the implementation that allows extract all information
 * about the observations of the Web Index
 *
 * This implementation is temporary because Web Foundation or TAS still has not
 * give us the final structure of the sheet that contains the observations.
 * Maybe the implementation has to change.
 *
 */
class SecondaryObservationDAOImpl(
  is: InputStream)(implicit val sFetcher: SpreadsheetsFetcher)
  extends ObservationDAO with PoiDAO[Observation] {

  import SecondaryObservationDAOImpl._

  private val observations: ListBuffer[Observation] = ListBuffer.empty

  load(is)

  protected def load(is: InputStream) {

    val datasets = sFetcher.getDatasets

    if (datasets == null) {
      sFetcher.issueManager.addError(
        message = "The datasets are not loaded. It is mandatory to load " +
          "the datasets in order to process the Observations",
        path = XslxFile)
    } else {

      logger.info("Begin observations extraction")
      val workbook: Workbook = WorkbookFactory.create(is)
      checkSheets(workbook)
      val obs = for {
        dataset <- datasets
        sheet = obtainSheet(workbook, dataset.id)
      } yield {
        println(dataset.id)
        if (!sheet.isDefined) 
          List.empty
        else parseData(workbook, sheet.get)
      }
      observations ++= obs.foldLeft(ListBuffer[Observation]())((a, b) => a ++= b)
      logger.info("Finish observations extraction")
    }
  }
  
  protected def obtainSheet(workbook : Workbook, datasetId : String) : Option[Sheet] = {
{'fraction_non_alphanumeric': 0.08676470588235294, 'fraction_numerical': 0.0003676470588235294, 'mean_word_length': 3.764448336252189, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6613669', 'n_tokens_mistral': 1646, 'n_tokens_neox': 1696, 'n_words': 402}