<commit_before>﻿using System.Collections.Generic;
using System.Linq;
using System.Web;
using System.Web.Http;

using Hatfield.EnviroData.Core;
using Hatfield.EnviroData.DataAcquisition;
using Hatfield.EnviroData.DataAcquisition.CSV;
using Hatfield.EnviroData.DataAcquisition.ESDAT;
using Hatfield.EnviroData.DataAcquisition.ESDAT.Converters;
using Hatfield.EnviroData.DataAcquisition.XML;
using Hatfield.EnviroData.FileSystems;
using Hatfield.EnviroData.FileSystems.FTPFileSystem;
using Hatfield.EnviroData.FileSystems.HttpFileSystem;
using Hatfield.EnviroData.MVC.Helpers;
using Hatfield.EnviroData.MVC.Models;
using Hatfield.EnviroData.WQDataProfile;

namespace Hatfield.EnviroData.MVC.Controllers
{
    public class ESDATImportAPIController : ApiController
    {
        private static string HeaderFileInputElementName = "headerFileInput";
        private static string SampleFileInputElementName = "sampleFileInput";
        private static string ChemistryFileInputElementName = "chemistryFileInput";

        private IDbContext _dbContext;
        private IWQDefaultValueProvider _wqDefaultValueProvider;

        WayToHandleNewData wayToHandleNewData = WayToHandleNewData.CreateInstanceForNewData;

        public ESDATImportAPIController(IDbContext dbContext, IWQDefaultValueProvider wqDefaultValueProvider)
        {
            _dbContext = dbContext;
            _wqDefaultValueProvider = wqDefaultValueProvider;
        }

        [HttpPost]
        public IEnumerable<ResultMessageViewModel> ImportLocalFiles()
        {
            var results = new List<ResultMessageViewModel>();

            var request = HttpContext.Current.Request;

            XMLDataToImport headerFileXMLFileToImport = null;

            if (request.Files.AllKeys.Contains(HeaderFileInputElementName))
            {
                var headerFile = request.Files[HeaderFileInputElementName];
                var headerFileData = new DataFromFileSystem(headerFile.FileName, headerFile.InputStream);
                headerFileXMLFileToImport = new XMLDataToImport(headerFileData);
            }
            else
            {
                results.Add(new ResultMessageViewModel(ResultMessageViewModel.RESULT_LEVEL_WARN, "Header file is not uploaded, use default value."));
            }
            
            var sampleFile = request.Files[SampleFileInputElementName];
            var chemistryFile = request.Files[ChemistryFileInputElementName];

            var sampleFileData = new DataFromFileSystem(sampleFile.FileName, sampleFile.InputStream);
            var sampleFileCSVFileToImport = new CSVDataToImport(sampleFileData);

            var chemistryFileData = new DataFromFileSystem(chemistryFile.FileName, chemistryFile.InputStream);
            var chemistryCSVFileToImport = new CSVDataToImport(chemistryFileData);

            var esdatDataToImport = new ESDATDataToImport(headerFileXMLFileToImport, sampleFileCSVFileToImport, chemistryCSVFileToImport);
            var importer = ESDATDataImportHelper.BuildESDATDataImporter(_wqDefaultValueProvider);

            results.AddRange(PersistESDATData(esdatDataToImport, importer));

            return results;
        }

        [HttpPost]
        public IEnumerable<ResultMessageViewModel> ImportHttpFiles(HttpFileImportDataViewModel data)
        {
            XMLDataToImport headerFileXMLFileToImport = null;

            if (!string.IsNullOrEmpty(data.HeaderFileURL))
            {
                var headerFileHttpFileSystem = new HttpFileSystem(data.HeaderFileURL);
                headerFileXMLFileToImport = new XMLDataToImport(headerFileHttpFileSystem.FetchData());
            }
            

            var sampleFileHttpFileSystem = new HttpFileSystem(data.SampleFileURL);
            var sampleFileCSVFileToImport = new CSVDataToImport(sampleFileHttpFileSystem.FetchData());

            var chemistryFileHttpFileSystem = new HttpFileSystem(data.ChemistryFileURL);
            var chemistryCSVFileToImport = new CSVDataToImport(chemistryFileHttpFileSystem.FetchData());

            var esdatDataToImport = new ESDATDataToImport(headerFileXMLFileToImport, sampleFileCSVFileToImport, chemistryCSVFileToImport);
            var importer = ESDATDataImportHelper.BuildESDATDataImporter(_wqDefaultValueProvider);

            var results = PersistESDATData(esdatDataToImport, importer);

            return results;
        }

        [HttpPost]
        public IEnumerable<ResultMessageViewModel> ImportFtpFiles(FtpFileImportDataViewModel data)
        {
            XMLDataToImport headerFileXMLFileToImport = null;

            if (!string.IsNullOrEmpty(data.HeaderFileURL))
            {
                var headerFileHttpFileSystem = new FTPFileSystem(data.HeaderFileURL);
                headerFileXMLFileToImport = new XMLDataToImport(headerFileHttpFileSystem.FetchData());
            }

            var sampleFileFtpFileSystem = new FTPFileSystem(data.SampleFileURL, data.UserName, data.Password);
            var sampleFileCSVFileToImport = new CSVDataToImport(sampleFileFtpFileSystem.FetchData());

            var chemistryFileFtpFileSystem = new FTPFileSystem(data.ChemistryFileURL, data.UserName, data.Password);
            var chemistryCSVFileToImport = new CSVDataToImport(chemistryFileFtpFileSystem.FetchData());

            var esdatDataToImport = new ESDATDataToImport(headerFileXMLFileToImport, sampleFileCSVFileToImport, chemistryCSVFileToImport);
            var importer = ESDATDataImportHelper.BuildESDATDataImporter(_wqDefaultValueProvider);

            var results = PersistESDATData(esdatDataToImport, importer);

            return results;
        }

        private IEnumerable<ResultMessageViewModel> PersistESDATData(ESDATDataToImport esdatDataToImport, IDataImporter importer)
        {
            var extractedResults = importer.Extract<ESDATModel>(esdatDataToImport);

            if (!extractedResults.IsExtractedSuccess)
            {
                var failResults = from parsingResult in ImportResultHelper.FilterWarningAndErrorResult(extractedResults.AllParsingResults)
                                  select new ResultMessageViewModel
                                  (
                                      parsingResult.Level.ToString(),
                                      parsingResult.Message
                                  );

                return failResults;
            }
            else
            {
                var esdatModel = extractedResults.ExtractedEntities.First();

                var duplicateChecker = new ESDATDuplicateChecker(_dbContext);
                var sampleCollectionFactory = new ESDATSampleCollectionMapperFactory(duplicateChecker, _wqDefaultValueProvider, wayToHandleNewData);


                var chemistryFactory = new ESDATChemistryMapperFactory(duplicateChecker, _wqDefaultValueProvider, wayToHandleNewData);

                var mapper = new SampleCollectionActionMapper(duplicateChecker, sampleCollectionFactory, _wqDefaultValueProvider, chemistryFactory, wayToHandleNewData);

                var converter = new ESDATConverter(mapper);

                var action = converter.Convert(esdatModel);

                _dbContext.Add<Hatfield.EnviroData.Core.Action>(action);
                _dbContext.SaveChanges();

                return new List<ResultMessageViewModel> {
                    new ResultMessageViewModel
                    (
                        ResultMessageViewModel.RESULT_LEVEL_INFO,
                        "Import success"
                    )
                };
            }
        }
    }
}<commit_msg>return all the result to view for the import api<commit_after>﻿using System.Collections.Generic;
using System.Linq;
using System.Web;
using System.Web.Http;

using Hatfield.EnviroData.Core;
using Hatfield.EnviroData.DataAcquisition;
using Hatfield.EnviroData.DataAcquisition.CSV;
using Hatfield.EnviroData.DataAcquisition.ESDAT;
using Hatfield.EnviroData.DataAcquisition.ESDAT.Converters;
using Hatfield.EnviroData.DataAcquisition.XML;
using Hatfield.EnviroData.FileSystems;
using Hatfield.EnviroData.FileSystems.FTPFileSystem;
using Hatfield.EnviroData.FileSystems.HttpFileSystem;
using Hatfield.EnviroData.MVC.Helpers;
using Hatfield.EnviroData.MVC.Models;
using Hatfield.EnviroData.WQDataProfile;

namespace Hatfield.EnviroData.MVC.Controllers
{
    public class ESDATImportAPIController : ApiController
    {
        private static string HeaderFileInputElementName = "headerFileInput";
        private static string SampleFileInputElementName = "sampleFileInput";
        private static string ChemistryFileInputElementName = "chemistryFileInput";

        private IDbContext _dbContext;
        private IWQDefaultValueProvider _wqDefaultValueProvider;

        WayToHandleNewData wayToHandleNewData = WayToHandleNewData.CreateInstanceForNewData;

        public ESDATImportAPIController(IDbContext dbContext, IWQDefaultValueProvider wqDefaultValueProvider)
        {
            _dbContext = dbContext;
            _wqDefaultValueProvider = wqDefaultValueProvider;
        }

        [HttpPost]
        public IEnumerable<ResultMessageViewModel> ImportLocalFiles()
        {
            var results = new List<ResultMessageViewModel>();

            var request = HttpContext.Current.Request;

            XMLDataToImport headerFileXMLFileToImport = null;

            if (request.Files.AllKeys.Contains(HeaderFileInputElementName))
            {
                var headerFile = request.Files[HeaderFileInputElementName];
                var headerFileData = new DataFromFileSystem(headerFile.FileName, headerFile.InputStream);
                headerFileXMLFileToImport = new XMLDataToImport(headerFileData);
            }
            else
            {
                results.Add(new ResultMessageViewModel(ResultMessageViewModel.RESULT_LEVEL_WARN, "Header file is not uploaded, use default value."));
            }
            
            var sampleFile = request.Files[SampleFileInputElementName];
            var chemistryFile = request.Files[ChemistryFileInputElementName];

            var sampleFileData = new DataFromFileSystem(sampleFile.FileName, sampleFile.InputStream);
            var sampleFileCSVFileToImport = new CSVDataToImport(sampleFileData);

            var chemistryFileData = new DataFromFileSystem(chemistryFile.FileName, chemistryFile.InputStream);
            var chemistryCSVFileToImport = new CSVDataToImport(chemistryFileData);

            var esdatDataToImport = new ESDATDataToImport(headerFileXMLFileToImport, sampleFileCSVFileToImport, chemistryCSVFileToImport);
            var importer = ESDATDataImportHelper.BuildESDATDataImporter(_wqDefaultValueProvider);

            results.AddRange(PersistESDATData(esdatDataToImport, importer));

            return results;
        }

        [HttpPost]
        public IEnumerable<ResultMessageViewModel> ImportHttpFiles(HttpFileImportDataViewModel data)
        {
            XMLDataToImport headerFileXMLFileToImport = null;

            if (!string.IsNullOrEmpty(data.HeaderFileURL))
            {
                var headerFileHttpFileSystem = new HttpFileSystem(data.HeaderFileURL);
                headerFileXMLFileToImport = new XMLDataToImport(headerFileHttpFileSystem.FetchData());
            }
            

            var sampleFileHttpFileSystem = new HttpFileSystem(data.SampleFileURL);
            var sampleFileCSVFileToImport = new CSVDataToImport(sampleFileHttpFileSystem.FetchData());

            var chemistryFileHttpFileSystem = new HttpFileSystem(data.ChemistryFileURL);
            var chemistryCSVFileToImport = new CSVDataToImport(chemistryFileHttpFileSystem.FetchData());

            var esdatDataToImport = new ESDATDataToImport(headerFileXMLFileToImport, sampleFileCSVFileToImport, chemistryCSVFileToImport);
            var importer = ESDATDataImportHelper.BuildESDATDataImporter(_wqDefaultValueProvider);

            var results = PersistESDATData(esdatDataToImport, importer);

            return results;
        }

        [HttpPost]
        public IEnumerable<ResultMessageViewModel> ImportFtpFiles(FtpFileImportDataViewModel data)
        {
            XMLDataToImport headerFileXMLFileToImport = null;

            if (!string.IsNullOrEmpty(data.HeaderFileURL))
            {
                var headerFileHttpFileSystem = new FTPFileSystem(data.HeaderFileURL);
                headerFileXMLFileToImport = new XMLDataToImport(headerFileHttpFileSystem.FetchData());
            }

            var sampleFileFtpFileSystem = new FTPFileSystem(data.SampleFileURL, data.UserName, data.Password);
            var sampleFileCSVFileToImport = new CSVDataToImport(sampleFileFtpFileSystem.FetchData());

            var chemistryFileFtpFileSystem = new FTPFileSystem(data.ChemistryFileURL, data.UserName, data.Password);
            var chemistryCSVFileToImport = new CSVDataToImport(chemistryFileFtpFileSystem.FetchData());

            var esdatDataToImport = new ESDATDataToImport(headerFileXMLFileToImport, sampleFileCSVFileToImport, chemistryCSVFileToImport);
            var importer = ESDATDataImportHelper.BuildESDATDataImporter(_wqDefaultValueProvider);

            var results = PersistESDATData(esdatDataToImport, importer);

            return results;
        }

        private IEnumerable<ResultMessageViewModel> PersistESDATData(ESDATDataToImport esdatDataToImport, IDataImporter importer)
        {
            var extractedResults = importer.Extract<ESDATModel>(esdatDataToImport);

            if (!extractedResults.IsExtractedSuccess)
            {
                var failResults = from parsingResult in ImportResultHelper.FilterWarningAndErrorResult(extractedResults.AllParsingResults)
                                  select new ResultMessageViewModel
                                  (
                                      parsingResult.Level.ToString(),
                                      parsingResult.Message
                                  );

                return failResults;
            }
            else
            {
                var esdatModel = extractedResults.ExtractedEntities.First();

                var allResults = from parsingResult in extractedResults.AllParsingResults
                                 select new ResultMessageViewModel(parsingResult.Level.ToString(), parsingResult.Message);

                var duplicateChecker = new ESDATDuplicateChecker(_dbContext);
                var sampleCollectionFactory = new ESDATSampleCollectionMapperFactory(duplicateChecker, _wqDefaultValueProvider, wayToHandleNewData);


                var chemistryFactory = new ESDATChemistryMapperFactory(duplicateChecker, _wqDefaultValueProvider, wayToHandleNewData);

                var mapper = new SampleCollectionActionMapper(duplicateChecker, sampleCollectionFactory, _wqDefaultValueProvider, chemistryFactory, wayToHandleNewData);

                var converter = new ESDATConverter(mapper);

                var action = converter.Convert(esdatModel);

                _dbContext.Add<Hatfield.EnviroData.Core.Action>(action);
                _dbContext.SaveChanges();

                var resultList = allResults.ToList();
                resultList.Add(
                    new ResultMessageViewModel
                    (
                        ResultMessageViewModel.RESULT_LEVEL_INFO,
                        "Import success"
                    )
                );
                return resultList;
            }
        }
    }
}
{'fraction_non_alphanumeric': 0.05968032998195411, 'fraction_numerical': 0.0, 'mean_word_length': 2.7535074987905177, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '141299', 'n_tokens_mistral': 3931, 'n_tokens_neox': 3799, 'n_words': 652}