<commit_before>'use strict';
// =========================================================================
//
// Routes for Projects
//
// =========================================================================
var policy     = require ('../policies/project.policy');
var Project    = require ('../controllers/project.controller');
var helpers    = require ('../../../core/server/controllers/core.helpers.controller');
var fs		   = require ('fs');
var CSVParse   = require('csv-parse');
var mongoose = require('mongoose');
var Model    = mongoose.model ('Project');
var _ = require ('lodash');

var loadProjects = function(file, req, res) {
	// Now parse and go through this thing.
	fs.readFile(file.path, 'utf8', function(err, data) {
		if (err) {
			return console.log(err);
		}
		// console.log("FILE DATA:",data);
		var colArray = ['id','ProjectName','Proponent','Region','shortD','locSpatial','locDescription','provincialED','federalED','capitalInvestment','projectCreateDate','projectDescriptionLivingData','tombstoneNote','projectURL','captialInvestmentNote','lat','long','constructionJobs','constructionJobsNotes','operationJobs','operationJobsNotes','sector','subSector','currentPhaseTypeActivity','active','CEAAInvolvement','eaIssues','environmentalAssessmentNotes','CEAA','firstNationsConsultation','firstNationsAccess','firstNationsNotification','stakeholdersNotes','federalAgencies','workingGroups','allOtherStakeholderGroups','responsibleEPD','projectLead','EAOCAARTRepresentative','projectOfficer','projectAnalyst','projectAssistant','administrativeAssistant','CELead','teamNotes'];
		var parse = new CSVParse(data, {delimiter: ',', columns: colArray}, function(err, output){
			// Skip this many rows
			var length = Object.keys(output).length;
			var projectProcessed = 0;
			console.log("length",length);
			Object.keys(output).forEach(function(key, index) {
				if (index > 0) {
					var row = output[key];
					// console.log("rowData:",row.id);
					Model.findOne({epicProjectID: row.id}, function (err, doc) {
						if (doc === null) {
							projectProcessed++;
							var p = new Project (req.user);
							p.new().then(function(model) {
								// console.log("MODEL:",model);
								// LATER
								model.epicProjectID = parseInt(row.id);
								// console.log("epicProjectID",model.epicProjectID);
								model.oldData = JSON.stringify(row);
								model.status = 'In Progress';
								model.name = row.ProjectName;
								model.description = row.shortD;
								model.code = model.name.toLowerCase ().replace (' ', '-').substr (0, model.name.length+1);
								model.phases = [];
								// _.each (phases, function (ph) {
								// 	var phase = new Phase (ph);
								// 	project.phases.push (phase._id);
								// 	phase.save ();
								// });
								model.type = 'Mine';
								model.region = row.Region;
								model.currentPhase = model.phases[0];
								if (row.lat) model.lat = parseFloat(row.lat);
								if (row.lat) model.lon = parseFloat(row.long);
								model.roles = ['mem', 'public'];
								//
								// cram in all other infor into the description
								//
								//var od = p.description;
								// project.description = 'Permit Number: '+p.permit+"\n";
								// project.description += 'Project Name: '+p.name+"\n";
								// project.description += 'Proponent/Operator: '+p.prop+"\n";
								// project.description += 'Ownership: '+p.ownership+"\n";
								// project.description += 'Latitude [decimal degrees N]: '+p.lat+"\n";
								// project.description += 'Longitude [decimal degrees E]: '+p.lon+"\n";
								// project.description += 'Status: '+p.status+"\n";
								// project.description += 'Commodity/ies: '+p.comm+"\n";
								// project.description += 'MEM Region: '+p.memRegion+"\n";
								// project.description += 'Tailings Impoundments: '+p.tail+"\n";
								model.description += 'Description: '+row.projectDescriptionLivingData+"\n";
								model.read = ['public'];
								model.submit = ['mem'];
								// console.log("epicProjectID",model.epicProjectID);
								// console.log("oldData",model.oldData);
								// model.fillmein = row.Proponent;
								// model.fillmein = row.Region;
								// model.fillmein = row.shortD;
								// model.fillmein = row.locSpatial;
								// model.fillmein = row.locDescription;
								// model.fillmein = row.provincialED;
								// model.fillmein = row.federalED;
								// model.fillmein = row.capitalInvestment;
								// model.fillmein = row.projectCreateDate;
								// model.fillmein = row.projectDescriptionLivingData;
								// model.fillmein = row.tombstoneNote;
								// model.fillmein = row.projectURL;
								// model.fillmein = row.captialInvestmentNote;
								// model.fillmein = row.lat;
								// model.fillmein = row.long;
								// model.fillmein = row.constructionJobs;
								// model.fillmein = row.constructionJobsNotes;
								// model.fillmein = row.operationJobs;
								// model.fillmein = row.operationJobsNotes;
								// model.fillmein = row.sector;
								// model.fillmein = row.subSector;<commit_msg>Jamming in all the meta into the description field<commit_after>'use strict';
// =========================================================================
//
// Routes for Projects
//
// =========================================================================
var policy     = require ('../policies/project.policy');
var Project    = require ('../controllers/project.controller');
var helpers    = require ('../../../core/server/controllers/core.helpers.controller');
var fs		   = require ('fs');
var CSVParse   = require('csv-parse');
var mongoose = require('mongoose');
var Model    = mongoose.model ('Project');
var _ = require ('lodash');

var loadProjects = function(file, req, res) {
	// Now parse and go through this thing.
	fs.readFile(file.path, 'utf8', function(err, data) {
		if (err) {
			return console.log(err);
		}
		// console.log("FILE DATA:",data);
		var colArray = ['id','ProjectName','Proponent','Region','shortD','locSpatial','locDescription','provincialED','federalED','capitalInvestment','projectCreateDate','projectDescriptionLivingData','tombstoneNote','projectURL','captialInvestmentNote','lat','long','constructionJobs','constructionJobsNotes','operationJobs','operationJobsNotes','sector','subSector','currentPhaseTypeActivity','active','CEAAInvolvement','eaIssues','environmentalAssessmentNotes','CEAA','firstNationsConsultation','firstNationsAccess','firstNationsNotification','stakeholdersNotes','federalAgencies','workingGroups','allOtherStakeholderGroups','responsibleEPD','projectLead','EAOCAARTRepresentative','projectOfficer','projectAnalyst','projectAssistant','administrativeAssistant','CELead','teamNotes'];
		var parse = new CSVParse(data, {delimiter: ',', columns: colArray}, function(err, output){
			// Skip this many rows
			var length = Object.keys(output).length;
			var projectProcessed = 0;
			console.log("length",length);
			Object.keys(output).forEach(function(key, index) {
				if (index > 0) {
					var row = output[key];
					// console.log("rowData:",row.id);
					Model.findOne({epicProjectID: row.id}, function (err, doc) {
						if (doc === null) {
							projectProcessed++;
							var p = new Project (req.user);
							p.new().then(function(model) {
								// console.log("MODEL:",model);
								// LATER
								model.epicProjectID = parseInt(row.id);
								// console.log("epicProjectID",model.epicProjectID);
								model.oldData = JSON.stringify(row);
								model.status = 'In Progress';
								model.name = row.ProjectName;
								model.description = row.shortD;
								model.code = model.name.toLowerCase ().replace (' ', '-').substr (0, model.name.length+1);
								model.phases = [];
								// _.each (phases, function (ph) {
								// 	var phase = new Phase (ph);
								// 	project.phases.push (phase._id);
								// 	phase.save ();
								// });
								model.type = row.sector;
								model.region = row.Region;
								model.currentPhase = model.phases[0];
								if (row.lat) model.lat = parseFloat(row.lat);
								if (row.lat) model.lon = parseFloat(row.long);
								model.roles = ['mem', 'public'];
								model.description = "";
								model.description += 'Location - Description: '+row.locDescription+"\n";
								model.description += 'Provincial Electoral Districts: '+row.provincialED+"\n";
								model.description += 'Capital Investment $M: '+row.capitalInvestment+"\n";
								model.description += 'Project File Creation Date: '+row.projectCreateDate+"\n";
								model.description += 'Construction Jobs: '+row.constructionJobs+"\n";
								model.description += 'Operation Jobs: '+row.operationJobs+"\n";
								model.description += 'Sub Sector: '+row.subSector+"\n";
								model.description += 'Active: '+row.active+"\n";
								model.description += 'CEAA Involvement (Fed EA Req. & Type): '+row.CEAAInvolvement+"\n";
								model.description += 'EA Issues: '+row.eaIssues+"\n";
								model.description += 'Environmental Assessment: '+row.environmentalAssessmentNotes+"\n";
								model.description += 'CEAA: '+row.CEAA+"\n";
								model.description += 'Description: '+row.projectDescriptionLivingData+"\n";
								model.description += 'firstNationsConsultation: '+row.firstNationsConsultation+"\n";
								model.description += 'firstNationsAccess: '+row.firstNationsAccess+"\n";
								model.description += 'firstNationsNotification: '+row.firstNationsNotification+"\n";
								model.description += 'stakeholdersNotes: '+row.stakeholdersNotes+"\n";
								model.description += 'federalAgencies: '+row.federalAgencies+"\n";
								model.description += 'workingGroups: '+row.workingGroups+"\n";
								model.description += 'allOtherStakeholderGroups: '+row.allOtherStakeholderGroups+"\n";
								model.description += 'responsibleEPD: '+row.responsibleEPD+"\n";
								model.description += 'projectLead: '+row.projectLead+"\n";
								model.description += 'EAOCAARTRepresentative: '+row.EAOCAARTRepresentative+"\n";
								model.description += 'projectOfficer: '+row.projectOfficer+"\n";
								model.description += 'projectAnalyst: '+row.projectAnalyst+"\n";
								model.description += 'projectAssistant: '+row.projectAssistant+"\n";
								model.description += 'administrativeAssistant: '+row.administrativeAssistant+"\n";
								model.description += 'CELead: '+row.CELead+"\n";
								model.description += 'teamNotes: '+row.teamNotes+"\n";
								model.read = ['public'];
								model.submit = ['mem'];
								// console.log("epicProjectID",model.epicProjectID);
								// console.log("oldData",model.oldData);
								// model.fillmein = row.Proponent;
								// model.fillmein = row.Region;
								// model.fillmein = row.shortD;
								// model.fillmein = row.locSpatial;
								// model.fillmein = row.locDescription;
								// model.fillmein = row.provincialED;
								// model.fillmein = row.federalED;
								// model.fillmein = row.capitalInvestment;
								// model.fillmein = row.projectCreateDate;
								// model.fillmein = row.projectDescriptionLivingData;
								// model.fillmein = row.tombstoneNote;
								// model.fillmein = row.projectURL;
								// model.fillmein = row.captialInvestmentNote;
								// model.fillmein = row.lat;
								// model.fillmein = row.long;
								// model.fillmein = row.constructionJobs;
								// model.fillmein = row.constructionJobsNotes;
								// model.fillmein = row.operationJobs;
								// model.fillmein = row.operationJobsNotes;
								// model.fillmein = row.sector;
								// model.fillmein = row.subSector;
{'fraction_non_alphanumeric': 0.1891984864121087, 'fraction_numerical': 0.0010319917440660474, 'mean_word_length': 3.2753676470588236, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1208900', 'n_tokens_mistral': 4578, 'n_tokens_neox': 3379, 'n_words': 569}