<commit_before>/*
 Copyright [2016] [Relevance Lab]
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 http://www.apache.org/licenses/LICENSE-2.0
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */

var logger = require('_pr/logger')(module);
var resourceService = module.exports = {};
var appConfig = require('_pr/config');
var Cryptography = require('_pr/lib/utils/cryptography');
var aws = require('aws-sdk');
var resources = require('_pr/model/resources/resources');
var CW = require('_pr/lib/cloudwatch.js');
var S3 = require('_pr/lib/s3.js');
var EC2 = require('_pr/lib/ec2.js');
var RDS = require('_pr/lib/rds.js');
var resourceCost = require('_pr/model/resource-costs');
var csv = require("fast-csv");
var fs = require('fs');
var async = require('async');
var dateUtil = require('_pr/lib/utils/dateUtil');
var unassignedInstancesModel = require('_pr/model/unassigned-instances');
var unManagedInstancesModel = require('_pr/model/unmanaged-instance');
var instancesModel = require('_pr/model/classes/instance/instance');
var entityCosts = require('_pr/model/entity-costs');
var mongoDbClient = require('mongodb').MongoClient;

resourceService.getCostForResources = getCostForResources_deprecated;
resourceService.getTotalCost = getTotalCost_deprecated;
resourceService.getCostForServices = getCostForServices_deprecated;
resourceService.getEC2InstanceUsageMetrics=getEC2InstanceUsageMetrics;
resourceService.getS3BucketsMetrics=getS3BucketsMetrics;
resourceService.getBucketsInfo=getBucketsInfo;
resourceService.getResources=getResources;
resourceService.getRDSInstancesInfo=getRDSInstancesInfo;
resourceService.getRDSDBInstanceMetrics=getRDSDBInstanceMetrics;
resourceService.bulkUpdateResourceProviderTags=bulkUpdateResourceProviderTags;
resourceService.bulkUpdateUnassignedResourceTags=bulkUpdateUnassignedResourceTags;
resourceService.bulkUpdateAWSResourcesTags=bulkUpdateAWSResourcesTags;
resourceService.getEC2InstancesInfo=getEC2InstancesInfo;
resourceService.getAllResourcesForProvider =  getAllResourcesForProvider;
resourceService.updateAWSResourceCostsFromCSV = updateAWSResourceCostsFromCSV
resourceService.aggregateEntityCosts = aggregateEntityCosts

// @TODO To be cached if needed. In memory data will not exceed 200MB for upto 2000 instances.
function getAllResourcesForProvider(provider, next) {
    async.parallel([
            function(callback) {
                instancesModel.getInstanceByProviderId(provider._id, callback);
            },
            function(callback) {
                //@TODO Duplicate function of  getByProviderId, to be cleaned up
                unManagedInstancesModel.getInstanceByProviderId(provider._id, callback);
            },
            function(callback) {
                unassignedInstancesModel.getUnAssignedInstancesByProviderId(provider._id, callback);
            }
            /*function(callback) {
                resources.getResourcesByProviderId(provider._id, callback);
            }*/
        ],
        function(err, results) {
            if (err) {
                var err = new Error('Internal server error');
                err.status = 500;
                next(err)
            } else {
                var resultsArray = [].concat.apply([], results);
                var resultsObject = resultsArray.reduce(function(temp, current) {
                    if('platformId' in current) {
                        temp[current.platformId] = current;
                    }
                    return temp;
                }, {})

                next(null, resultsObject);
            }
        }
    );
}

function updateAWSResourceCostsFromCSV(provider, resources, downlaodedCSVPath, updateTime, callback) {
    var awsBillIndexes = appConfig.aws.billIndexes
    var awsServices = appConfig.aws.services
    var awsZones = appConfig.aws.zones

    var stream = fs.createReadStream(downlaodedCSVPath)
    csv.fromStream(stream, {headers: false}).on('data', function(data) {
        if((data[awsBillIndexes.totalCost] == 'LineItem')
            && ((provider.lastUpdateTime == null)
                || (Date.parse(data[awsBillIndexes.endDate]) > provider.lastUpdateTime))) {
            var resourceCostEntry = {platformDetails: {}}

            resourceCostEntry.organizationId = provider.orgId
            resourceCostEntry.providerId = provider._id
            resourceCostEntry.providerType = provider.providerType
            resourceCostEntry.cost = data[awsBillIndexes.cost]
            resourceCostEntry.startTime = Date.parse(data[awsBillIndexes.startDate])
            resourceCostEntry.endTime = Date.parse(data[awsBillIndexes.endDate])
            resourceCostEntry.lastUpdateTime = Date.parse(updateTime)
            resourceCostEntry.interval = 3600
            resourceCostEntry.platformDetails.serviceName = data[awsBillIndexes.prod]
            resourceCostEntry.billLineRecordId = data[awsBillIndexes.recordId]

            if (data[awsBillIndexes.prod] in awsServices) {
                resourceCostEntry.platformDetails.serviceId = awsServices[data[awsBillIndexes.prod]]
            }

            resourceCostEntry.platformDetails.zone = (data[awsBillIndexes.zone] == null)
                ? 'Unknown' : data[awsBillIndexes.zone]

            resourceCostEntry.platformDetails.region = (data[awsBillIndexes.zone] in awsZones)
                ? awsZones[data[awsBillIndexes.zone]] : 'Unknown'

            if (data[awsBillIndexes.instanceId] != null) {
                resourceCostEntry.platformDetails.instanceId = data[awsBillIndexes.instanceId]
            }
<commit_msg>resource service update to avoid duplicate entries in resource costs<commit_after>/*
 Copyright [2016] [Relevance Lab]
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 http://www.apache.org/licenses/LICENSE-2.0
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */

var logger = require('_pr/logger')(module);
var resourceService = module.exports = {};
var appConfig = require('_pr/config');
var Cryptography = require('_pr/lib/utils/cryptography');
var aws = require('aws-sdk');
var resources = require('_pr/model/resources/resources');
var CW = require('_pr/lib/cloudwatch.js');
var S3 = require('_pr/lib/s3.js');
var EC2 = require('_pr/lib/ec2.js');
var RDS = require('_pr/lib/rds.js');
var resourceCost = require('_pr/model/resource-costs');
var csv = require("fast-csv");
var fs = require('fs');
var async = require('async');
var dateUtil = require('_pr/lib/utils/dateUtil');
var unassignedInstancesModel = require('_pr/model/unassigned-instances');
var unManagedInstancesModel = require('_pr/model/unmanaged-instance');
var instancesModel = require('_pr/model/classes/instance/instance');
var entityCosts = require('_pr/model/entity-costs');
var mongoDbClient = require('mongodb').MongoClient;

resourceService.getCostForResources = getCostForResources_deprecated;
resourceService.getTotalCost = getTotalCost_deprecated;
resourceService.getCostForServices = getCostForServices_deprecated;
resourceService.getEC2InstanceUsageMetrics=getEC2InstanceUsageMetrics;
resourceService.getS3BucketsMetrics=getS3BucketsMetrics;
resourceService.getBucketsInfo=getBucketsInfo;
resourceService.getResources=getResources;
resourceService.getRDSInstancesInfo=getRDSInstancesInfo;
resourceService.getRDSDBInstanceMetrics=getRDSDBInstanceMetrics;
resourceService.bulkUpdateResourceProviderTags=bulkUpdateResourceProviderTags;
resourceService.bulkUpdateUnassignedResourceTags=bulkUpdateUnassignedResourceTags;
resourceService.bulkUpdateAWSResourcesTags=bulkUpdateAWSResourcesTags;
resourceService.getEC2InstancesInfo=getEC2InstancesInfo;
resourceService.getAllResourcesForProvider =  getAllResourcesForProvider;
resourceService.updateAWSResourceCostsFromCSV = updateAWSResourceCostsFromCSV
resourceService.aggregateEntityCosts = aggregateEntityCosts

// @TODO To be cached if needed. In memory data will not exceed 200MB for upto 2000 instances.
function getAllResourcesForProvider(provider, next) {
    async.parallel([
            function(callback) {
                instancesModel.getInstanceByProviderId(provider._id, callback);
            },
            function(callback) {
                //@TODO Duplicate function of  getByProviderId, to be cleaned up
                unManagedInstancesModel.getInstanceByProviderId(provider._id, callback);
            },
            function(callback) {
                unassignedInstancesModel.getUnAssignedInstancesByProviderId(provider._id, callback);
            }
            /*function(callback) {
                resources.getResourcesByProviderId(provider._id, callback);
            }*/
        ],
        function(err, results) {
            if (err) {
                var err = new Error('Internal server error');
                err.status = 500;
                next(err)
            } else {
                var resultsArray = [].concat.apply([], results);
                var resultsObject = resultsArray.reduce(function(temp, current) {
                    if('platformId' in current) {
                        temp[current.platformId] = current;
                    }
                    return temp;
                }, {})

                next(null, resultsObject);
            }
        }
    );
}

function updateAWSResourceCostsFromCSV(provider, resources, downlaodedCSVPath, updateTime, callback) {
    var awsBillIndexes = appConfig.aws.billIndexes
    var awsServices = appConfig.aws.services
    var awsZones = appConfig.aws.zones

    var stream = fs.createReadStream(downlaodedCSVPath)
    csv.fromStream(stream, {headers: false}).on('data', function(data) {
        if((data[awsBillIndexes.totalCost] == 'LineItem')
            && ((provider.lastBillUpdateTime == null)
                || (Date.parse(data[awsBillIndexes.endDate]) > Date.parse(provider.lastBillUpdateTime)))) {
            var resourceCostEntry = {platformDetails: {}}

            resourceCostEntry.organizationId = provider.orgId
            resourceCostEntry.providerId = provider._id
            resourceCostEntry.providerType = provider.providerType
            resourceCostEntry.cost = data[awsBillIndexes.cost]
            resourceCostEntry.startTime = Date.parse(data[awsBillIndexes.startDate])
            resourceCostEntry.endTime = Date.parse(data[awsBillIndexes.endDate])
            resourceCostEntry.lastUpdateTime = Date.parse(updateTime)
            resourceCostEntry.interval = 3600
            resourceCostEntry.platformDetails.serviceName = data[awsBillIndexes.prod]
            resourceCostEntry.billLineRecordId = data[awsBillIndexes.recordId]

            if (data[awsBillIndexes.prod] in awsServices) {
                resourceCostEntry.platformDetails.serviceId = awsServices[data[awsBillIndexes.prod]]
            }

            resourceCostEntry.platformDetails.zone = (data[awsBillIndexes.zone] == null)
                ? 'Unknown' : data[awsBillIndexes.zone]

            resourceCostEntry.platformDetails.region = (data[awsBillIndexes.zone] in awsZones)
                ? awsZones[data[awsBillIndexes.zone]] : 'Unknown'

            if (data[awsBillIndexes.instanceId] != null) {
                resourceCostEntry.platformDetails.instanceId = data[awsBillIndexes.instanceId]
            }

{'fraction_non_alphanumeric': 0.09350237717908082, 'fraction_numerical': 0.005338226707815498, 'mean_word_length': 3.4489795918367347, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4996886', 'n_tokens_mistral': 3258, 'n_tokens_neox': 3163, 'n_words': 631}