<commit_before>'use strict';

const path = require('path');
const { URL } = require('url');
const settings = require('../../lib/settings');
var mongo = require('../../lib/mongo');
var utils = require('../../lib/utils');
var parse = require('../../lib/parse');
var CategoriesService = require('./productCategories');
const SettingsService = require('../settings/settings');
var ObjectID = require('mongodb').ObjectID;
var fs = require('fs-extra');

class ProductsService {
  constructor() {}

  getProducts(params = {}) {
    return Promise.all([
        CategoriesService.getCategories(),
        SettingsService.getSettings()
      ]).then(([ categories, generalSettings ]) => {
        const domain = generalSettings.domain || '';
        const fieldsArray = this.getArrayFromCSV(params.fields);
        const limit = parse.getNumberIfPositive(params.limit) || 1000;
        const offset = parse.getNumberIfPositive(params.offset) || 0;
        const projectQuery = this.getProjectQuery(fieldsArray);
        const sortQuery = this.getSortQuery(params); // todo: validate every sort field
        const matchQuery = this.getMatchQuery(params, categories);
        const matchTextQuery = this.getMatchTextQuery(params);
        const itemsAggregation = [];

        // $match with $text is only allowed as the first pipeline stage"
        if(matchTextQuery) {
          itemsAggregation.push({ $match: matchTextQuery });
        }
        itemsAggregation.push({ $project: projectQuery });
        itemsAggregation.push({ $match: matchQuery });
        if(sortQuery){
          itemsAggregation.push({ $sort: sortQuery });
        }
        itemsAggregation.push({ $skip : offset });
        itemsAggregation.push({ $limit : limit });


        return Promise.all([
            mongo.db.collection('products').aggregate(itemsAggregation).toArray(),
            this.getCountIfNeeded(params, matchQuery, matchTextQuery, projectQuery),
            this.getMinMaxPriceIfNeeded(params, categories, matchTextQuery, projectQuery),
            this.getAllAttributesIfNeeded(params, categories, matchTextQuery, projectQuery),
            this.getAttributesIfNeeded(params, categories, matchTextQuery, projectQuery)
          ]).then(([ itemsResult, countResult, minMaxPriceResult, allAttributesResult, attributesResult ]) => {

            const ids = this.getArrayFromCSV(parse.getString(params.ids));
            const sku = this.getArrayFromCSV(parse.getString(params.sku));

            let items = itemsResult.map(item => this.changeProperties(categories, item, domain)).filter(item => item !== null);
            items = this.sortItemsByArrayOfIdsIfNeed(items, ids, sortQuery);
            items = this.sortItemsByArrayOfSkuIfNeed(items, sku, sortQuery);

            let total_count = 0;
            let min_price = 0;
            let max_price = 0;

            if(countResult && countResult.length === 1) {
              total_count = countResult[0].count;
            }

            if(minMaxPriceResult && minMaxPriceResult.length === 1) {
              min_price = minMaxPriceResult[0].min_price || 0;
              max_price = minMaxPriceResult[0].max_price || 0;
            }

            let attributes = [];
            if(allAttributesResult) {
              attributes = this.getOrganizedAttributes(allAttributesResult, attributesResult, params);
            }

            return {
              price: {
                min: min_price,
                max: max_price
              },
              attributes: attributes,
              total_count: total_count,
              has_more: (offset + items.length) < total_count,
              data: items<commit_msg>Fixed product list with null items<commit_after>'use strict';

const path = require('path');
const { URL } = require('url');
const settings = require('../../lib/settings');
var mongo = require('../../lib/mongo');
var utils = require('../../lib/utils');
var parse = require('../../lib/parse');
var CategoriesService = require('./productCategories');
const SettingsService = require('../settings/settings');
var ObjectID = require('mongodb').ObjectID;
var fs = require('fs-extra');

class ProductsService {
  constructor() {}

  getProducts(params = {}) {
    return Promise.all([
        CategoriesService.getCategories(),
        SettingsService.getSettings()
      ]).then(([ categories, generalSettings ]) => {
        const domain = generalSettings.domain || '';
        const fieldsArray = this.getArrayFromCSV(params.fields);
        const limit = parse.getNumberIfPositive(params.limit) || 1000;
        const offset = parse.getNumberIfPositive(params.offset) || 0;
        const projectQuery = this.getProjectQuery(fieldsArray);
        const sortQuery = this.getSortQuery(params); // todo: validate every sort field
        const matchQuery = this.getMatchQuery(params, categories);
        const matchTextQuery = this.getMatchTextQuery(params);
        const itemsAggregation = [];

        // $match with $text is only allowed as the first pipeline stage"
        if(matchTextQuery) {
          itemsAggregation.push({ $match: matchTextQuery });
        }
        itemsAggregation.push({ $project: projectQuery });
        itemsAggregation.push({ $match: matchQuery });
        if(sortQuery){
          itemsAggregation.push({ $sort: sortQuery });
        }
        itemsAggregation.push({ $skip : offset });
        itemsAggregation.push({ $limit : limit });


        return Promise.all([
            mongo.db.collection('products').aggregate(itemsAggregation).toArray(),
            this.getCountIfNeeded(params, matchQuery, matchTextQuery, projectQuery),
            this.getMinMaxPriceIfNeeded(params, categories, matchTextQuery, projectQuery),
            this.getAllAttributesIfNeeded(params, categories, matchTextQuery, projectQuery),
            this.getAttributesIfNeeded(params, categories, matchTextQuery, projectQuery)
          ]).then(([ itemsResult, countResult, minMaxPriceResult, allAttributesResult, attributesResult ]) => {

            const ids = this.getArrayFromCSV(parse.getString(params.ids));
            const sku = this.getArrayFromCSV(parse.getString(params.sku));

            let items = itemsResult.map(item => this.changeProperties(categories, item, domain));
            items = this.sortItemsByArrayOfIdsIfNeed(items, ids, sortQuery);
            items = this.sortItemsByArrayOfSkuIfNeed(items, sku, sortQuery);
            items = items.filter(item => !!item);

            let total_count = 0;
            let min_price = 0;
            let max_price = 0;

            if(countResult && countResult.length === 1) {
              total_count = countResult[0].count;
            }

            if(minMaxPriceResult && minMaxPriceResult.length === 1) {
              min_price = minMaxPriceResult[0].min_price || 0;
              max_price = minMaxPriceResult[0].max_price || 0;
            }

            let attributes = [];
            if(allAttributesResult) {
              attributes = this.getOrganizedAttributes(allAttributesResult, attributesResult, params);
            }

            return {
              price: {
                min: min_price,
                max: max_price
              },
              attributes: attributes,
              total_count: total_count,
              has_more: (offset + items.length) < total_count,
              data: items
{'fraction_non_alphanumeric': 0.1204868154158215, 'fraction_numerical': 0.004056795131845842, 'mean_word_length': 2.5850702859912746, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '404138', 'n_tokens_mistral': 1939, 'n_tokens_neox': 1874, 'n_words': 407}