<commit_before>import { MongoClient } from "mongodb";
import _ from "lodash";

import config from "../config/configuration";

export default class MongoDbApi {
  _getMongoClient() {
    return MongoClient.connect(config.mongoDbUrl, {
      useNewUrlParser: true,
      useUnifiedTopology: true
    });
  }

  async getTagsOfFiles(fileNamesArray) {
    const client = await this._getMongoClient();

    const db = client.db();

    const imageTags = db.collection(config.mongoDbCollection);

    const tags = await imageTags
      .find({ name: { $in: fileNamesArray } })
      .toArray();

    await client.close();

    return tags;
  }

  async getFilesWithTags(tagsArray, limit, offset, sortBy = "name") {
    const client = await this._getMongoClient();

    const db = client.db();

    const imageTags = db.collection(config.mongoDbCollection);

    const tagsRegexes = _(tagsArray)
      .map(t => new RegExp("^" + t, "i"))
      .value();

    const sortByName = sortBy !== "name" ? -1 : 1;

    const items = await imageTags
      .find({ tags: { $all: tagsRegexes } })
      .sort({ name: sortByName })
      .skip(offset)
      .limit(limit)
      .toArray();

    const total = await imageTags.countDocuments({
      tags: { $all: tagsRegexes }
    });

    await client.close();

    return {
      items: items,
      limit: limit,
      offset: offset,
      total: total
    };
  }

  async getRandomFiles(tagsArray, limit) {
    const client = await this._getMongoClient();

    const db = client.db();

    const imageTags = db.collection(config.mongoDbCollection);

    const pipelineSteps = [];

    if (!!tagsArray && tagsArray.length > 0) {
      const tagsRegexes = _(tagsArray)
        .map(t => new RegExp("^" + t, "i"))
        .value();
      pipelineSteps.push({ $match: { tags: { $all: tagsRegexes } } });
    }

    pipelineSteps.push({ $sample: { size: limit } });

    const items = await imageTags.aggregate(pipelineSteps).toArray();

    await client.close();

    return {<commit_msg>Implement search by excluding tags<commit_after>import { MongoClient } from "mongodb";
import _, { find } from "lodash";

import config from "../config/configuration";

export default class MongoDbApi {
  _getMongoClient() {
    return MongoClient.connect(config.mongoDbUrl, {
      useNewUrlParser: true,
      useUnifiedTopology: true
    });
  }

  async getTagsOfFiles(fileNamesArray) {
    const client = await this._getMongoClient();

    const db = client.db();

    const imageTags = db.collection(config.mongoDbCollection);

    const tags = await imageTags
      .find({ name: { $in: fileNamesArray } })
      .toArray();

    await client.close();

    return tags;
  }

  getTagsSearchMongoObject(tagsArray)
  {
    if (!tagsArray || tagsArray.length == 0) {
      return  null;
    }

    const positiveTags = _(tagsArray)
      .filter(t => t[0] !== "-")
      .toArray();
    const negativeTags = _(tagsArray)
      .filter(t => t[0] === "-")
      .map(t => t.substring(1))
      .toArray();

    const positiveRegularExpressions = positiveTags
      .map(t => new RegExp("^" + t, "i"))
      .value();
    const negativeRegularExpressions = negativeTags
      .map(t => new RegExp("^" + t, "i"))
      .value();

    const positiveObject = positiveRegularExpressions.length == 0
      ? null
      : { tags: { $all: positiveRegularExpressions } };
    const negativeObject = negativeRegularExpressions.length == 0
      ? null
      : { tags: { $not: {$elemMatch: { $in: negativeRegularExpressions } } } };

    if(positiveObject === null) return negativeObject;
    if(negativeObject === null) return positiveObject;

    return { $and: [ positiveObject, negativeObject ] };

  }

  async getFilesWithTags(tagsArray, limit, offset, sortBy = "name") {
    const client = await this._getMongoClient();

    const db = client.db();

    const imageTags = db.collection(config.mongoDbCollection);

    const findObject = this.getTagsSearchMongoObject(tagsArray);

    const sortByName = sortBy !== "name" ? -1 : 1;

    const items = await imageTags
      .find(findObject)
      .sort({ name: sortByName })
      .skip(offset)
      .limit(limit)
      .toArray();

    const total = await imageTags.countDocuments(findObject);

    await client.close();

    return {
      items: items,
      limit: limit,
      offset: offset,
      total: total
    };
  }

  async getRandomFiles(tagsArray, limit) {
    const client = await this._getMongoClient();

    const db = client.db();

    const imageTags = db.collection(config.mongoDbCollection);

    const pipelineSteps = [];

    const findObject = this.getTagsSearchMongoObject(tagsArray);

    if (findObject !== null) {
      pipelineSteps.push({ $match: findObject });
    }

    pipelineSteps.push({ $sample: { size: limit } });

    const items = await imageTags.aggregate(pipelineSteps).toArray();

    await client.close();

    return {
{'fraction_non_alphanumeric': 0.13381995133819952, 'fraction_numerical': 0.0022303325223033254, 'mean_word_length': 2.8720565149136577, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6478295', 'n_tokens_mistral': 1546, 'n_tokens_neox': 1478, 'n_words': 361}