<commit_before>//
//   Copyright 2015 <NAME> <<EMAIL>>
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing,
//   software distributed under the License is distributed on an "AS
//   IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//   express or implied.  See the License for the specific language
//   governing permissions and limitations under the License.
//

'use strict';

const Promise = require('bluebird');
const elasticSearchClient = Promise.promisifyAll(require('./elasticSearch').getClient());
const log = require('./log');

exports.storeMessage = function storeMessage(conversationId, msg) {
    if (!elasticSearchAvailable()) {
        return false;
    }

    elasticSearchClient.create({
        index: 'messages',
        type: 'message',
        id: msg.gid,
        body: {
            ts: msg.ts * 1000,
            body: msg.body,
            cat: msg.cat,
            userId: msg.userId,
            conversationId
        }
    }, error => {
        if (error) {
            // TODO: Temporary
            log.warn(`Elasticsearch error. Failed to index messsage: ${error}`);
        }
    });

    return true;
};

exports.updateMessage = function updateMessage(gid, msg) {
    if (!elasticSearchAvailable()) {
        return false;
    }

    elasticSearchClient.update({
        index: 'messages',
        type: 'message',
        id: gid,
        body: {
            doc: {
                body: msg
            }
        }
    }, error => {
        if (error) {
            log.warn(msg.userId, `Elasticsearch error. Failed to index messsage: ${error}`);
        }
    });

    return true;
};

exports.getMessageRange = async function getMessageRange(conversationId, start, end, amount) {
    if (!elasticSearchAvailable()) {
        return [];
    }

    // TODO: If there are multiple messages at the boundary start/end ts, part of them can be lost.
    // Theoretical problem mostly. Solution is not lte.

    const range = {
        ts: {
            lt: end * 1000
        }
    };

    if (start) {
        range.ts.gte = start * 1000;
    }

    const response = await elasticSearchClient.search({
        index: 'messages',
        body: {
            size: amount || 1000,
            sort: {
                ts: {
                    order: 'desc'
                }
            },
            query: {
                bool: {
                    filter: {
                        and: [
                            {
                                term: {
                                    conversationId
                                }
                            }, {
                                range
                            }
                        ]
                    }
                }
            }
        }
    });

    return convertToMsgs(response.hits.hits);
};

function convertToMsgs(hits) {
    return hits.map(hit => ({
        gid: hit._id,
        ts: Math.floor(hit._source.ts / 1000),
        body: hit._source.body,
        cat: hit._source.cat,
        userId: hit._source.userId
    }));
}

function elasticSearchAvailable() {
    if (!elasticSearchClient) {
        return false;
    }

    return true;
}
<commit_msg>Fix bool elascticsearch query<commit_after>//
//   Copyright 2015 <NAME> <<EMAIL>>
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing,
//   software distributed under the License is distributed on an "AS
//   IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//   express or implied.  See the License for the specific language
//   governing permissions and limitations under the License.
//

'use strict';

const Promise = require('bluebird');
const elasticSearchClient = Promise.promisifyAll(require('./elasticSearch').getClient());
const log = require('./log');

exports.storeMessage = function storeMessage(conversationId, msg) {
    if (!elasticSearchAvailable()) {
        return false;
    }

    elasticSearchClient.create({
        index: 'messages',
        type: 'message',
        id: msg.gid,
        body: {
            ts: msg.ts * 1000,
            body: msg.body,
            cat: msg.cat,
            userId: msg.userId,
            conversationId
        }
    }, error => {
        if (error) {
            // TODO: Temporary
            log.warn(`Elasticsearch error. Failed to index messsage: ${error}`);
        }
    });

    return true;
};

exports.updateMessage = function updateMessage(gid, msg) {
    if (!elasticSearchAvailable()) {
        return false;
    }

    elasticSearchClient.update({
        index: 'messages',
        type: 'message',
        id: gid,
        body: {
            doc: {
                body: msg
            }
        }
    }, error => {
        if (error) {
            log.warn(msg.userId, `Elasticsearch error. Failed to index messsage: ${error}`);
        }
    });

    return true;
};

exports.getMessageRange = async function getMessageRange(conversationId, start, end, amount) {
    if (!elasticSearchAvailable()) {
        return [];
    }

    // TODO: If there are multiple messages at the boundary start/end ts, part of them can be lost.
    // Theoretical problem mostly. Solution is not lte.
    const response = await elasticSearchClient.search({
        index: 'messages',
        body: {
            size: amount || 1000,
            sort: {
                ts: {
                    order: 'desc'
                }
            },
            query: {
                bool: {
                    must: [
                        { term: { conversationId } },
                        { range: { ts: { lt: end * 1000, gte: start ? start * 1000 : false } } }
                    ]
                }
            }
        }
    });

    return convertToMsgs(response.hits.hits);
};

function convertToMsgs(hits) {
    return hits.map(hit => ({
        gid: hit._id,
        ts: Math.floor(hit._source.ts / 1000),
        body: hit._source.body,
        cat: hit._source.cat,
        userId: hit._source.userId
    }));
}

function elasticSearchAvailable() {
    if (!elasticSearchClient) {
        return false;
    }

    return true;
}

{'fraction_non_alphanumeric': 0.11226486079759218, 'fraction_numerical': 0.008427389014296464, 'mean_word_length': 1.581973581973582, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '769388', 'n_tokens_mistral': 1863, 'n_tokens_neox': 1766, 'n_words': 514}