<commit_before>/// <reference path="../../../typings/tsd.d.ts" />
/// <reference path="../client/api.d.ts" />
/// <reference path="../../../typings/ui-grid/ui-grid.d.ts" />

/*
 * Contains helper methods for the generated typescript API
 */
module apiHelpers {
    export class SourceRecord {
        SessionId: string;
        SourceId: string;
        SourceRecordId: string;
        SourceContent: string;
        SourceDisplayName: string;
        SourceStatus: string;
        TotalDupes: number;
        Notes: string;
    }

    export class TargetRecord {
        TargetId: string;
        TargetRecordId: string;
        TargetDisplayName: string;
        TrueDuplicate: boolean;
        Quality: string;
        TargetScore: number;
    }

    export class Session {
        Id: string;
        SessionId: string;
        IndexName: string;
        ProfileName: string;
        JobStartTime: Date;
        JobEndTime: Date;
        SelectionQuery: string;
        DisplayFieldName: string;
        RecordsReturned: number;
        RecordsAvailable: number;
        ThreadCount: number;
    }

    export class Duplicate extends SourceRecord {
        Targets: TargetRecord[]
        SourceStatusName: any
        FlexSearchId: string
    }

    function getFirstDocumentFromSearch(sr: API.Client.SearchResultsResponse) {
        return sr.data.recordsReturned > 0 ? sr.data.documents[0] : null;
    }

    function getOrderByDirection(sortDirection: string) {
        return sortDirection && sortDirection.indexOf("esc") > 0 ? API.Client.SearchQuery.OrderByDirectionEnum.Descending : API.Client.SearchQuery.OrderByDirectionEnum.Ascending;
    }

    export function updateDuplicate(duplicate: Duplicate, commonApi: API.Client.CommonApi) {
        var fields: { [key: string]: string } = {
            sessionid: duplicate.SessionId,
            sourcedisplayname: duplicate.SourceDisplayName,
            sourceid: duplicate.SourceId,
            sourcerecordid: duplicate.SourceRecordId,
            totaldupesfound: duplicate.TotalDupes.toString(),
            type: "source",
            notes: duplicate.Notes,
            sourcestatus: duplicate.SourceStatus,
            targetrecords: JSON.stringify(duplicate.Targets)
        };

        return commonApi.updateDocumentHandled({
            fields: fields,
            id: duplicate.FlexSearchId,
            indexName: "duplicates"
        }, "duplicates", duplicate.FlexSearchId);
    }
    
    // Get the duplicate that needs to be displayed
    export function getDuplicateBySourceId(sessionId, sourceId, commonApi: API.Client.CommonApi) {
        return commonApi.postSearchHandled({
            queryString: "allof(type, 'source') and allof(sessionid, '" + sessionId + "') and allof(sourceid = '" + sourceId + "')",
            columns: ["*"],
            indexName: "duplicates"
        }, "duplicates")
            .then(getFirstDocumentFromSearch);
    }

    export function getRecordsByIds(indexName, ids: any[], commonApi: API.Client.CommonApi) {
        var idStr = ids
            .reduce(function(acc, val) { return acc + ",'" + val + "'" }, "")
            .substr(1);
        return commonApi.postSearchHandled({
            queryString: "anyof(_id, " + idStr + ")",
            columns: ["*"],
            indexName: indexName
        }, indexName)
            .then(r => r.data.documents);
    }

    export function getSessionBySessionId(sessionId, commonApi: API.Client.CommonApi) {
        return commonApi.postSearchHandled({
            queryString: "allof(type, 'session') and allof(sessionid, '" + sessionId + "')",
            columns: ["*"],
            indexName: "duplicates"
        }, "duplicates")
            .then(getFirstDocumentFromSearch);
    }

    export function getDuplicatesFromSession(sessionId, count, skip, commonApi: API.Client.CommonApi, sortby?, sortDirection?) {
        return commonApi.postSearchHandled({
            queryString: "allof(type, 'source') and allof(sessionid, '" + sessionId + "')",
            columns: ["*"],
            count: count,
            skip: skip,
            orderBy: sortby,
            orderByDirection: getOrderByDirection(sortDirection),
            indexName: "duplicates"
        }, "duplicates")
            .then(r => r.data);
    }

    export function getSessions(count, skip, commonApi: API.Client.CommonApi, sortby?, sortDirection?) {
        return commonApi.postSearchHandled({
            queryString: "allof(type, 'session')",
            columns: ["*"],
            count: count,
            skip: skip,
            orderBy: sortby,
            orderByDirection: getOrderByDirection(sortDirection),
            indexName: "duplicates"
        }, "duplicates")
            .then(r => r.data);
    }

    export function submitDuplicateDetection(indexName, searchProfile, displayFieldName, selectionQuery, fileName, commonApi: API.Client.CommonApi,
        threadCount?, maxRecordsToScan?, maxDupsToReturn?) {
        return commonApi.duplicateDetectionHandled({
            selectionQuery: selectionQuery,
            displayName: displayFieldName,
            fileName: fileName,
            threadCount: threadCount,
            indexName: indexName,
            profileName: searchProfile,
            maxRecordsToScan: maxRecordsToScan,
            duplicatesCount: maxDupsToReturn
        }, indexName, searchProfile);
    }
}<commit_msg>use predefined query instead of search profile in apiHelpers.ts<commit_after>/// <reference path="../../../typings/tsd.d.ts" />
/// <reference path="../client/api.d.ts" />
/// <reference path="../../../typings/ui-grid/ui-grid.d.ts" />

/*
 * Contains helper methods for the generated typescript API
 */
module apiHelpers {
    export class SourceRecord {
        SessionId: string;
        SourceId: string;
        SourceRecordId: string;
        SourceContent: string;
        SourceDisplayName: string;
        SourceStatus: string;
        TotalDupes: number;
        Notes: string;
    }

    export class TargetRecord {
        TargetId: string;
        TargetRecordId: string;
        TargetDisplayName: string;
        TrueDuplicate: boolean;
        Quality: string;
        TargetScore: number;
    }

    export class Session {
        Id: string;
        SessionId: string;
        IndexName: string;
        ProfileName: string;
        JobStartTime: Date;
        JobEndTime: Date;
        SelectionQuery: string;
        DisplayFieldName: string;
        RecordsReturned: number;
        RecordsAvailable: number;
        ThreadCount: number;
    }

    export class Duplicate extends SourceRecord {
        Targets: TargetRecord[]
        SourceStatusName: any
        FlexSearchId: string
    }

    function getFirstDocumentFromSearch(sr: API.Client.SearchResultsResponse) {
        return sr.data.recordsReturned > 0 ? sr.data.documents[0] : null;
    }

    function getOrderByDirection(sortDirection: string) {
        return sortDirection && sortDirection.indexOf("esc") > 0 ? API.Client.SearchQuery.OrderByDirectionEnum.Descending : API.Client.SearchQuery.OrderByDirectionEnum.Ascending;
    }

    export function updateDuplicate(duplicate: Duplicate, commonApi: API.Client.CommonApi) {
        var fields: { [key: string]: string } = {
            sessionid: duplicate.SessionId,
            sourcedisplayname: duplicate.SourceDisplayName,
            sourceid: duplicate.SourceId,
            sourcerecordid: duplicate.SourceRecordId,
            totaldupesfound: duplicate.TotalDupes.toString(),
            type: "source",
            notes: duplicate.Notes,
            sourcestatus: duplicate.SourceStatus,
            targetrecords: JSON.stringify(duplicate.Targets)
        };

        return commonApi.updateDocumentHandled({
            fields: fields,
            id: duplicate.FlexSearchId,
            indexName: "duplicates"
        }, "duplicates", duplicate.FlexSearchId);
    }
    
    // Get the duplicate that needs to be displayed
    export function getDuplicateBySourceId(sessionId, sourceId, commonApi: API.Client.CommonApi) {
        return commonApi.postSearchHandled({
            queryString: "allof(type, 'source') and allof(sessionid, '" + sessionId + "') and allof(sourceid = '" + sourceId + "')",
            columns: ["*"],
            indexName: "duplicates"
        }, "duplicates")
            .then(getFirstDocumentFromSearch);
    }

    export function getRecordsByIds(indexName, ids: any[], commonApi: API.Client.CommonApi) {
        var idStr = ids
            .reduce(function(acc, val) { return acc + ",'" + val + "'" }, "")
            .substr(1);
        return commonApi.postSearchHandled({
            queryString: "anyof(_id, " + idStr + ")",
            columns: ["*"],
            indexName: indexName
        }, indexName)
            .then(r => r.data.documents);
    }

    export function getSessionBySessionId(sessionId, commonApi: API.Client.CommonApi) {
        return commonApi.postSearchHandled({
            queryString: "allof(type, 'session') and allof(sessionid, '" + sessionId + "')",
            columns: ["*"],
            indexName: "duplicates"
        }, "duplicates")
            .then(getFirstDocumentFromSearch);
    }

    export function getDuplicatesFromSession(sessionId, count, skip, commonApi: API.Client.CommonApi, sortby?, sortDirection?) {
        return commonApi.postSearchHandled({
            queryString: "allof(type, 'source') and allof(sessionid, '" + sessionId + "')",
            columns: ["*"],
            count: count,
            skip: skip,
            orderBy: sortby,
            orderByDirection: getOrderByDirection(sortDirection),
            indexName: "duplicates"
        }, "duplicates")
            .then(r => r.data);
    }

    export function getSessions(count, skip, commonApi: API.Client.CommonApi, sortby?, sortDirection?) {
        return commonApi.postSearchHandled({
            queryString: "allof(type, 'session')",
            columns: ["*"],
            count: count,
            skip: skip,
            orderBy: sortby,
            orderByDirection: getOrderByDirection(sortDirection),
            indexName: "duplicates"
        }, "duplicates")
            .then(r => r.data);
    }

    export function submitDuplicateDetection(indexName, searchProfile, displayFieldName, selectionQuery, fileName, commonApi: API.Client.CommonApi,
        threadCount?, maxRecordsToScan?, maxDupsToReturn?) {
        return commonApi.duplicateDetectionHandled({
            selectionQuery: selectionQuery,
            displayName: displayFieldName,
            fileName: fileName,
            threadCount: threadCount,
            indexName: indexName,
            predefinedQueryName: searchProfile,
            maxRecordsToScan: maxRecordsToScan,
            duplicatesCount: maxDupsToReturn
        }, indexName, searchProfile);
    }
}
{'fraction_non_alphanumeric': 0.10856775142489428, 'fraction_numerical': 0.0007354293068578783, 'mean_word_length': 2.3453259532595325, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4614696', 'n_tokens_mistral': 2939, 'n_tokens_neox': 2891, 'n_words': 624}