<commit_before>/*********************************************************************
 * Copyright (c) 2019 Red Hat, Inc.
 *
 * This program and the accompanying materials are made
 * available under the terms of the Eclipse Public License 2.0
 * which is available at https://www.eclipse.org/legal/epl-2.0/
 *
 * SPDX-License-Identifier: EPL-2.0
 **********************************************************************/
import axios from 'axios';
import { DriverHelper } from '../../utils/DriverHelper';
import { injectable, inject } from 'inversify';
import { CLASSES } from '../../inversify.types';
import { TestConstants } from '../../TestConstants';
import { By, WebElement, error } from 'selenium-webdriver';
import { TestWorkspaceUtil, WorkspaceStatus } from '../../utils/workspace/TestWorkspaceUtil';

export enum RightToolbarButton {
    Explorer = 'Explorer',
    Git = 'Git',
    Debug = 'Debug'
}

@injectable()
export class Ide {
    public static readonly EXPLORER_BUTTON_XPATH: string = '(//ul[@class=\'p-TabBar-content\']//li[@title=\'Explorer\'])[1]';
    public static readonly SELECTED_EXPLORER_BUTTON_XPATH: string = '(//ul[@class=\'p-TabBar-content\']//li[@title=\'Explorer\' and contains(@class, \'p-mod-current\')])[1]';
    public static readonly ACTIVATED_IDE_IFRAME_CSS: string = '#ide-iframe-window[aria-hidden=\'false\']';
    public static readonly SELECTED_GIT_BUTTON_XPATH: string = '(//ul[@class=\'p-TabBar-content\']//li[@title=\'Git\' and contains(@class, \'p-mod-current\')])[1]';
    private static readonly TOP_MENU_PANEL_CSS: string = '#theia-app-shell #theia-top-panel .p-MenuBar-content';
    private static readonly LEFT_CONTENT_PANEL_CSS: string = '#theia-left-content-panel';
    private static readonly PRELOADER_CSS: string = '.theia-preload';
    private static readonly IDE_IFRAME_CSS: string = 'iframe#ide-application-iframe';

    constructor(
        @inject(CLASSES.DriverHelper) private readonly driverHelper: DriverHelper,
        @inject(CLASSES.TestWorkspaceUtil) private readonly testWorkspaceUtil: TestWorkspaceUtil) { }

    async waitAndSwitchToIdeFrame(timeout: number = TestConstants.TS_SELENIUM_LOAD_PAGE_TIMEOUT) {
        await this.driverHelper.waitAndSwitchToFrame(By.css(Ide.IDE_IFRAME_CSS), timeout);
    }

    async waitNotification(notificationText: string, timeout: number = TestConstants.TS_SELENIUM_DEFAULT_TIMEOUT) {
        const notificationLocator: By = By.xpath(this.getNotificationXpathLocator(notificationText));

        await this.driverHelper.waitVisibility(notificationLocator, timeout);
    }

    async waitNotificationAndClickOnButton(notificationText: string,
        buttonText: string,
        timeout: number = TestConstants.TS_SELENIUM_DEFAULT_TIMEOUT) {

        await this.driverHelper.getDriver().wait(async () => {
            await this.waitNotification(notificationText, timeout);
            await this.clickOnNotificationButton(notificationText, buttonText);

            try {
                await this.waitNotificationDisappearance(notificationText);
                return true;
            } catch (err) {
                if (!(err instanceof error.TimeoutError)) {
                    throw err;
                }

                console.log(`After clicking on "${buttonText}" button of the notification with text "${notificationText}" \n` +
                    'it is still visible (issue #14121), try again.');

                await this.driverHelper.wait(TestConstants.TS_SELENIUM_DEFAULT_POLLING);
            }
        }, timeout);
    }

    async waitNotificationAndConfirm(notificationText: string, timeout: number = TestConstants.TS_SELENIUM_DEFAULT_TIMEOUT) {
        await this.waitNotificationAndClickOnButton(notificationText, 'yes', timeout);
    }

    async waitNotificationAndOpenLink(notificationText: string, timeout: number = TestConstants.TS_SELENIUM_DEFAULT_TIMEOUT) {
        await this.waitApplicationReadyToUse(await this.getApplicationUrl(notificationText));
        await this.waitNotificationAndClickOnButton(notificationText, 'Open Link', timeout);
    }

    async isNotificationPresent(notificationText: string): Promise<boolean> {
        const notificationLocator: By = By.xpath(this.getNotificationXpathLocator(notificationText));

        return await this.driverHelper.waitVisibilityBoolean(notificationLocator);
    }

    async waitNotificationDisappearance(notificationText: string,
        attempts: number = TestConstants.TS_SELENIUM_DEFAULT_ATTEMPTS,
        polling: number = TestConstants.TS_SELENIUM_DEFAULT_POLLING) {
        const notificationLocator: By = By.xpath(this.getNotificationXpathLocator(notificationText));

        await this.driverHelper.waitDisappearance(notificationLocator, attempts, polling);
    }

    async clickOnNotificationButton(notificationText: string, buttonText: string) {
        const notificationLocator: string = this.getNotificationXpathLocator(notificationText);
        const yesButtonLocator: string = notificationLocator + `//button[text()=\'${buttonText}\']`;

        await this.driverHelper.waitAndClick(By.xpath(yesButtonLocator));
    }

    async waitWorkspaceAndIde(workspaceNamespace: string,
        workspaceName: string,
        timeout: number = TestConstants.TS_SELENIUM_LOAD_PAGE_TIMEOUT) {

        await this.waitAndSwitchToIdeFrame(timeout);
        await this.testWorkspaceUtil.waitWorkspaceStatus(workspaceNamespace, workspaceName, WorkspaceStatus.RUNNING);
        await this.waitIde(timeout);
    }

    async waitIde(timeout: number = TestConstants.TS_SELENIUM_LOAD_PAGE_TIMEOUT) {
        const mainIdeParts: Array<By> = [By.css(Ide.TOP_MENU_PANEL_CSS), By.css(Ide.LEFT_CONTENT_PANEL_CSS), By.xpath(Ide.EXPLORER_BUTTON_XPATH)];

        for (const idePartLocator of mainIdeParts) {
            await this.driverHelper.waitVisibility(idePartLocator, timeout);
        }
    }

    async waitRightToolbarButton(buttonTitle: RightToolbarButton, timeout: number = TestConstants.TS_SELENIUM_DEFAULT_TIMEOUT) {
        const buttonLocator: By = this.getRightToolbarButtonLocator(buttonTitle);

        await this.driverHelper.waitVisibility(buttonLocator, timeout);
    }

    async waitAndClickRightToolbarButton(buttonTitle: RightToolbarButton, timeout: number = TestConstants.TS_SELENIUM_DEFAULT_TIMEOUT) {
        const buttonLocator: By = this.getRightToolbarButtonLocator(buttonTitle);

        await this.driverHelper.waitAndClick(buttonLocator, timeout);
    }

    async waitTopMenuPanel(timeout: number = TestConstants.TS_SELENIUM_DEFAULT_TIMEOUT) {
        await this.driverHelper.waitVisibility(By.css(Ide.TOP_MENU_PANEL_CSS), timeout);
    }

    async waitLeftContentPanel(timeout: number = TestConstants.TS_SELENIUM_DEFAULT_TIMEOUT) {
        await this.driverHelper.waitVisibility(By.css(Ide.LEFT_CONTENT_PANEL_CSS));
    }

    async waitPreloaderAbsent(attempts: number = TestConstants.TS_SELENIUM_DEFAULT_ATTEMPTS,
        polling: number = TestConstants.TS_SELENIUM_DEFAULT_POLLING) {
        await this.driverHelper.waitDisappearance(By.css(Ide.PRELOADER_CSS), attempts, polling);
    }

    async waitStatusBarContains(expectedText: string, timeout: number = TestConstants.TS_SELENIUM_LANGUAGE_SERVER_START_TIMEOUT) {
        const statusBarLocator: By = By.css('div[id=\'theia-statusBar\']');

        await this.driverHelper.getDriver().wait(async () => {
            const elementText: string = await this.driverHelper.waitAndGetText(statusBarLocator, timeout);
            const isTextPresent: boolean = elementText.search(expectedText) > 0;

            if (isTextPresent) {
                return true;
            }

        }, timeout);
    }

    async waitStatusBarTextAbsence(expectedText: string, timeout: number = TestConstants.TS_SELENIUM_DEFAULT_TIMEOUT) {
        const statusBarLocator: By = By.css('div[id=\'theia-statusBar\']');

        // for ensuring that check is not invoked in the gap of status displaying
        for (let i: number = 0; i < 3; i++) {
            await this.driverHelper.getDriver().wait(async () => {
                const elementText: string = await this.driverHelper.waitAndGetText(statusBarLocator, timeout);

                const isTextAbsent: boolean = elementText.search(expectedText) === -1;

                if (isTextAbsent) {
                    return true;
                }

            }, timeout);
        }
    }

    async waitIdeFrameAndSwitchOnIt(timeout: number = TestConstants.TS_SELENIUM_LOAD_PAGE_TIMEOUT) {
        await this.driverHelper.waitAndSwitchToFrame(By.css(Ide.IDE_IFRAME_CSS), timeout);
    }

    async checkLsInitializationStart(expectedTextInStatusBar: string) {
        try {
            await this.waitStatusBarContains(expectedTextInStatusBar, 20000);
        } catch (err) {
            if (!(err instanceof error.TimeoutError)) {
                throw err;
            }

            await this.driverHelper.getDriver().navigate().refresh();
            await this.waitAndSwitchToIdeFrame();
            await this.waitStatusBarContains(expectedTextInStatusBar);
        }

    }

    async closeAllNotifications() {
        const notificationLocator: By = By.css('.theia-Notification');

        if (! await this.driverHelper.isVisible(notificationLocator)) {
            return;
        }

        const notifications: WebElement[] = await this.driverHelper.waitAllPresence(notificationLocator);
        const notificationsCapacity: number = notifications.length;

        for (let i = 1; i <= notificationsCapacity; i++) {
            const notificationLocator: By = By.xpath('//div[@class=\'theia-Notification\']//button[text()=\'Close\']');

            try {
                await this.driverHelper.waitAndClick(notificationLocator);
            } catch (err) {
                if (err instanceof error.TimeoutError) {
                    console.log(`The '${notificationLocator}' element is not visible and can't be clicked`);
                    continue;
                }

                throw err;
            }
        }
    }

    async performKeyCombination(keyCombination: string) {
        const bodyLocator: By = By.tagName('body');

        await this.driverHelper.type(bodyLocator, keyCombination);
    }

    async waitRightToolbarButtonSelection(buttonTitle: string, timeout: number = TestConstants.TS_SELENIUM_DEFAULT_TIMEOUT) {
        const selectedRightToolbarButtonLocator: By = this.getSelectedRightToolbarButtonLocator(buttonTitle);

        await this.driverHelper.waitVisibility(selectedRightToolbarButtonLocator, timeout);
    }

    async getApplicationUrl(notificationText: string) {
        const notificationTextLocator: By = By.xpath(`//div[@class='theia-Notification']//p[contains(@id,'${notificationText}')]`);

         let notification = await this.driverHelper.waitAndGetText(notificationTextLocator);

         return notification.substring(54);
    }

    async waitApplicationReadyToUse(url: string,
        timeout: number = TestConstants.TS_SELENIUM_DEFAULT_TIMEOUT) {

         await this.driverHelper.getDriver().wait(async () => {
            try {
                let res = await axios.get(url);

                 if (res.status === 200) {
                    console.log('Application is ready for use.');
                    return true;
                }
            } catch (error) {
                console.log('Application is not yet ready for use');
            }

             await this.driverHelper.wait(TestConstants.TS_SELENIUM_DEFAULT_POLLING);
        }, timeout);
    }

    private getSelectedRightToolbarButtonLocator(buttonTitle: string): By {
        return By.xpath(`//div[@id='theia-left-content-panel']//ul[@class='p-TabBar-content']` +
            `//li[@title='${buttonTitle}' and contains(@id, 'shell-tab')] and contains(@class, 'p-mod-current')`);
    }

    private getRightToolbarButtonLocator(buttonTitle: String): By {
        return By.xpath(`//div[@id='theia-left-content-panel']//ul[@class='p-TabBar-content']` +
            `//li[@title='${buttonTitle}' and contains(@id, 'shell-tab')]`);
    }

    private getNotificationXpathLocator(notificationText: string): string {
        return `//div[@class='theia-Notification' and contains(@id,'${notificationText}')]`;
    }

}<commit_msg>use regexp to get app url from notification message<commit_after>/*********************************************************************
 * Copyright (c) 2019 Red Hat, Inc.
 *
 * This program and the accompanying materials are made
 * available under the terms of the Eclipse Public License 2.0
 * which is available at https://www.eclipse.org/legal/epl-2.0/
 *
 * SPDX-License-Identifier: EPL-2.0
 **********************************************************************/
import axios from 'axios';
import { DriverHelper } from '../../utils/DriverHelper';
import { injectable, inject } from 'inversify';
import { CLASSES } from '../../inversify.types';
import { TestConstants } from '../../TestConstants';
import { By, WebElement, error } from 'selenium-webdriver';
import { TestWorkspaceUtil, WorkspaceStatus } from '../../utils/workspace/TestWorkspaceUtil';

export enum RightToolbarButton {
    Explorer = 'Explorer',
    Git = 'Git',
    Debug = 'Debug'
}

@injectable()
export class Ide {
    public static readonly EXPLORER_BUTTON_XPATH: string = '(//ul[@class=\'p-TabBar-content\']//li[@title=\'Explorer\'])[1]';
    public static readonly SELECTED_EXPLORER_BUTTON_XPATH: string = '(//ul[@class=\'p-TabBar-content\']//li[@title=\'Explorer\' and contains(@class, \'p-mod-current\')])[1]';
    public static readonly ACTIVATED_IDE_IFRAME_CSS: string = '#ide-iframe-window[aria-hidden=\'false\']';
    public static readonly SELECTED_GIT_BUTTON_XPATH: string = '(//ul[@class=\'p-TabBar-content\']//li[@title=\'Git\' and contains(@class, \'p-mod-current\')])[1]';
    private static readonly TOP_MENU_PANEL_CSS: string = '#theia-app-shell #theia-top-panel .p-MenuBar-content';
    private static readonly LEFT_CONTENT_PANEL_CSS: string = '#theia-left-content-panel';
    private static readonly PRELOADER_CSS: string = '.theia-preload';
    private static readonly IDE_IFRAME_CSS: string = 'iframe#ide-application-iframe';

    constructor(
        @inject(CLASSES.DriverHelper) private readonly driverHelper: DriverHelper,
        @inject(CLASSES.TestWorkspaceUtil) private readonly testWorkspaceUtil: TestWorkspaceUtil) { }

    async waitAndSwitchToIdeFrame(timeout: number = TestConstants.TS_SELENIUM_LOAD_PAGE_TIMEOUT) {
        await this.driverHelper.waitAndSwitchToFrame(By.css(Ide.IDE_IFRAME_CSS), timeout);
    }

    async waitNotification(notificationText: string, timeout: number = TestConstants.TS_SELENIUM_DEFAULT_TIMEOUT) {
        const notificationLocator: By = By.xpath(this.getNotificationXpathLocator(notificationText));

        await this.driverHelper.waitVisibility(notificationLocator, timeout);
    }

    async waitNotificationAndClickOnButton(notificationText: string,
        buttonText: string,
        timeout: number = TestConstants.TS_SELENIUM_DEFAULT_TIMEOUT) {

        await this.driverHelper.getDriver().wait(async () => {
            await this.waitNotification(notificationText, timeout);
            await this.clickOnNotificationButton(notificationText, buttonText);

            try {
                await this.waitNotificationDisappearance(notificationText);
                return true;
            } catch (err) {
                if (!(err instanceof error.TimeoutError)) {
                    throw err;
                }

                console.log(`After clicking on "${buttonText}" button of the notification with text "${notificationText}" \n` +
                    'it is still visible (issue #14121), try again.');

                await this.driverHelper.wait(TestConstants.TS_SELENIUM_DEFAULT_POLLING);
            }
        }, timeout);
    }

    async waitNotificationAndConfirm(notificationText: string, timeout: number = TestConstants.TS_SELENIUM_DEFAULT_TIMEOUT) {
        await this.waitNotificationAndClickOnButton(notificationText, 'yes', timeout);
    }

    async waitNotificationAndOpenLink(notificationText: string, timeout: number = TestConstants.TS_SELENIUM_DEFAULT_TIMEOUT) {
        await this.waitApplicationReadyToUse(await this.getApplicationUrl(notificationText));
        await this.waitNotificationAndClickOnButton(notificationText, 'Open Link', timeout);
    }

    async isNotificationPresent(notificationText: string): Promise<boolean> {
        const notificationLocator: By = By.xpath(this.getNotificationXpathLocator(notificationText));

        return await this.driverHelper.waitVisibilityBoolean(notificationLocator);
    }

    async waitNotificationDisappearance(notificationText: string,
        attempts: number = TestConstants.TS_SELENIUM_DEFAULT_ATTEMPTS,
        polling: number = TestConstants.TS_SELENIUM_DEFAULT_POLLING) {
        const notificationLocator: By = By.xpath(this.getNotificationXpathLocator(notificationText));

        await this.driverHelper.waitDisappearance(notificationLocator, attempts, polling);
    }

    async clickOnNotificationButton(notificationText: string, buttonText: string) {
        const notificationLocator: string = this.getNotificationXpathLocator(notificationText);
        const yesButtonLocator: string = notificationLocator + `//button[text()=\'${buttonText}\']`;

        await this.driverHelper.waitAndClick(By.xpath(yesButtonLocator));
    }

    async waitWorkspaceAndIde(workspaceNamespace: string,
        workspaceName: string,
        timeout: number = TestConstants.TS_SELENIUM_LOAD_PAGE_TIMEOUT) {

        await this.waitAndSwitchToIdeFrame(timeout);
        await this.testWorkspaceUtil.waitWorkspaceStatus(workspaceNamespace, workspaceName, WorkspaceStatus.RUNNING);
        await this.waitIde(timeout);
    }

    async waitIde(timeout: number = TestConstants.TS_SELENIUM_LOAD_PAGE_TIMEOUT) {
        const mainIdeParts: Array<By> = [By.css(Ide.TOP_MENU_PANEL_CSS), By.css(Ide.LEFT_CONTENT_PANEL_CSS), By.xpath(Ide.EXPLORER_BUTTON_XPATH)];

        for (const idePartLocator of mainIdeParts) {
            await this.driverHelper.waitVisibility(idePartLocator, timeout);
        }
    }

    async waitRightToolbarButton(buttonTitle: RightToolbarButton, timeout: number = TestConstants.TS_SELENIUM_DEFAULT_TIMEOUT) {
        const buttonLocator: By = this.getRightToolbarButtonLocator(buttonTitle);

        await this.driverHelper.waitVisibility(buttonLocator, timeout);
    }

    async waitAndClickRightToolbarButton(buttonTitle: RightToolbarButton, timeout: number = TestConstants.TS_SELENIUM_DEFAULT_TIMEOUT) {
        const buttonLocator: By = this.getRightToolbarButtonLocator(buttonTitle);

        await this.driverHelper.waitAndClick(buttonLocator, timeout);
    }

    async waitTopMenuPanel(timeout: number = TestConstants.TS_SELENIUM_DEFAULT_TIMEOUT) {
        await this.driverHelper.waitVisibility(By.css(Ide.TOP_MENU_PANEL_CSS), timeout);
    }

    async waitLeftContentPanel(timeout: number = TestConstants.TS_SELENIUM_DEFAULT_TIMEOUT) {
        await this.driverHelper.waitVisibility(By.css(Ide.LEFT_CONTENT_PANEL_CSS));
    }

    async waitPreloaderAbsent(attempts: number = TestConstants.TS_SELENIUM_DEFAULT_ATTEMPTS,
        polling: number = TestConstants.TS_SELENIUM_DEFAULT_POLLING) {
        await this.driverHelper.waitDisappearance(By.css(Ide.PRELOADER_CSS), attempts, polling);
    }

    async waitStatusBarContains(expectedText: string, timeout: number = TestConstants.TS_SELENIUM_LANGUAGE_SERVER_START_TIMEOUT) {
        const statusBarLocator: By = By.css('div[id=\'theia-statusBar\']');

        await this.driverHelper.getDriver().wait(async () => {
            const elementText: string = await this.driverHelper.waitAndGetText(statusBarLocator, timeout);
            const isTextPresent: boolean = elementText.search(expectedText) > 0;

            if (isTextPresent) {
                return true;
            }

        }, timeout);
    }

    async waitStatusBarTextAbsence(expectedText: string, timeout: number = TestConstants.TS_SELENIUM_DEFAULT_TIMEOUT) {
        const statusBarLocator: By = By.css('div[id=\'theia-statusBar\']');

        // for ensuring that check is not invoked in the gap of status displaying
        for (let i: number = 0; i < 3; i++) {
            await this.driverHelper.getDriver().wait(async () => {
                const elementText: string = await this.driverHelper.waitAndGetText(statusBarLocator, timeout);

                const isTextAbsent: boolean = elementText.search(expectedText) === -1;

                if (isTextAbsent) {
                    return true;
                }

            }, timeout);
        }
    }

    async waitIdeFrameAndSwitchOnIt(timeout: number = TestConstants.TS_SELENIUM_LOAD_PAGE_TIMEOUT) {
        await this.driverHelper.waitAndSwitchToFrame(By.css(Ide.IDE_IFRAME_CSS), timeout);
    }

    async checkLsInitializationStart(expectedTextInStatusBar: string) {
        try {
            await this.waitStatusBarContains(expectedTextInStatusBar, 20000);
        } catch (err) {
            if (!(err instanceof error.TimeoutError)) {
                throw err;
            }

            await this.driverHelper.getDriver().navigate().refresh();
            await this.waitAndSwitchToIdeFrame();
            await this.waitStatusBarContains(expectedTextInStatusBar);
        }

    }

    async closeAllNotifications() {
        const notificationLocator: By = By.css('.theia-Notification');

        if (! await this.driverHelper.isVisible(notificationLocator)) {
            return;
        }

        const notifications: WebElement[] = await this.driverHelper.waitAllPresence(notificationLocator);
        const notificationsCapacity: number = notifications.length;

        for (let i = 1; i <= notificationsCapacity; i++) {
            const notificationLocator: By = By.xpath('//div[@class=\'theia-Notification\']//button[text()=\'Close\']');

            try {
                await this.driverHelper.waitAndClick(notificationLocator);
            } catch (err) {
                if (err instanceof error.TimeoutError) {
                    console.log(`The '${notificationLocator}' element is not visible and can't be clicked`);
                    continue;
                }

                throw err;
            }
        }
    }

    async performKeyCombination(keyCombination: string) {
        const bodyLocator: By = By.tagName('body');

        await this.driverHelper.type(bodyLocator, keyCombination);
    }

    async waitRightToolbarButtonSelection(buttonTitle: string, timeout: number = TestConstants.TS_SELENIUM_DEFAULT_TIMEOUT) {
        const selectedRightToolbarButtonLocator: By = this.getSelectedRightToolbarButtonLocator(buttonTitle);

        await this.driverHelper.waitVisibility(selectedRightToolbarButtonLocator, timeout);
    }

    async getApplicationUrl(notificationText: string) {
        const notificationTextLocator: By = By.xpath(`//div[@class='theia-Notification']//p[contains(@id,'${notificationText}')]`);

        let notification = await this.driverHelper.waitAndGetText(notificationTextLocator);
        let regexp: RegExp = new RegExp('^.*(https?://.*)$');

        return notification.split(regexp)[1];
    }

    async waitApplicationReadyToUse(url: string,
        timeout: number = TestConstants.TS_SELENIUM_DEFAULT_TIMEOUT) {

         await this.driverHelper.getDriver().wait(async () => {
            try {
                let res = await axios.get(url);

                if (res.status === 200) {
                    console.log('Application is ready for use.');
                    return true;
                }
            } catch (error) {
                    console.log('Application is not yet ready for use');
            }

            await this.driverHelper.wait(TestConstants.TS_SELENIUM_DEFAULT_POLLING);
        }, timeout);
    }

    private getSelectedRightToolbarButtonLocator(buttonTitle: string): By {
        return By.xpath(`//div[@id='theia-left-content-panel']//ul[@class='p-TabBar-content']` +
            `//li[@title='${buttonTitle}' and contains(@id, 'shell-tab')] and contains(@class, 'p-mod-current')`);
    }

    private getRightToolbarButtonLocator(buttonTitle: String): By {
        return By.xpath(`//div[@id='theia-left-content-panel']//ul[@class='p-TabBar-content']` +
            `//li[@title='${buttonTitle}' and contains(@id, 'shell-tab')]`);
    }

    private getNotificationXpathLocator(notificationText: string): string {
        return `//div[@class='theia-Notification' and contains(@id,'${notificationText}')]`;
    }

}
{'fraction_non_alphanumeric': 0.1188459824141983, 'fraction_numerical': 0.002633818226022124, 'mean_word_length': 3.341248900615655, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2197223', 'n_tokens_mistral': 6967, 'n_tokens_neox': 6808, 'n_words': 1267}