<commit_before>/*
 * Copyright 2012 Adobe Systems Incorporated. All Rights Reserved.
 */

/*jslint vars: true, plusplus: true, devel: true, browser: true, nomen: true, indent: 4, maxerr: 50 */
/*global define: false, $: false, CodeMirror: false */

/**
 * Set of utilities for simple parsing of CSS text.
 */
define(function (require, exports, module) {
    'use strict';
    
    var Async               = require("Async"),
        DocumentManager     = require("DocumentManager"),
        FileIndexManager    = require("FileIndexManager"),
        NativeFileSystem    = require("NativeFileSystem").NativeFileSystem;

    /**
     * Extracts all CSS selectors from the given text
     * Returns an array of selectors. Each selector is an object with the following properties:
         selector:                 the text of the selector (note: comma separated selector groups like 
                                   "h1, h2" are broken into separate selectors)
         line:                     line in the text where the selector appears
         character:                column in the line where the selector starts
         selectorEndLine:          line where the selector ends
         selectorEndChar:          column where the selector ends
         selectorGroupStartLine:   line where the comma-separated selector group (e.g. .foo, .bar, .baz)
                                   starts that this selector (e.g. .baz) is part of. Particularly relevant for
                                   groups that are on multiple lines.
         selectorGroupStartChar:   column in line where the selector group starts.
         declListStartLine:        line where the declaration list for the rule starts
         declListStartChar:        column in line where the declaration list for the rule starts
         declListEndLine:          line where the declaration list for the rule ends
         declListEndChar:          column in the line where the declaration list for the rule ends
     * @param text {!String} CSS text to extract from
     * @return {Array.<Object>} Array with objects specifying selectors.
     */
    function extractAllSelectors(text) {
        var selectors = [];
        var mode = CodeMirror.getMode({indentUnit: 2}, "css");
        var state = CodeMirror.startState(mode);

        var lines = CodeMirror.splitLines(text);
        var lineCount = lines.length;
        
        var currentSelector = "", currentPosition = -1, selectorStartLine;
        var token, style, stream, i, j;

        var inDeclList = false, inAtRule = false;
        var selectorGroupStartLine = -1, selectorGroupStartChar = -1;
        var declListStartLine = -1, declListStartChar = -1;

        for (i = 0; i < lineCount; ++i) {
            if (currentSelector.trim() !== "") {
                // If we are in a current selector and starting a newline, make sure there is whitespace in the selector
                currentSelector += " ";
            }
            
            stream = new CodeMirror.StringStream(lines[i]);
            while (!stream.eol()) {
                style = mode.token(stream, state);
                token = stream.current();

                // DEBUG STATEMENT -- printer(token, style, i, stream.start, state.stack);

                if (state.stack.indexOf("{") === -1 && // not in a declaration list
                        (state.stack.length === 0 || state.stack[state.stack.length - 1] !== "@media") && // not parsing a media query
                        ((style === null && token !== "{" && token !== "}" && token !== ",") || (style !== null))) { // not at a non-selector token

                    // check for these special cases:
                    if (inAtRule) {
                        // once we're in at @rule, consume tokens until next semi-colon
                        if (token === ";") {
                            inAtRule = false;
                        }
                    } else if (token === "@charset" || token === "@import") {
                        inAtRule = true;
                        currentPosition = -1;  // reset so we don't get @rules following comments
                        selectorGroupStartLine = -1;
                    } else {
                        // detect non-crlf whitespace, comments on same line as '}'
                        if (currentPosition < 0 && (token.trim() !== "") &&
                                !(style === "comment" && stream.start > 0 && lines[i].substr(0, stream.start).indexOf('}') !== -1)) {
                             
                            // start of a new selector, or comment above selector
                            currentSelector = "";
                            currentPosition = stream.start;
                            selectorStartLine = i;
                            if (selectorGroupStartLine < 0) {
                                // this is the start of a new comma-separated selector group
                                // (whenever we start parsing a declaration list, we set selectorGroupStartLine to -1)
                                selectorGroupStartLine = selectorStartLine;
                                selectorGroupStartChar = currentPosition;
                            }
                        }
                        currentSelector += token;
                    }
                } else { // we aren't parsing a selector
                    if (currentSelector.trim() !== "") { // we have a selector, and we parsed something that is not part of a selector, so we just finished parsing a selector
                        selectors.push({selector: currentSelector.trim(),
                                        line: selectorStartLine,
                                        character: currentPosition,
                                        declListEndLine: -1,
                                        selectorEndLine: i,
                                        selectorEndChar: stream.start - 1, // stream.start points to the first char of the non-selector token<commit_msg>switch to RegExp. add suppport for namespace<commit_after>/*
 * Copyright 2012 Adobe Systems Incorporated. All Rights Reserved.
 */

/*jslint vars: true, plusplus: true, devel: true, browser: true, nomen: true, indent: 4, maxerr: 50 */
/*global define: false, $: false, CodeMirror: false */

/**
 * Set of utilities for simple parsing of CSS text.
 */
define(function (require, exports, module) {
    'use strict';
    
    var Async               = require("Async"),
        DocumentManager     = require("DocumentManager"),
        FileIndexManager    = require("FileIndexManager"),
        NativeFileSystem    = require("NativeFileSystem").NativeFileSystem;

    /**
     * Extracts all CSS selectors from the given text
     * Returns an array of selectors. Each selector is an object with the following properties:
         selector:                 the text of the selector (note: comma separated selector groups like 
                                   "h1, h2" are broken into separate selectors)
         line:                     line in the text where the selector appears
         character:                column in the line where the selector starts
         selectorEndLine:          line where the selector ends
         selectorEndChar:          column where the selector ends
         selectorGroupStartLine:   line where the comma-separated selector group (e.g. .foo, .bar, .baz)
                                   starts that this selector (e.g. .baz) is part of. Particularly relevant for
                                   groups that are on multiple lines.
         selectorGroupStartChar:   column in line where the selector group starts.
         declListStartLine:        line where the declaration list for the rule starts
         declListStartChar:        column in line where the declaration list for the rule starts
         declListEndLine:          line where the declaration list for the rule ends
         declListEndChar:          column in the line where the declaration list for the rule ends
     * @param text {!String} CSS text to extract from
     * @return {Array.<Object>} Array with objects specifying selectors.
     */
    function extractAllSelectors(text) {
        var selectors = [];
        var mode = CodeMirror.getMode({indentUnit: 2}, "css");
        var state = CodeMirror.startState(mode);

        var lines = CodeMirror.splitLines(text);
        var lineCount = lines.length;
        
        var currentSelector = "", currentPosition = -1, selectorStartLine;
        var token, style, stream, i, j;

        var inDeclList = false, inAtRule = false;
        var selectorGroupStartLine = -1, selectorGroupStartChar = -1;
        var declListStartLine = -1, declListStartChar = -1;

        for (i = 0; i < lineCount; ++i) {
            if (currentSelector.trim() !== "") {
                // If we are in a current selector and starting a newline, make sure there is whitespace in the selector
                currentSelector += " ";
            }
            
            stream = new CodeMirror.StringStream(lines[i]);
            while (!stream.eol()) {
                style = mode.token(stream, state);
                token = stream.current();

                // DEBUG STATEMENT -- printer(token, style, i, stream.start, state.stack);

                if (state.stack.indexOf("{") === -1 && // not in a declaration list
                        (state.stack.length === 0 || state.stack[state.stack.length - 1] !== "@media") && // not parsing a media query
                        ((style === null && token !== "{" && token !== "}" && token !== ",") || (style !== null))) { // not at a non-selector token

                    // check for these special cases:
                    if (inAtRule) {
                        // once we're in at @rule, consume tokens until next semi-colon
                        if (token === ";") {
                            inAtRule = false;
                        }
                    } else if (token.match(/@(charset|import|namespace)/i)) {
                        inAtRule = true;
                        currentPosition = -1;  // reset so we don't get @rules following comments
                        selectorGroupStartLine = -1;
                    } else {
                        // detect non-crlf whitespace, comments on same line as '}'
                        if (currentPosition < 0 && (token.trim() !== "") &&
                                !(style === "comment" && stream.start > 0 && lines[i].substr(0, stream.start).indexOf('}') !== -1)) {
                             
                            // start of a new selector, or comment above selector
                            currentSelector = "";
                            currentPosition = stream.start;
                            selectorStartLine = i;
                            if (selectorGroupStartLine < 0) {
                                // this is the start of a new comma-separated selector group
                                // (whenever we start parsing a declaration list, we set selectorGroupStartLine to -1)
                                selectorGroupStartLine = selectorStartLine;
                                selectorGroupStartChar = currentPosition;
                            }
                        }
                        currentSelector += token;
                    }
                } else { // we aren't parsing a selector
                    if (currentSelector.trim() !== "") { // we have a selector, and we parsed something that is not part of a selector, so we just finished parsing a selector
                        selectors.push({selector: currentSelector.trim(),
                                        line: selectorStartLine,
                                        character: currentPosition,
                                        declListEndLine: -1,
                                        selectorEndLine: i,
                                        selectorEndChar: stream.start - 1, // stream.start points to the first char of the non-selector token
{'fraction_non_alphanumeric': 0.08840734942737084, 'fraction_numerical': 0.004778775644722749, 'mean_word_length': 1.4796731358529112, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2306162', 'n_tokens_mistral': 2799, 'n_tokens_neox': 2620, 'n_words': 1004}