<commit_before>/* global module */


var fs = require('fs');
var async = require('async');
var cssom = require('cssom');
var cheerio = require('cheerio');
var url = require('url');
var crypto = require('crypto');
var request = require('request');


var ucss = {
    /**
     * Find selectors in CSS string
     * @param {String} css CSS code
     * @param {Object} selectors (optional) object to append matched selectors
     *        to.
     * @returns {Object} Object containing matched selectors, and number of
     *           occurences for each selector.
     */
    _extractSelectorsFromCssString: function(self, css, selectors) {
        if (!selectors) {
            selectors = {};
        }
        if (!css) {
            return selectors;
        }

        // Delete unsupported rules before CSSOM parsing, to avoid crash
        // TODO: Remove these, when/if they get supported by CSSOM
        var unsupported = [
            // "@supports { .foo { ... }}" or
            // "@-prefix-supports { .foo { ... }}"
            /@-*\w*-*supports\s.*?\}\s*?\}/g,

            // "@document url(http://example.com) { .foo { ... }}" or
            // "@-prefix-document url(http://example.com) { .foo { ... }}"
            /@-*\w*-*document\s.*?\}\s*?\}/g];
        for (var i=0, l=unsupported.length; i<l; i++) {
            css = css.replace(unsupported[i], "");
        }

        var styles = cssom.parse(css);
        var rules = styles.cssRules;

        if (!rules) {
            return selectors;
        }

        selectors = self._getSelectorsFromRules(self, rules, selectors);

        return selectors;
    },

    // TODO: Add docstring
    _getSelectorsFromRules: function(self, rules, selectors) {
        for (var i=0, l=rules.length; i<l; i++) {
            var rule = rules[i];

            // @-rules are ignored, except media queries. For media queries,
            // child rules are handled. Other rules are handled as if they
            // have a selector text.
            //
            // @media:
            if (rule.media && rule.cssRules) {
                selectors = self._getSelectorsFromRules(self, rule.cssRules, selectors);

                // Rules without selectorText are not processed (@-rules,
                // except @media)
            } else if (!rule.selectorText) {
                // Cleaning: Only want the first part (e.g. @font-face),
                // not full rule
                var sel = rule.cssText.split("{")[0].trim();
                selectors[sel] = null;

                // Other rules, containing selector(s)
            } else {
                var selectorGroup = rule.selectorText;

                // Several selectors can be grouped together, separated by
                // comma, e.g. ".foo, .bar":
                var selectorList = selectorGroup.split(",");

                for (var j=0, sl=selectorList.length; j<sl; j++) {
                    var s = selectorList[j].trim();
                    selectors[s] = (selectors[s] || 0) + 1;
                }
            }
        }
        return selectors;
    },

    /**
     * Get CSS selectors from CSS
     * @param {Array} css CSS to search through. This can be either an array
     *         of Strings (containing CSS), an array of URIs to CSS files, or
     *         an array of paths to CSS files.
     * @param {Function} doneCallback Callback for when done. Should take two
     *         arguments, one list of selectors and one set of results (see
     *         documentation for _processItem for more info on these)
     */
    _collectCssSelectors: function(self, css, doneCallback) {
        var result = { used: {}, duplicates: {}, ignored: {} };
        var matchedSelectors = {}; // Selectors matched in CSS file
        var selectors = []; // Selectors to search for in HTML

        // Find all selectors
        async.forEach(css, function(item, forEachCallback) {
            if (0 === item.indexOf("http")) { // From URI
                var options = { uri: item };

                request.get(options, function(error, res, data) {
                        matchedSelectors = ucss._extractSelectorsFromCssString(
                            self, data, matchedSelectors);
                        forEachCallback();
                });

                return;
            } else if (-1 === item.indexOf("{")) { // From file
                try {
                    item = fs.readFileSync(item).toString(); // TODO: Could this be async?
                } catch (e) {
                    console.log(e.message);
                }
                matchedSelectors = ucss._extractSelectorsFromCssString(self, item, matchedSelectors);
            } else { // From string
                matchedSelectors = ucss._extractSelectorsFromCssString(self, item, matchedSelectors);
            }

            forEachCallback();
        }, function(err) {
            if (err) {
                // TODO: Error handling
            }

            if (!matchedSelectors) {
                return null;
            }
            for (var selector in matchedSelectors) {
                if ("" === selector) {
                    continue;
                }
                selectors.push(selector);
                if (matchedSelectors[selector] > 1) {
                    result.duplicates[selector] = matchedSelectors[selector];
                }
            }
            if (doneCallback) {
                doneCallback(selectors, result);
            }
        });
    },

    /*<commit_msg>Rename callback.<commit_after>/* global module */


var fs = require('fs');
var async = require('async');
var cssom = require('cssom');
var cheerio = require('cheerio');
var url = require('url');
var crypto = require('crypto');
var request = require('request');


var ucss = {
    /**
     * Find selectors in CSS string
     * @param {String} css CSS code
     * @param {Object} selectors (optional) object to append matched selectors
     *        to.
     * @returns {Object} Object containing matched selectors, and number of
     *           occurences for each selector.
     */
    _extractSelectorsFromCssString: function(self, css, selectors) {
        if (!selectors) {
            selectors = {};
        }
        if (!css) {
            return selectors;
        }

        // Delete unsupported rules before CSSOM parsing, to avoid crash
        // TODO: Remove these, when/if they get supported by CSSOM
        var unsupported = [
            // "@supports { .foo { ... }}" or
            // "@-prefix-supports { .foo { ... }}"
            /@-*\w*-*supports\s.*?\}\s*?\}/g,

            // "@document url(http://example.com) { .foo { ... }}" or
            // "@-prefix-document url(http://example.com) { .foo { ... }}"
            /@-*\w*-*document\s.*?\}\s*?\}/g];
        for (var i=0, l=unsupported.length; i<l; i++) {
            css = css.replace(unsupported[i], "");
        }

        var styles = cssom.parse(css);
        var rules = styles.cssRules;

        if (!rules) {
            return selectors;
        }

        selectors = self._getSelectorsFromRules(self, rules, selectors);

        return selectors;
    },

    // TODO: Add docstring
    _getSelectorsFromRules: function(self, rules, selectors) {
        for (var i=0, l=rules.length; i<l; i++) {
            var rule = rules[i];

            // @-rules are ignored, except media queries. For media queries,
            // child rules are handled. Other rules are handled as if they
            // have a selector text.
            //
            // @media:
            if (rule.media && rule.cssRules) {
                selectors = self._getSelectorsFromRules(self, rule.cssRules, selectors);

                // Rules without selectorText are not processed (@-rules,
                // except @media)
            } else if (!rule.selectorText) {
                // Cleaning: Only want the first part (e.g. @font-face),
                // not full rule
                var sel = rule.cssText.split("{")[0].trim();
                selectors[sel] = null;

                // Other rules, containing selector(s)
            } else {
                var selectorGroup = rule.selectorText;

                // Several selectors can be grouped together, separated by
                // comma, e.g. ".foo, .bar":
                var selectorList = selectorGroup.split(",");

                for (var j=0, sl=selectorList.length; j<sl; j++) {
                    var s = selectorList[j].trim();
                    selectors[s] = (selectors[s] || 0) + 1;
                }
            }
        }
        return selectors;
    },

    /**
     * Get CSS selectors from CSS
     * @param {Array} css CSS to search through. This can be either an array
     *         of Strings (containing CSS), an array of URIs to CSS files, or
     *         an array of paths to CSS files.
     * @param {Function} doneCallback Callback for when done. Should take two
     *         arguments, one list of selectors and one set of results (see
     *         documentation for _processItem for more info on these)
     */
    _collectCssSelectors: function(self, css, selectorsCollectedCallback) {
        var result = { used: {}, duplicates: {}, ignored: {} };
        var matchedSelectors = {}; // Selectors matched in CSS file
        var selectors = []; // Selectors to search for in HTML

        // Find all selectors
        async.forEach(css, function(item, forEachCallback) {
            if (0 === item.indexOf("http")) { // From URI
                var options = { uri: item };

                request.get(options, function(error, res, data) {
                        matchedSelectors = ucss._extractSelectorsFromCssString(
                            self, data, matchedSelectors);
                        forEachCallback();
                });

                return;
            } else if (-1 === item.indexOf("{")) { // From file
                try {
                    item = fs.readFileSync(item).toString(); // TODO: Could this be async?
                } catch (e) {
                    console.log(e.message);
                }
                matchedSelectors = ucss._extractSelectorsFromCssString(self, item, matchedSelectors);
            } else { // From string
                matchedSelectors = ucss._extractSelectorsFromCssString(self, item, matchedSelectors);
            }

            forEachCallback();
        }, function(err) {
            if (err) {
                // TODO: Error handling
            }

            if (!matchedSelectors) {
                return null;
            }
            for (var selector in matchedSelectors) {
                if ("" === selector) {
                    continue;
                }
                selectors.push(selector);
                if (matchedSelectors[selector] > 1) {
                    result.duplicates[selector] = matchedSelectors[selector];
                }
            }
            if (selectorsCollectedCallback) {
                selectorsCollectedCallback(selectors, result);
            }
        });
    },

    /*
{'fraction_non_alphanumeric': 0.1212148441009974, 'fraction_numerical': 0.0016173959924521521, 'mean_word_length': 1.5141179128077704, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '285531', 'n_tokens_mistral': 3002, 'n_tokens_neox': 2845, 'n_words': 831}