<commit_before>/*
 * Grunt Image Embed
 * https://github.com/ehynds/grunt-image-embed
 *
 * Copyright (c) 2012 <NAME>
 * Licensed under the MIT license.
 */

// Node libs
var fs = require("fs");
var path = require("path");
var mime = require("mime");

var extend = require("extend");
var async = require("async");

// Internal Libs
var fetch = require("./fetch");

// Cache regex's
var rImages = /([\s\S]*?)(url\(([^)]+)\))(?!\s*[;,]?\s*\/\*\s*base64:skip\s*\*\/)|([\s\S]+)/img;
var rExternal = /^(http|https|\/\/)/;
var rSchemeless = /^\/\//;
var rData = /^data:/;
var rQuotes = /['"]/g;
var rParams = /([?#].*)$/g;

// Grunt export wrapper
module.exports = (function () {
  "use strict";

  var exports = {};

  /**
   * Takes a CSS file as input, goes through it line by line, and base64
   * encodes any images it finds.
   *
   * @param srcFile Relative or absolute path to a source stylesheet file.
   * @param opts Options object
   * @param done Function to call once encoding has finished.
   */
  exports.stylesheet = function(file, opts, done) {
    opts = opts || {};

    // Cache of already converted images
    var cache = {};

    // Shift args if no options object is specified
    if(typeof opts === "function") {
      done = opts;
      opts = {};
    }

    var deleteAfterEncoding = opts.deleteAfterEncoding;
    var src = file.contents.toString();
    var result = "";
    var match, img, line, tasks, group;

    async.whilst(function() {
      group = rImages.exec(src);
      return group != null;
    },
    function(complete) {
      // if there is another url to be processed, then:
      //    group[1] will hold everything up to the url declaration
      //    group[2] will hold the complete url declaration (useful if no encoding will take place)
      //    group[3] will hold the contents of the url declaration
      //    group[4] will be undefined
      // if there is no other url to be processed, then group[1-3] will be undefined
      //    group[4] will hold the entire string

      // console.log(group[2]);

      if(group[4] == null) {
        result += group[1];

        img = group[3].trim()
          .replace(rQuotes, "")
          .replace(rParams, ""); // remove query string/hash parmams in the filename, like foo.png?bar or foo.png#bar

        if (opts.extensions) { //test for extensions if it provided
          var imgExt = img.split('.').pop();
          var test = opts.extensions.some(function (ext) {
            return (ext instanceof RegExp) ? ext.test(img) : (ext === imgExt);
          });
          if (!test) {
            if (opts.debug) {
              console.log(img + ' skipped by extensions filter');
            }
            result += group[2];
            return complete();
          }
        }
        // see if this img was already processed before...
        if(cache[img]) {
          // grunt.log.error("The image " + img + " has already been encoded elsewhere in your stylesheet. I'm going to do it again, but it's going to make your stylesheet a lot larger than it needs to be.");
          result = result += cache[img];
          complete();
        } else {
          // process it and put it into the cache
          var loc = img,
            is_local_file = !rData.test(img) && !rExternal.test(img);

          // Resolve the image path relative to the CSS file
          if (is_local_file) {
            // local file system.. fix up the path
            // loc = path.join(path.dirname(file.path), img);<commit_msg>Provide RegExp extension patterns access to the whole URL<commit_after>/*
 * Grunt Image Embed
 * https://github.com/ehynds/grunt-image-embed
 *
 * Copyright (c) 2012 <NAME>
 * Licensed under the MIT license.
 */

// Node libs
var fs = require("fs");
var path = require("path");
var mime = require("mime");

var extend = require("extend");
var async = require("async");

// Internal Libs
var fetch = require("./fetch");

// Cache regex's
var rImages = /([\s\S]*?)(url\(([^)]+)\))(?!\s*[;,]?\s*\/\*\s*base64:skip\s*\*\/)|([\s\S]+)/img;
var rExternal = /^(http|https|\/\/)/;
var rSchemeless = /^\/\//;
var rData = /^data:/;
var rQuotes = /['"]/g;
var rParams = /([?#].*)$/g;

// Grunt export wrapper
module.exports = (function () {
  "use strict";

  var exports = {};

  /**
   * Takes a CSS file as input, goes through it line by line, and base64
   * encodes any images it finds.
   *
   * @param srcFile Relative or absolute path to a source stylesheet file.
   * @param opts Options object
   * @param done Function to call once encoding has finished.
   */
  exports.stylesheet = function(file, opts, done) {
    opts = opts || {};

    // Cache of already converted images
    var cache = {};

    // Shift args if no options object is specified
    if(typeof opts === "function") {
      done = opts;
      opts = {};
    }

    var deleteAfterEncoding = opts.deleteAfterEncoding;
    var src = file.contents.toString();
    var result = "";
    var match, img, line, tasks, group;

    async.whilst(function() {
      group = rImages.exec(src);
      return group != null;
    },
    function(complete) {
      // if there is another url to be processed, then:
      //    group[1] will hold everything up to the url declaration
      //    group[2] will hold the complete url declaration (useful if no encoding will take place)
      //    group[3] will hold the contents of the url declaration
      //    group[4] will be undefined
      // if there is no other url to be processed, then group[1-3] will be undefined
      //    group[4] will hold the entire string

      // console.log(group[2]);

      if(group[4] == null) {
        result += group[1];

        var rawUrl = group[3].trim();
        img = rawUrl
          .replace(rQuotes, "")
          .replace(rParams, ""); // remove query string/hash parmams in the filename, like foo.png?bar or foo.png#bar

        if (opts.extensions) { //test for extensions if it provided
          var imgExt = img.split('.').pop();
          var test = opts.extensions.some(function (ext) {
            return (ext instanceof RegExp) ? ext.test(rawUrl) : (ext === imgExt);
          });
          if (!test) {
            if (opts.debug) {
              console.log(img + ' skipped by extensions filter');
            }
            result += group[2];
            return complete();
          }
        }
        // see if this img was already processed before...
        if(cache[img]) {
          // grunt.log.error("The image " + img + " has already been encoded elsewhere in your stylesheet. I'm going to do it again, but it's going to make your stylesheet a lot larger than it needs to be.");
          result = result += cache[img];
          complete();
        } else {
          // process it and put it into the cache
          var loc = img,
            is_local_file = !rData.test(img) && !rExternal.test(img);

          // Resolve the image path relative to the CSS file
          if (is_local_file) {
            // local file system.. fix up the path
            // loc = path.join(path.dirname(file.path), img);
{'fraction_non_alphanumeric': 0.1425753733446041, 'fraction_numerical': 0.005635390250774866, 'mean_word_length': 2.1848362494392104, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4573746', 'n_tokens_mistral': 2124, 'n_tokens_neox': 2063, 'n_words': 722}