<commit_before>

carto_quotables = [
  'text-face-name'
];

carto_variables = [
  'text-name'
];

var carto_functionMap= {
  'Equal Interval': 'quartiles',
  'Jenks': 'jenkBins',
  'Heads/Tails': 'headTails',
  'Quantile': 'quantileBins'
};

DEFAULT_QFUNCTION = 'Quantile';

/**
 *  Manage some carto properties depending on
 *  type (line, polygon or point), for choropleth.
 */
function manage_choropleth_props(type, props) {
  var carto_props = {
    'marker-width': props['marker-width'],
    'marker-opacity': props['marker-opacity'],
    'marker-line-width': props['marker-line-width'],
    'marker-line-color': props['marker-line-color'],
    'marker-line-opacity': props['marker-line-opacity'],
    'marker-allow-overlap': props['marker-allow-overlap'],
    'line-color': props['line-color'],
    'line-opacity': props['line-opacity'],
    'line-width': props['line-width'],
    'polygon-opacity': type == "line" ? 0 : props['polygon-opacity'],
    'text-name': props['text-name'],
    'text-halo-fill': props['text-halo-fill'],
    'text-halo-radius': props['text-halo-radius'],
    'text-face-name': props['text-face-name'],
    'text-size': props['text-size'],
    'text-dy': props['text-dy'],
    'text-allow-overlap': props['text-allow-overlap'],
    'text-placement': props['text-placement'],
    'text-placement-type': props['text-placement-type'],
    'text-label-position-tolerance': props['text-label-position-tolerance'],
    'text-fill': props['text-fill']
  }

  // Remove all undefined properties
  _.each(carto_props, function(v, k){
    if(v === undefined) delete carto_props[k];
  });

  return carto_props;
}

function getProp(obj, prop) {
  var p = [];
  for(var k in obj) {
    var v = obj[k];
    if (k === prop) {
      p.push(v);
    } else if (typeof(v) === 'object') {
      p = p.concat(getProp(v, prop));
    }
  }
  return p;
}

var _cartocss_spec_props = getProp(_mapnik_reference_latest, 'css');

/**
 * some carto properties depends on others, this function
 * remove or add properties needed to carto works
 */
function manage_carto_properies(props) {

  if(/none/i.test(props['text-name']) || !props['text-name']) {
    // remove all text-* properties
    for(var p in props) {
      if(isTextProperty(p)) {
        delete props[p];
      }
    }
  }

  if(/none/i.test(props['polygon-comp-op'])) {
    delete props['polygon-comp-op'];
  }
  if(/none/i.test(props['line-comp-op'])) {
    delete props['line-comp-op'];
  }
  if(/none/i.test(props['marker-comp-op'])) {
    delete props['marker-comp-op'];
  }

  // if polygon-pattern-file is present polygon-fill should be removed
  if('polygon-pattern-file' in props) {
    delete props['polygon-fill'];
  }

  delete props.zoom;

  return _.pick(props, _cartocss_spec_props)

}

function isTextProperty(p) {
  return /^text-/.test(p);
}

function generate_carto_properties(props) {
  return _(props).map(function(v, k) {
    if(_.include(carto_quotables, k)) {
      v = "'" + v + "'";
    }
    if(_.include(carto_variables, k)) {
      v = "[" + v + "]";
    }
    return "  " + k + ": " + v + ";";
  });
}

function filter_props(props, fn) {
  var p = {};
  for(var k in props) {
    var v = props[k];
    if(fn(k, v)) {
      p[k] = v;
    }
  }
  return p;
}

function simple_polygon_generator(table, props, changed, callback) {

  // remove unnecesary properties, for example
  // if the text-name is not present remove all the
  // properties related to text
  props = manage_carto_properies(props);

  var text_properties = filter_props(props, function(k, v) { return isTextProperty(k); });
  var general_properties = filter_props(props, function(k, v) { return !isTextProperty(k); });

  generalLayerProps = generate_carto_properties(general_properties);
  textLayerProps = generate_carto_properties(text_properties);

  // layer with non-text properties
  var generalLayer = "#" + table.get('name') + "{\n" + generalLayerProps.join('\n') + "\n}";
  var textLayer = '';
  if(_.size(textLayerProps)) {
    textLayer = "\n\n#" + table.get('name') + "::labels {\n" + textLayerProps.join('\n') + "\n}\n";
  }

  // text properties layer
  callback(generalLayer + textLayer);
}

function intensity_generator(table, props, changed, callback) {
  var table_name = table.get('name');

  // remove unnecesary properties, for example
  // if the text-name is not present remove all the
  // properties related to text
  props = manage_carto_properies(props);

  var carto_props = {
    'marker-fill': props['marker-fill'],
    'marker-width': props['marker-width'],
    'marker-line-color': props['marker-line-color'],
    'marker-line-width': props['marker-line-width'],
    'marker-line-opacity': props['marker-line-opacity'],
    'marker-opacity': props['marker-opacity'],
    'marker-comp-op': 'multiply',
    'marker-type': 'ellipse',
    'marker-placement': 'point',
    'marker-allow-overlap': true,
    'marker-clip': false,
    'marker-multi-policy': 'largest'
  };

  var css = "\n#" + table_name +"{\n";

  _(carto_props).each(function(prop, name) {
    css += "  " + name + ": " + prop + "; \n";
  });

  css += "}";
  callback(css);

}

function cluster_sql(table, zoom, props, nquartiles) {

  var grids = ["A", "B", "C", "D", "E"];
  var sizes = [5, 25, 125, 325, 625];
  var baseSize = 12;

  var size = baseSize * (nquartiles + 1);
  var bucket = "bucket" + grids[0];
  var mainBucket = bucket;

  var sql = "WITH filtered_table AS <%= table %>, ";

  sql += "hgridA AS (SELECT CDB_HexagonGrid(ST_Expand(!bbox!, greatest(!pixel_width!,!pixel_height!) * " + size + "), greatest(!pixel_width!,!pixel_height!) * " + size + ") as cell), " +

  bucket + " AS (" +
  "\nSELECT * FROM (SELECT ST_Centroid(ST_Collect(i.the_geom_webmercator)) AS the_geom_webmercator, COUNT(i.cartodb_id) AS points_count, 1 AS cartodb_id, array_agg(cartodb_id) AS id_list " +
  "\nFROM hgridA, filtered_table i " +
  "\nWHERE ST_Intersects(i.the_geom_webmercator, hgridA.cell) " +
  "\n\tGROUP BY hgridA.cell) t " +
  "\n\tWHERE points_count > " + sizes[nquartiles - 1] +
  "), ";

  for (var i = 1; i < nquartiles; i++) {

    var hgrid = "hgrid" + grids[i];
    size = baseSize * (nquartiles - i + 1);
    bucket = "bucket" + grids[i];

    sql += "\n\n" + hgrid + " AS (SELECT CDB_HexagonGrid(ST_Expand(!bbox!, greatest(!pixel_width!,!pixel_height!) * " + size + "), greatest(!pixel_width!,!pixel_height!) * " + size + ") AS cell), " +
    "\n" + bucket + " AS (" +
    "\n\tSELECT * FROM (SELECT ST_Centroid(ST_Collect(i.the_geom_webmercator)) AS the_geom_webmercator, COUNT(i.cartodb_id) AS points_count, 1 AS cartodb_id, array_agg(cartodb_id) AS id_list " +
    "\n\tFROM " + hgrid + ", filtered_table i " +
    "\n\tWHERE ST_Intersects(i.the_geom_webmercator, " + hgrid + ".cell) ";

    for (var j = 0; j <= i - 1; j++) {
      var b = "bucket" + grids[j];
      sql += "\n\tAND cartodb_id NOT IN (SELECT unnest(id_list) FROM " + b + ") ";
    }

    sql += "\n\t\tGROUP BY " + hgrid + ".cell) t " +
    "\n\t\t WHERE points_count >  "  + sizes[nquartiles - i - 1] +
    ") ";

    if  (i < nquartiles - 1) sql += ", ";

  }

  sql += "\nSELECT the_geom_webmercator, 1 points_count, cartodb_id, ARRAY[cartodb_id] AS id_list, 'origin' AS src, cartodb_id::text cdb_list FROM filtered_table " +

  "\nWHERE cartodb_id NOT IN (select unnest(id_list) FROM " + mainBucket + ") ";

  for (var i = 1; i < nquartiles; i++) {
    bucket     = "bucket" + grids[i];
    sql += "\n\tAND cartodb_id NOT IN (select unnest(id_list) FROM " + bucket + ") ";
  }

  sql +="\nUNION ALL SELECT *, '" + mainBucket + "' AS src, array_to_string(id_list, ',') ";<commit_msg>Adds metatile_extent<commit_after>

carto_quotables = [
  'text-face-name'
];

carto_variables = [
  'text-name'
];

var carto_functionMap= {
  'Equal Interval': 'quartiles',
  'Jenks': 'jenkBins',
  'Heads/Tails': 'headTails',
  'Quantile': 'quantileBins'
};

DEFAULT_QFUNCTION = 'Quantile';

/**
 *  Manage some carto properties depending on
 *  type (line, polygon or point), for choropleth.
 */
function manage_choropleth_props(type, props) {
  var carto_props = {
    'marker-width': props['marker-width'],
    'marker-opacity': props['marker-opacity'],
    'marker-line-width': props['marker-line-width'],
    'marker-line-color': props['marker-line-color'],
    'marker-line-opacity': props['marker-line-opacity'],
    'marker-allow-overlap': props['marker-allow-overlap'],
    'line-color': props['line-color'],
    'line-opacity': props['line-opacity'],
    'line-width': props['line-width'],
    'polygon-opacity': type == "line" ? 0 : props['polygon-opacity'],
    'text-name': props['text-name'],
    'text-halo-fill': props['text-halo-fill'],
    'text-halo-radius': props['text-halo-radius'],
    'text-face-name': props['text-face-name'],
    'text-size': props['text-size'],
    'text-dy': props['text-dy'],
    'text-allow-overlap': props['text-allow-overlap'],
    'text-placement': props['text-placement'],
    'text-placement-type': props['text-placement-type'],
    'text-label-position-tolerance': props['text-label-position-tolerance'],
    'text-fill': props['text-fill']
  }

  // Remove all undefined properties
  _.each(carto_props, function(v, k){
    if(v === undefined) delete carto_props[k];
  });

  return carto_props;
}

function getProp(obj, prop) {
  var p = [];
  for(var k in obj) {
    var v = obj[k];
    if (k === prop) {
      p.push(v);
    } else if (typeof(v) === 'object') {
      p = p.concat(getProp(v, prop));
    }
  }
  return p;
}

var _cartocss_spec_props = getProp(_mapnik_reference_latest, 'css');

/**
 * some carto properties depends on others, this function
 * remove or add properties needed to carto works
 */
function manage_carto_properies(props) {

  if(/none/i.test(props['text-name']) || !props['text-name']) {
    // remove all text-* properties
    for(var p in props) {
      if(isTextProperty(p)) {
        delete props[p];
      }
    }
  }

  if(/none/i.test(props['polygon-comp-op'])) {
    delete props['polygon-comp-op'];
  }
  if(/none/i.test(props['line-comp-op'])) {
    delete props['line-comp-op'];
  }
  if(/none/i.test(props['marker-comp-op'])) {
    delete props['marker-comp-op'];
  }

  // if polygon-pattern-file is present polygon-fill should be removed
  if('polygon-pattern-file' in props) {
    delete props['polygon-fill'];
  }

  delete props.zoom;

  return _.pick(props, _cartocss_spec_props)

}

function isTextProperty(p) {
  return /^text-/.test(p);
}

function generate_carto_properties(props) {
  return _(props).map(function(v, k) {
    if(_.include(carto_quotables, k)) {
      v = "'" + v + "'";
    }
    if(_.include(carto_variables, k)) {
      v = "[" + v + "]";
    }
    return "  " + k + ": " + v + ";";
  });
}

function filter_props(props, fn) {
  var p = {};
  for(var k in props) {
    var v = props[k];
    if(fn(k, v)) {
      p[k] = v;
    }
  }
  return p;
}

function simple_polygon_generator(table, props, changed, callback) {

  // remove unnecesary properties, for example
  // if the text-name is not present remove all the
  // properties related to text
  props = manage_carto_properies(props);

  var text_properties = filter_props(props, function(k, v) { return isTextProperty(k); });
  var general_properties = filter_props(props, function(k, v) { return !isTextProperty(k); });

  generalLayerProps = generate_carto_properties(general_properties);
  textLayerProps = generate_carto_properties(text_properties);

  // layer with non-text properties
  var generalLayer = "#" + table.get('name') + "{\n" + generalLayerProps.join('\n') + "\n}";
  var textLayer = '';
  if(_.size(textLayerProps)) {
    textLayer = "\n\n#" + table.get('name') + "::labels {\n" + textLayerProps.join('\n') + "\n}\n";
  }

  // text properties layer
  callback(generalLayer + textLayer);
}

function intensity_generator(table, props, changed, callback) {
  var table_name = table.get('name');

  // remove unnecesary properties, for example
  // if the text-name is not present remove all the
  // properties related to text
  props = manage_carto_properies(props);

  var carto_props = {
    'marker-fill': props['marker-fill'],
    'marker-width': props['marker-width'],
    'marker-line-color': props['marker-line-color'],
    'marker-line-width': props['marker-line-width'],
    'marker-line-opacity': props['marker-line-opacity'],
    'marker-opacity': props['marker-opacity'],
    'marker-comp-op': 'multiply',
    'marker-type': 'ellipse',
    'marker-placement': 'point',
    'marker-allow-overlap': true,
    'marker-clip': false,
    'marker-multi-policy': 'largest'
  };

  var css = "\n#" + table_name +"{\n";

  _(carto_props).each(function(prop, name) {
    css += "  " + name + ": " + prop + "; \n";
  });

  css += "}";
  callback(css);

}

function cluster_sql(table, zoom, props, nquartiles) {

  var grids = ["A", "B", "C", "D", "E"];
  var sizes = [5, 25, 125, 325, 625];
  var baseSize = 12;

  var size = baseSize * (nquartiles + 1);
  var bucket = "bucket" + grids[0];
  var mainBucket = bucket;

  var sql = "WITH metatile_extent AS ( SELECT ST_SetSRID('BOX3D(-9715452.043159042 4529964.044292687,-9617612.646954017 4627803.440497711)'::box3d, 3857) as ext ), " +
  "  filtered_table AS ( SELECT t.* FROM <%= table %> t, metatile_extent m WHERE t.the_geom_webmercator && m.ext ), " +
  "hgridA AS (SELECT ST_SnapToGrid(ST_Expand(!bbox!, greatest(!pixel_width!,!pixel_height!) * " + size + "), greatest(!pixel_width!,!pixel_height!) * " + size + ") as cell), " +
  bucket + " AS (" +
  "\nSELECT * FROM (SELECT ST_Centroid(ST_Collect(i.the_geom_webmercator)) AS the_geom_webmercator, COUNT(i.cartodb_id) AS points_count, 1 AS cartodb_id, array_agg(cartodb_id) AS id_list " +
  "\nFROM hgridA, filtered_table i " +
  "\nWHERE ST_Intersects(i.the_geom_webmercator, hgridA.cell) " +
  "\n\tGROUP BY hgridA.cell) t " +
  "\n\tWHERE points_count > " + sizes[nquartiles - 1] +
  "), ";

  for (var i = 1; i < nquartiles; i++) {

    var hgrid = "hgrid" + grids[i];
    size = baseSize * (nquartiles - i + 1);
    bucket = "bucket" + grids[i];

    sql += "\n\n" + hgrid + " AS (SELECT ST_SnapToGrid(ST_Expand(!bbox!, greatest(!pixel_width!,!pixel_height!) * " + size + "), greatest(!pixel_width!,!pixel_height!) * " + size + ") AS cell), " +
    "\n" + bucket + " AS (" +
    "\n\tSELECT * FROM (SELECT ST_Centroid(ST_Collect(i.the_geom_webmercator)) AS the_geom_webmercator, COUNT(i.cartodb_id) AS points_count, 1 AS cartodb_id, array_agg(cartodb_id) AS id_list " +
    "\n\tFROM " + hgrid + ", filtered_table i " +
    "\n\tWHERE ST_Intersects(i.the_geom_webmercator, " + hgrid + ".cell) ";

    for (var j = 0; j <= i - 1; j++) {
      var b = "bucket" + grids[j];
      sql += "\n\tAND cartodb_id NOT IN (SELECT unnest(id_list) FROM " + b + ") ";
    }

    sql += "\n\t\tGROUP BY " + hgrid + ".cell) t " +
    "\n\t\t WHERE points_count >  "  + sizes[nquartiles - i - 1] +
    ") ";

    if  (i < nquartiles - 1) sql += ", ";

  }

  sql += "\nSELECT the_geom_webmercator, 1 points_count, cartodb_id, ARRAY[cartodb_id] AS id_list, 'origin' AS src, cartodb_id::text cdb_list FROM filtered_table " +

  "\nWHERE cartodb_id NOT IN (select unnest(id_list) FROM " + mainBucket + ") ";

  for (var i = 1; i < nquartiles; i++) {
    bucket     = "bucket" + grids[i];
    sql += "\n\tAND cartodb_id NOT IN (select unnest(id_list) FROM " + bucket + ") ";
  }

  sql +="\nUNION ALL SELECT *, '" + mainBucket + "' AS src, array_to_string(id_list, ',') ";
{'fraction_non_alphanumeric': 0.1816598955310505, 'fraction_numerical': 0.00812536273940801, 'mean_word_length': 3.4885672937771344, 'pattern_counts': {'":': 6, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7383102', 'n_tokens_mistral': 5844, 'n_tokens_neox': 5558, 'n_words': 1184}