<commit_before>async = require 'async'
fs    = require 'fs'
path  = require 'path'

pattern = /@import\s(?:url\()?["']([^"';]+?)["']\)?;/i
depth = 0

pad = (levels)->
  ('  ' for x in [1..levels]).join('')

pad_content = (content, levels = depth)->
  if content
    lines = (pad(levels)+x for x in content.split("\n"))
    return lines.join "\n"
  ''


concat_file = (filename, callback)->
  filename = path.resolve filename
  path.exists filename, (exists)->
    if exists
      fs.readFile filename, 'utf8', (err, data)->
        concat_string data, path.dirname(filename), (err, content) ->
          callback err, content
    else
      callback new Error "File #{filename} doesn't exist"


concat_string = (content, filepath, callback)->
  lines = content.split "\n"
  line_numbers = (x for x in [0..lines.length-1])

  check_line = (line_number, next)->

    # Get all matches.
    matches = lines[line_number]?.match pattern

    # No matches? Onward.
    return next() unless matches

    filename = matches[1]

    # Ignore data URIs and protocol-based imports
    return next() unless !filename.match(/^data:/i) && !filename.match(/^[a-zA-Z]+:\/\//)

    # Load the file and concatenate that
    concat_file filepath + '/' + filename, (err, contents)->
      depth++;
      lines[line_number] = \
        """
        /* BEGIN \"#{filename}\" */
        #{pad_content contents}
        /* END \"#{filename}\" */
        """
      depth--;
      next()


  async.forEach line_numbers, check_line, (err)->

    # Nuke empty lines
    lines.forEach (line, index)->
      if line.match /^\s*$/
        lines = lines.slice(0, index).concat lines.slice(index+1)

    callback null, lines.join "\n"

concat = (args, callback)->

  if args.content && args.path
    concat_string args.content, args.path, callback
  else if args.filename
    concat_file args.filename, callback
  else
    callback new Error "Content and path or a filename must be specified to concatenate"

module.exports.concat = concat<commit_msg>Removed empty line nuking for now<commit_after>async = require 'async'
fs    = require 'fs'
path  = require 'path'

pattern = /@import\s(?:url\()?["']([^"';]+?)["']\)?;/i
depth = 0

pad = (levels)->
  ('  ' for x in [1..levels]).join('')

pad_content = (content, levels = depth)->
  if content
    lines = (pad(levels)+x for x in content.split("\n"))
    return lines.join "\n"
  ''


concat_file = (filename, callback)->
  filename = path.resolve filename
  path.exists filename, (exists)->
    if exists
      fs.readFile filename, 'utf8', (err, data)->
        concat_string data, path.dirname(filename), (err, content) ->
          callback err, content
    else
      callback new Error "File #{filename} doesn't exist"


concat_string = (content, filepath, callback)->
  lines = content.split "\n"
  line_numbers = (x for x in [0..lines.length-1])

  check_line = (line_number, next)->

    # Get all matches.
    matches = lines[line_number]?.match pattern

    # No matches? Onward.
    return next() unless matches

    filename = matches[1]

    # Ignore data URIs and protocol-based imports
    return next() unless !filename.match(/^data:/i) && !filename.match(/^[a-zA-Z]+:\/\//)

    # Load the file and concatenate that
    concat_file filepath + '/' + filename, (err, contents)->
      depth++;
      lines[line_number] = \
        """
        /* BEGIN \"#{filename}\" */
        #{pad_content contents}
        /* END \"#{filename}\" */
        """
      depth--;
      next()


  async.forEach line_numbers, check_line, (err)->

    # Nuke empty lines
    ###
    lines.forEach (line, index)->
      if line.match /^\s*$/
        lines = lines.slice(0, index).concat lines.slice(index+1)
    ###

    callback null, lines.join "\n"

concat = (args, callback)->

  if args.content && args.path
    concat_string args.content, args.path, callback
  else if args.filename
    concat_file args.filename, callback
  else
    callback new Error "Content and path or a filename must be specified to concatenate"

module.exports.concat = concat
{'fraction_non_alphanumeric': 0.15724815724815724, 'fraction_numerical': 0.003931203931203931, 'mean_word_length': 2.728021978021978, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6491092', 'n_tokens_mistral': 1307, 'n_tokens_neox': 1252, 'n_words': 370}