<commit_before># encoding: utf-8
require "business_catalyst/version"
require "business_catalyst/csv/product_attribute"
require "business_catalyst/csv/transformers"
require "business_catalyst/csv/row"
require "business_catalyst/csv/product_row"
require "business_catalyst/csv/catalog_row"
require "business_catalyst/csv/file_splitter"

module BusinessCatalyst

  # The following characters cause an error message if they appear in a catalog name.
  CATALOG_CHAR_BLACKLIST = /[\\\/;,#:"\|_@=\?]/.freeze
  MORE_STRICT_BLACKLIST = /[\\\/;&,#:"\|\._@=\?]/.freeze

  # Strip all characters out of a catalog name that will cause an error. Replaces them with
  # " " and then squishes all whitespace to single space to preserve word structure.
  def self.sanitize_catalog_name(name)
    return name if name.nil?
    sanitized = name.strip
    sanitized.gsub!(CATALOG_CHAR_BLACKLIST, " ")
    sanitized.gsub!(/\s+/, " ")
    sanitized
  end

  # A guess as to how business catalyst converts names to URL's, based on this blog entry:
  # http://www.businesscatalyst.com/bc-blog/seo-friendly-urls-for-products-and-catalogs
  #
  # Downcases, converts invalid characters and whitespace to '-', and finally removes multiple
  # consecutive dashes and leading and trailing dashes. Does NOT append
  # numbers to ensure uniqueness, you must do this yourself after conversion.
  def self.seo_friendly_url(name)
    name.strip.downcase.gsub(/[^a-z0-9\-]/, "-").gsub(/-{2,}/, "-").gsub(/\A-+|-+\Z/, "")
  end

end<commit_msg>remove commas from catalog char blacklist<commit_after># encoding: utf-8
require "business_catalyst/version"
require "business_catalyst/csv/product_attribute"
require "business_catalyst/csv/transformers"
require "business_catalyst/csv/row"
require "business_catalyst/csv/product_row"
require "business_catalyst/csv/catalog_row"
require "business_catalyst/csv/file_splitter"

module BusinessCatalyst

  # The following characters cause an error message if they appear in a catalog name.
  CATALOG_CHAR_BLACKLIST = /[\\\/;#:"\|_@=\?]/.freeze
  MORE_STRICT_BLACKLIST = /[\\\/;&,#:"\|\._@=\?]/.freeze

  # Strip all characters out of a catalog name that will cause an error. Replaces them with
  # " " and then squishes all whitespace to single space to preserve word structure.
  def self.sanitize_catalog_name(name)
    return name if name.nil?
    sanitized = name.strip
    sanitized.gsub!(CATALOG_CHAR_BLACKLIST, " ")
    sanitized.gsub!(/\s+/, " ")
    sanitized
  end

  # A guess as to how business catalyst converts names to URL's, based on this blog entry:
  # http://www.businesscatalyst.com/bc-blog/seo-friendly-urls-for-products-and-catalogs
  #
  # Downcases, converts invalid characters and whitespace to '-', and finally removes multiple
  # consecutive dashes and leading and trailing dashes. Does NOT append
  # numbers to ensure uniqueness, you must do this yourself after conversion.
  def self.seo_friendly_url(name)
    name.strip.downcase.gsub(/[^a-z0-9\-]/, "-").gsub(/-{2,}/, "-").gsub(/\A-+|-+\Z/, "")
  end

end
{'fraction_non_alphanumeric': 0.1252877342979283, 'fraction_numerical': 0.0026307135810588623, 'mean_word_length': 4.104026845637584, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2816397', 'n_tokens_mistral': 1017, 'n_tokens_neox': 963, 'n_words': 270}