<commit_before>'use strict';

const isEmpty = require('lodash.isempty');
const crypto = require('crypto');
const urlParser = require('url');

module.exports = function(url, options) {
  if (options.useSameDir) {
    return '';
  } else {
    const useHash = options.useHash;
    const parsedUrl = urlParser.parse(decodeURIComponent(url)),
      pathSegments = parsedUrl.pathname.split('/').filter(Boolean);

    if (useHash && !isEmpty(parsedUrl.hash)) {
      const md5 = crypto.createHash('md5'),
        hash = md5
          .update(parsedUrl.hash)
          .digest('hex')
          .substring(0, 8);
      pathSegments.push('hash-' + hash);
    }

    pathSegments.unshift(parsedUrl.hostname);

    pathSegments.unshift('pages');

    if (!isEmpty(parsedUrl.search)) {
      const md5 = crypto.createHash('md5'),
        hash = md5
          .update(parsedUrl.search)
          .digest('hex')
          .substring(0, 8);
      pathSegments.push('query-' + hash);
    }

    pathSegments.push('data');

    pathSegments.forEach(function(segment, index) {
      if (segment) {
        pathSegments[index] = segment.replace(/[^-a-z0-9_.{Arabic}]]/gi, '-');
      }
    });

    return pathSegments.join('/').concat('/');
  }
};<commit_msg>Fixing Arabic characters in a better way (#1012)<commit_after>'use strict';

const isEmpty = require('lodash.isempty');
const crypto = require('crypto');
const urlParser = require('url');

module.exports = function(url, options) {
  if (options.useSameDir) {
    return '';
  } else {
    const useHash = options.useHash;
    const parsedUrl = urlParser.parse(decodeURIComponent(url)),
      pathSegments = parsedUrl.pathname.split('/').filter(Boolean);

    if (useHash && !isEmpty(parsedUrl.hash)) {
      const md5 = crypto.createHash('md5'),
        hash = md5
          .update(parsedUrl.hash)
          .digest('hex')
          .substring(0, 8);
      pathSegments.push('hash-' + hash);
    }

    pathSegments.unshift(parsedUrl.hostname);

    pathSegments.unshift('pages');

    if (!isEmpty(parsedUrl.search)) {
      const md5 = crypto.createHash('md5'),
        hash = md5
          .update(parsedUrl.search)
          .digest('hex')
          .substring(0, 8);
      pathSegments.push('query-' + hash);
    }

    pathSegments.push('data');

    pathSegments.forEach(function(segment, index) {
      if (segment) {
        pathSegments[index] = segment.replace(
          /[^-a-z0-9_.\u0621-\u064A]/gi,
          '-'
        );
      }
    });

    return pathSegments.join('/').concat('/');
  }
};
{'fraction_non_alphanumeric': 0.16503332026656214, 'fraction_numerical': 0.013720109760878087, 'mean_word_length': 2.6509298998569384, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7029440', 'n_tokens_mistral': 834, 'n_tokens_neox': 811, 'n_words': 136}