<commit_before>"""Filesystem related helper functions.
"""

import contextlib
import errno
import os
import shutil
import sys
import tarfile
import tempfile
import urllib2


def mkdir_p(path):
  try:
    os.makedirs(path)
  except OSError as e:
    if e.errno != errno.EEXIST:
      raise


def rm_f(path):
  try:
    os.remove(path)
  except OSError as e:
    if e.errno != errno.ENOENT:
      raise


def rm_rf(path):
  try:
    shutil.rmtree(path)
  except OSError as e:
    if e.errno != errno.ENOENT:
      raise


def safe_unlink(path):
  try:
    os.unlink(path)
  except OSError as e:
    if e.errno != errno.ENOENT:
      raise

def byte_to_mb(n):
  return str(n / 1024 / 1024) + 'MB'

def download_and_extract(destination, url):
  print url
  with tempfile.TemporaryFile() as t:
    with contextlib.closing(urllib2.urlopen(url)) as u:
      total = int(u.headers['content-length'])
      done = 0
      last_length = 0
      while True:
        chunk = u.read(1024)
        done += len(chunk)
        if not len(chunk):
          break
        percent = '{0:.2f}'.format(round(float(done) / float(total), 4) * 100) + '%'
        ratio = '(' + byte_to_mb(done) + '/' + byte_to_mb(total) + ')'
        line = '-> ' + percent + ' ' + ratio
        c_length = last_length
        last_length = len(line)
        sys.stderr.write(line.ljust(c_length) + '\r')
        sys.stderr.flush()
        t.write(chunk)
    sys.stderr.write('\nExtracting...\n')
    sys.stderr.flush()
    with tarfile.open(fileobj=t, mode='r:bz2') as z:
      z.extractall(destination)
<commit_msg>chore: remove unsued variable in filesystem script<commit_after>"""Filesystem related helper functions.
"""

import contextlib
import errno
import os
import shutil
import sys
import tarfile
import tempfile
import urllib2


def mkdir_p(path):
  try:
    os.makedirs(path)
  except OSError as e:
    if e.errno != errno.EEXIST:
      raise


def rm_f(path):
  try:
    os.remove(path)
  except OSError as e:
    if e.errno != errno.ENOENT:
      raise


def rm_rf(path):
  try:
    shutil.rmtree(path)
  except OSError as e:
    if e.errno != errno.ENOENT:
      raise


def safe_unlink(path):
  try:
    os.unlink(path)
  except OSError as e:
    if e.errno != errno.ENOENT:
      raise

def byte_to_mb(n):
  return str(n / 1024 / 1024) + 'MB'

def download_and_extract(destination, url):
  print url
  with tempfile.TemporaryFile() as t:
    with contextlib.closing(urllib2.urlopen(url)) as u:
      total = int(u.headers['content-length'])
      done = 0
      last_length = 0
      while True:
        chunk = u.read(1024)
        done += len(chunk)
        if not len(chunk):
          break
        percent = '{0:.2f}%%'.format(round(float(done) / float(total), 4) * 100)
        ratio = '(' + byte_to_mb(done) + '/' + byte_to_mb(total) + ')'
        line = '-> ' + percent + ' ' + ratio
        sys.stderr.write(line.ljust(c_length) + '\r')
        last_length = len(line)
        sys.stderr.flush()
        t.write(chunk)
    sys.stderr.write('\nExtracting...\n')
    sys.stderr.flush()
    with tarfile.open(fileobj=t, mode='r:bz2') as z:
      z.extractall(destination)

{'fraction_non_alphanumeric': 0.13316423589093215, 'fraction_numerical': 0.014584654407102092, 'mean_word_length': 2.569004524886878, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3361516', 'n_tokens_mistral': 1190, 'n_tokens_neox': 1122, 'n_words': 266}