<commit_before>import os
import internetarchive
import patoolib
import shutil

list_of_archives = [['Pauly Supp.rar', 'PaulySupp'],
                    ['Pauly-Wissowa_54-66.rar', 'PaulyWissowa5466'],
                    ['Pauly-Wissowa_41-53.rar', 'PaulyWissowa4153_201412'],
                    ['Pauly-Wissowa_31-40.rar', 'PaulyWissowa3140_201412'],
                    ['Pauly-Wissowa_21-30.rar', 'PaulyWissowa2130'],
                    ['Pauly-Wissowa_11-20.rar', 'PaulyWissowa1120'],
                    ['Pauly-Wissowa_1-10.rar', 'PaulyWissowa110']]

def get_archive(filename, item):
    archivename = os.getcwd() + os.sep + 'archives' + os.sep + filename
    if not os.path.isfile(archivename):
        session = internetarchive.ArchiveSession()
        download = session.get_item(item).get_file(filename)
        download.download(archivename)

def unrar(item):
    patoolib.extract_archive(os.getcwd() + os.sep + 'archives' + os.sep + item,
                             outdir=os.getcwd() + os.sep + 'rawdata',
                             interactive=False)

def get_archives_and_unzip():
   if not os.path.exists(os.getcwd() + os.sep + 'archives'):
       os.makedirs(os.getcwd() + os.sep + 'archives')
   if os.path.exists(os.getcwd() + os.sep + 'rawdata'):
       shutil.rmtree(os.getcwd() + os.sep + 'rawdata')
   os.makedirs(os.getcwd() + os.sep + 'rawdata')
   for item in list_of_archives:
       get_archive(item[0], item[1])
       unrar(item[0])


if __name__ == "__main__":
    get_archives_and_unzip()<commit_msg>this should be enough ... if archive there then it should be unpacked ... dont overthink this<commit_after>import os
import internetarchive
import patoolib
import shutil

list_of_archives = [['Pauly Supp.rar', 'PaulySupp'],
                    ['Pauly-Wissowa_54-66.rar', 'PaulyWissowa5466'],
                    ['Pauly-Wissowa_41-53.rar', 'PaulyWissowa4153_201412'],
                    ['Pauly-Wissowa_31-40.rar', 'PaulyWissowa3140_201412'],
                    ['Pauly-Wissowa_21-30.rar', 'PaulyWissowa2130'],
                    ['Pauly-Wissowa_11-20.rar', 'PaulyWissowa1120'],
                    ['Pauly-Wissowa_1-10.rar', 'PaulyWissowa110']]

def get_archive(filename, item):
    session = internetarchive.ArchiveSession()
    download = session.get_item(item).get_file(filename)
    download.download(os.getcwd() + os.sep + 'archives' + os.sep + filename)

def unrar(item):
    patoolib.extract_archive(os.getcwd() + os.sep + 'archives' + os.sep + item,
                             outdir=os.getcwd() + os.sep + 'rawdata',
                             interactive=False)

def get_archives_and_unzip():
   if not os.path.exists(os.getcwd() + os.sep + 'archives'):
       os.makedirs(os.getcwd() + os.sep + 'archives')
   if not os.path.exists(os.getcwd() + os.sep + 'rawdata'):
       os.makedirs(os.getcwd() + os.sep + 'rawdata')
   for item in list_of_archives:
       if not os.path.isfile(os.getcwd() + os.sep + 'archives' + os.sep + item[0]):
           get_archive(item[0], item[1])
           unrar(item[0])


if __name__ == "__main__":
    get_archives_and_unzip()
{'fraction_non_alphanumeric': 0.1465213209361975, 'fraction_numerical': 0.039435716575825584, 'mean_word_length': 2.627906976744186, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4329562', 'n_tokens_mistral': 1213, 'n_tokens_neox': 1137, 'n_words': 179}