<commit_before>import urllib.request
import os
import tarfile
import tempfile
import pypandoc

def fetch_and_convert_tex(id):
    with tempfile.TemporaryDirectory() as workdir:
        # download an archive from arXiv
        archive_path = os.path.join(workdir, 'archive.tar.gz')
        urllib.request.urlretrieve("https://arxiv.org/e-print/{}".format(id), archive_path)

        # extract the archive
        tar = tarfile.open(archive_path)
        tar.extractall(workdir)
        tar.close()

        # DEBUG
        print('List of contents', os.listdir(workdir))

        # search for a TeX source
        tex_files = [x for x in os.listdir(workdir) if x.endswith('.tex')]
        print('TeX', tex_files)
        if len(tex_files) == 0:
            return False
        tex_filepath = os.path.join(workdir, tex_files[0])
        print(tex_filepath)

        # convert a TeX source to HTML
        pandoc_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), './pandoc')
        os.chdir(workdir)
        extra_args = [
            '--self-contained',
            '--data-dir', pandoc_dir
        ]
        output = pypandoc.convert_file(tex_filepath, 'html5', extra_args=extra_args)

        return output<commit_msg>chore: add debug command<commit_after>import urllib.request
import os
import tarfile
import tempfile
import pypandoc

def fetch_and_convert_tex(id):
    with tempfile.TemporaryDirectory() as workdir:
        # download an archive from arXiv
        archive_path = os.path.join(workdir, 'archive.tar.gz')
        urllib.request.urlretrieve("https://arxiv.org/e-print/{}".format(id), archive_path)

        # extract the archive
        tar = tarfile.open(archive_path)
        tar.extractall(workdir)
        tar.close()

        # DEBUG
        print('List of contents', os.listdir(workdir))

        # search for a TeX source
        tex_files = [x for x in os.listdir(workdir) if x.endswith('.tex')]
        print('TeX', tex_files)
        if len(tex_files) == 0:
            return False
        tex_filepath = os.path.join(workdir, tex_files[0])
        print(tex_filepath)

        # convert a TeX source to HTML
        pandoc_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), './pandoc')
        os.chdir(workdir)
        extra_args = [
            '--self-contained',
            '--data-dir', pandoc_dir
        ]

        print(pandoc_dir)
        print(os.listdir(pandoc_dir))

        output = pypandoc.convert_file(tex_filepath, 'html5', extra_args=extra_args)

        return output
{'fraction_non_alphanumeric': 0.11325206449075895, 'fraction_numerical': 0.0023594180102241447, 'mean_word_length': 2.4331983805668016, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6237836', 'n_tokens_mistral': 833, 'n_tokens_neox': 806, 'n_words': 168}