<commit_before>#!/usr/bin/python3
import urllib.request
import sys
import logging
import tempfile
import os
import re
import json
import argparse
import shutil
import stat

from zipfile import ZipFile

libindex = {
    'win32': {
        'x64': {
            'generic': [
                {
                    'name': 'clang',
                    'version': '10.0.0',
                    'url': 'https://www.dropbox.com/s/04oglvnci4x00fa/clang-10.0.0-x64-windows-release.zip?dl=1',
                    'defs': {
                        'CMAKE_C_COMPILER': '$$HOME$$/bin/clang-cl.exe',
                        'CMAKE_CXX_COMPILER': '$$HOME$$/bin/clang-cl.exe',
                    }
                }
            ],
            'debug': [
                {
                    'name': 'antlr4-runtime-cpp',
                    'version': '4.8',
                    'url': 'https://www.dropbox.com/s/fkfcvxxoml9mzaa/antlr4-4.8-x64-windows-debug.zip?dl=1'
                }
            ],
            'release': [
                {
                    'name': 'antlr4-runtime-cpp',
                    'version': '4.8',
                    'url': 'https://www.dropbox.com/s/c08fuwh63upe1ux/antlr4-4.8-x64-windows-release.zip?dl=1'
                }
            ]
        }
    },
    'linux': {
        'x64': {
            'generic': [
                {
                    'name': 'clang',
                    'version': '10.0.0',
                    'url': 'https://www.dropbox.com/s/fb7qtvh3t03e7zs/clang-10.0.0-x64-linux-release.zip?dl=1',
                    'defs': {
                        'CMAKE_C_COMPILER': '$$HOME$$/bin/clang',
                        'CMAKE_CXX_COMPILER': '$$HOME$$/bin/clang++',
                    },
                    'execs': [
                        '$$HOME$$/bin/clang',
                        '$$HOME$$/bin/clang++'
                    ]
                },
            ],
            'debug': [
                {
                    'name': 'antlr4-runtime-cpp',
                    'version': '4.8',
                    'url': 'https://www.dropbox.com/s/9vz3tmurvqhztqx/antlr4-4.8-x64-linux.zip?dl=1'
                }
            ],
            'release': [
                {
                    'name': 'antlr4-runtime-cpp',
                    'version': '4.8',
                    'url': 'https://www.dropbox.com/s/9vz3tmurvqhztqx/antlr4-4.8-x64-linux.zip?dl=1'
                }
            ]
        }
    }
}

SIZE_ONE_MB = 1048576


def nomalizeSlash(p: str):
    if sys.platform == 'win32':
        p = p.replace('\\', '/')
    return p

<commit_msg>Added llvm 1.0.0 debug and release packages for windows and linux<commit_after>#!/usr/bin/python3
import urllib.request
import sys
import logging
import tempfile
import os
import re
import json
import argparse
import shutil
import stat

from zipfile import ZipFile

libindex = {
    'win32': {
        'x64': {
            'generic': [
                {
                    'name': 'clang',
                    'version': '10.0.0',
                    'url': 'https://www.dropbox.com/s/04oglvnci4x00fa/clang-10.0.0-x64-windows-release.zip?dl=1',
                    'defs': {
                        'CMAKE_C_COMPILER': '$$HOME$$/bin/clang-cl.exe',
                        'CMAKE_CXX_COMPILER': '$$HOME$$/bin/clang-cl.exe',
                    }
                }
            ],
            'debug': [
                {
                    'name': 'antlr4-runtime-cpp',
                    'version': '4.8',
                    'url': 'https://www.dropbox.com/s/fkfcvxxoml9mzaa/antlr4-4.8-x64-windows-debug.zip?dl=1'
                },
                {
                    'name': 'llvm',
                    'version': '10.0.0',
                    'url': 'https://www.dropbox.com/s/oayfcbdp6neirp6/llvm-10.0.0-x64-windows-debug.zip?dl=1'
                }
            ],
            'release': [
                {
                    'name': 'antlr4-runtime-cpp',
                    'version': '4.8',
                    'url': 'https://www.dropbox.com/s/c08fuwh63upe1ux/antlr4-4.8-x64-windows-release.zip?dl=1'
                },
                {
                    'name': 'llvm',
                    'version': '10.0.0',
                    'url': 'https://www.dropbox.com/s/y27mhcrlppkaoqc/llvm-10.0.0-x64-windows-release.zip?dl=1'
                }
            ]
        }
    },
    'linux': {
        'x64': {
            'generic': [
                {
                    'name': 'clang',
                    'version': '10.0.0',
                    'url': 'https://www.dropbox.com/s/fb7qtvh3t03e7zs/clang-10.0.0-x64-linux-release.zip?dl=1',
                    'defs': {
                        'CMAKE_C_COMPILER': '$$HOME$$/bin/clang',
                        'CMAKE_CXX_COMPILER': '$$HOME$$/bin/clang++',
                    },
                    'execs': [
                        '$$HOME$$/bin/clang',
                        '$$HOME$$/bin/clang++'
                    ]
                },
            ],
            'debug': [
                {
                    'name': 'antlr4-runtime-cpp',
                    'version': '4.8',
                    'url': 'https://www.dropbox.com/s/9vz3tmurvqhztqx/antlr4-4.8-x64-linux.zip?dl=1'
                },
                {
                    'name': 'llvm',
                    'version': '10.0.0',
                    'url': 'https://www.dropbox.com/s/ay865217z3z3uw5/llvm-10.0.0-x64-linux-debug.zip?dl=1'
                }
            ],
            'release': [
                {
                    'name': 'antlr4-runtime-cpp',
                    'version': '4.8',
                    'url': 'https://www.dropbox.com/s/9vz3tmurvqhztqx/antlr4-4.8-x64-linux.zip?dl=1'
                },
                {
                    'name': 'llvm',
                    'version': '10.0.0',
                    'url': 'https://www.dropbox.com/s/ekv2dikbi4tn0us/llvm-10.0.0-x64-linux-release.zip?dl=1'
                }
            ]
        }
    }
}

SIZE_ONE_MB = 1048576


def nomalizeSlash(p: str):
    if sys.platform == 'win32':
        p = p.replace('\\', '/')
    return p


{'fraction_non_alphanumeric': 0.1584770582492678, 'fraction_numerical': 0.040839570452326716, 'mean_word_length': 1.0094802222948676, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 16, 'lorem ipsum': 0, 'www.': 16, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5232866', 'n_tokens_mistral': 2181, 'n_tokens_neox': 2001, 'n_words': 228}