<commit_before>#!/usr/bin/python
import confluent.messages as msg
import eventlet
import eventlet.green.select as select
import eventlet.green.subprocess as subprocess
import glob
import logging
logging.getLogger('libarchive').addHandler(logging.NullHandler())
import libarchive
import hashlib
import os
import shutil
import sys
import time
import yaml

COPY = 1
EXTRACT = 2
READFILES = set([
    'README.diskdefines',
    'media.1/products',
    'media.2/products',
    '.DISCINFO',
    '.discinfo',
])

HEADERSUMS = set([b'\x85\xeddW\x86\xc5\xbdhx\xbe\x81\x18X\x1e\xb4O\x14\x9d\x11\xb7C8\x9b\x97R\x0c-\xb8Ht\xcb\xb3'])
HASHPRINTS = {
    '69d5f1c5e4474d70b0fb5374bfcb29bf57ba828ff00a55237cd757e61ed71048': {'name': 'cumulus-broadcom-amd64-4.0.0', 'method': COPY},
}

from ctypes import byref, c_longlong, c_size_t, c_void_p

from libarchive.ffi import (
    write_disk_new, write_disk_set_options, write_free, write_header,
    read_data_block, write_data_block, write_finish_entry, ARCHIVE_EOF
)

def relax_umask():
    os.umask(0o22)

def update_boot(profiledir):
    profile = {}
    if profiledir.endswith('/'):
        profiledir = profiledir[:-1]
    profname = os.path.basename(profiledir)
    with open('{0}/profile.yaml'.format(profiledir)) as profileinfo:
        profile = yaml.safe_load(profileinfo)
    label = profile.get('label', profname)
    ostype = profile.get('ostype', 'linux')
    if ostype == 'linux':
        update_boot_linux(profiledir, profile, label)
    elif ostype == 'esxi':
        update_boot_esxi(profiledir, profile, label)

def update_boot_esxi(profiledir, profile, label):
    kernelargs = profile.get('kernelargs', '')
    bootcfg = open('{0}/distribution/BOOT.CFG'.format(profiledir), 'r').read()
    bootcfg = bootcfg.split('\n')
    newbootcfg = ''
    filesneeded = []
    for cfgline in bootcfg:
        if cfgline.startswith('title='):
            newbootcfg += 'title={0}\n'.format(label)
        elif cfgline.startswith('kernelopt='):
            newbootcfg += 'kernelopt={0}\n'.format(kernelargs)
        elif cfgline.startswith('kernel='):
            newbootcfg += cfgline + '\n'
            kern = cfgline.split('=', 1)[1]
            filesneeded.append(kern)
        elif cfgline.startswith('modules='):
            modlist = cfgline.split('=', 1)[1]
            mods = modlist.split(' --- ')
            mods = [x.replace('/', '') for x in mods]
            filesneeded.extend(mods)
            newbootcfg += cfgline + ' --- initramfs/addons.tgz --- site.tgz\n'
        else:
            newbootcfg += cfgline + '\n'
    os.makedirs('{0}/boot/efi/boot/'.format(profiledir))
    with open('{0}/boot/efi/boot/boot.cfg'.format(profiledir), 'w+') as bcfg:
        bcfg.write(newbootcfg)
    os.symlink('/var/lib/confluent/public/site/initramfs.tgz',
               '{0}/boot/site.tgz'.format(profiledir))
    os.symlink('{0}/boot/efi/boot/boot.cfg'.format(profiledir), '{0}/boot/boot.cfg'.format(profiledir))
    for fn in filesneeded:
        if fn.startswith('/'):
            fn = fn[1:]
        sourcefile = '{0}/distribution/{1}'.format(profiledir, fn)
        if not os.path.exists(sourcefile):
            sourcefile = '{0}/distribution/{1}'.format(profiledir, fn.upper())
        os.symlink(sourcefile, '{0}/boot/{1}'.format(profiledir, fn))
    os.symlink('{0}/distribution/EFI/BOOT/BOOTX64.EFI'.format(profiledir), '{0}/boot/efi/boot/bootx64.efi'.format(profiledir))
    oum = os.umask(0o22)
    ipout = os.open(profiledir + '/boot.ipxe', os.O_WRONLY|os.O_CREAT, 0o644)
    ipxeout = os.fdopen(ipout, 'w')
    try:
        os.umask(oum)
        ipxeout.write('#!ipxe\n')
        pname = os.path.split(profiledir)[-1]
        ipxeout.write(
            'chain boot/efi/boot/bootx64.efi -c /confluent-public/os/{0}/boot/boot.cfg'.format(pname))
    finally:
        ipxeout.close()
    subprocess.check_call(
        ['/opt/confluent/bin/dir2img', '{0}/boot'.format(profiledir),<commit_msg>Write more useful boot.cfg<commit_after>#!/usr/bin/python
import confluent.messages as msg
import eventlet
import eventlet.green.select as select
import eventlet.green.subprocess as subprocess
import glob
import logging
logging.getLogger('libarchive').addHandler(logging.NullHandler())
import libarchive
import hashlib
import os
import shutil
import sys
import time
import yaml

COPY = 1
EXTRACT = 2
READFILES = set([
    'README.diskdefines',
    'media.1/products',
    'media.2/products',
    '.DISCINFO',
    '.discinfo',
])

HEADERSUMS = set([b'\x85\xeddW\x86\xc5\xbdhx\xbe\x81\x18X\x1e\xb4O\x14\x9d\x11\xb7C8\x9b\x97R\x0c-\xb8Ht\xcb\xb3'])
HASHPRINTS = {
    '69d5f1c5e4474d70b0fb5374bfcb29bf57ba828ff00a55237cd757e61ed71048': {'name': 'cumulus-broadcom-amd64-4.0.0', 'method': COPY},
}

from ctypes import byref, c_longlong, c_size_t, c_void_p

from libarchive.ffi import (
    write_disk_new, write_disk_set_options, write_free, write_header,
    read_data_block, write_data_block, write_finish_entry, ARCHIVE_EOF
)

def relax_umask():
    os.umask(0o22)

def update_boot(profiledir):
    profile = {}
    if profiledir.endswith('/'):
        profiledir = profiledir[:-1]
    profname = os.path.basename(profiledir)
    with open('{0}/profile.yaml'.format(profiledir)) as profileinfo:
        profile = yaml.safe_load(profileinfo)
    label = profile.get('label', profname)
    ostype = profile.get('ostype', 'linux')
    if ostype == 'linux':
        update_boot_linux(profiledir, profile, label)
    elif ostype == 'esxi':
        update_boot_esxi(profiledir, profile, label)

def update_boot_esxi(profiledir, profile, label):
    kernelargs = profile.get('kernelargs', '')
    oum = os.umask(0o22)
    bootcfg = open('{0}/distribution/BOOT.CFG'.format(profiledir), 'r').read()
    bootcfg = bootcfg.split('\n')
    newbootcfg = ''
    filesneeded = []
    for cfgline in bootcfg:
        if cfgline.startswith('title='):
            newbootcfg += 'title={0}\n'.format(label)
        elif cfgline.startswith('kernelopt='):
            newbootcfg += 'kernelopt={0}\n'.format(kernelargs)
        elif cfgline.startswith('kernel='):
            newbootcfg += cfgline + '\n'
            kern = cfgline.split('=', 1)[1]
            filesneeded.append(kern)
        elif cfgline.startswith('modules='):
            modlist = cfgline.split('=', 1)[1]
            mods = modlist.split(' --- ')
            mods = [x.replace('/', '') for x in mods]
            filesneeded.extend(mods)

            newbootcfg += 'modules= + ' --- '.join(mods) + ' --- initramfs/addons.tgz --- site.tgz\n'
        else:
            newbootcfg += cfgline + '\n'
    os.makedirs('{0}/boot/efi/boot/'.format(profiledir))
    with open('{0}/boot/efi/boot/boot.cfg'.format(profiledir), 'w+') as bcfg:
        bcfg.write(newbootcfg)
    os.symlink('/var/lib/confluent/public/site/initramfs.tgz',
               '{0}/boot/site.tgz'.format(profiledir))
    os.symlink('{0}/boot/efi/boot/boot.cfg'.format(profiledir), '{0}/boot/boot.cfg'.format(profiledir))
    for fn in filesneeded:
        if fn.startswith('/'):
            fn = fn[1:]
        sourcefile = '{0}/distribution/{1}'.format(profiledir, fn)
        if not os.path.exists(sourcefile):
            sourcefile = '{0}/distribution/{1}'.format(profiledir, fn.upper())
        os.symlink(sourcefile, '{0}/boot/{1}'.format(profiledir, fn))
    os.symlink('{0}/distribution/EFI/BOOT/BOOTX64.EFI'.format(profiledir), '{0}/boot/efi/boot/bootx64.efi'.format(profiledir))
    ipout = os.open(profiledir + '/boot.ipxe', os.O_WRONLY|os.O_CREAT, 0o644)
    ipxeout = os.fdopen(ipout, 'w')
    try:
        os.umask(oum)
        ipxeout.write('#!ipxe\n')
        pname = os.path.split(profiledir)[-1]
        ipxeout.write(
            'chain boot/efi/boot/bootx64.efi -c /confluent-public/os/{0}/boot/boot.cfg'.format(pname))
    finally:
        ipxeout.close()
    subprocess.check_call(
        ['/opt/confluent/bin/dir2img', '{0}/boot'.format(profiledir),
{'fraction_non_alphanumeric': 0.16109920584898524, 'fraction_numerical': 0.029244926257405775, 'mean_word_length': 3.1151452282157677, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1868888', 'n_tokens_mistral': 3051, 'n_tokens_neox': 2899, 'n_words': 445}