<commit_before>command! -nargs=1 Utf8Code :call s:comUtf8Code("<args>")
let s:Utf8TextPrefix = '#utf8#'
let s:Utf8TextPrefixLen = strlen(s:Utf8TextPrefix)

function! Utf8TextPrefix()
	return s:Utf8TextPrefix
endfunction

function! Utf8Text(data)
	if s:testUtf8Code(a:data) > 0
		let l:txt = iconv(strpart(a:data,s:Utf8TextPrefixLen), 'utf-8', &enc)
	else
		let l:txt = a:data
	endif
	return l:txt
endfunction

function! Utf8Code(data)
	" save current encoding
	let l:curenc = &encoding
	" change input text into utf-8 encoding
	let l:u8data = iconv(a:data, l:curenc, 'utf-8')
	" open a temp buffer
	silent exe 'enew'
	silent exe 'set encoding=utf-8'
	call setline(1, l:u8data)
	silent exe '%!xxd -ps'
	let l:hexdata = getline(1)
	let l:hexlen  = strlen(l:hexdata)
	" strip line break
	if &fileformat == 'dos'
		call setline(1, strpart(l:hexdata,0,l:hexlen-4))
	else
		call setline(1, strpart(l:hexdata,0,l:hexlen-2))
	endif
	" add '\x' in front of each byte
	silent exe 's/.\{2}/\\x\0/g'
	" copy result into register u
	silent exe 'normal v$h"uy'
	let l:hexdata = getline(1)
	" delete temp buffer
	silent exe 'bd!'
	" restore encoding
	silent exe 'set encoding=' . l:curenc
	return l:hexdata
endfunction

function! ListAndSelect(title, itemlist, markindex)
	let l:choices = copy(a:itemlist)
	" generate choice-list
	call map(l:choices, '"  " . (v:key + 1) . ". " . v:val')
	" insert '*' at the start of selected item
	if (a:markindex >= 0) && (a:markindex < len(a:itemlist))<commit_msg>Update:vim setting(Fix: Convert long text to Utf8Code)<commit_after>command! -nargs=1 Utf8Code :call s:comUtf8Code("<args>")
let s:Utf8TextPrefix = '#utf8#'
let s:Utf8TextPrefixLen = strlen(s:Utf8TextPrefix)

function! Utf8TextPrefix()
	return s:Utf8TextPrefix
endfunction

function! Utf8Text(data)
	if s:testUtf8Code(a:data) > 0
		let l:txt = iconv(strpart(a:data,s:Utf8TextPrefixLen), 'utf-8', &enc)
	else
		let l:txt = a:data
	endif
	return l:txt
endfunction

function! Utf8Code(data)
	" save current encoding
	let l:curenc = &encoding
	" change input text into utf-8 encoding
	let l:u8data = iconv(a:data, l:curenc, 'utf-8')
	" open a temp buffer
	silent exe 'enew'
	silent exe 'set encoding=utf-8'
	call setline(1, l:u8data)
	silent exe '%!xxd -ps'
	silent exe 'normal gg' . repeat('Jx', line('$'))
	let l:hexdata = getline(1)
	let l:hexlen  = strlen(l:hexdata)
	" strip line break
	if &fileformat == 'dos'
		call setline(1, strpart(l:hexdata,0,l:hexlen-4))
	else
		call setline(1, strpart(l:hexdata,0,l:hexlen-2))
	endif
	" add '\x' in front of each byte
	silent exe 's/.\{2}/\\x\0/g'
	" copy result into register u
	silent exe 'normal v$h"uy'
	let l:hexdata = getline(1)
	" delete temp buffer
	silent exe 'bd!'
	" restore encoding
	silent exe 'set encoding=' . l:curenc
	return l:hexdata
endfunction

function! ListAndSelect(title, itemlist, markindex)
	let l:choices = copy(a:itemlist)
	" generate choice-list
	call map(l:choices, '"  " . (v:key + 1) . ". " . v:val')
	" insert '*' at the start of selected item
	if (a:markindex >= 0) && (a:markindex < len(a:itemlist))
{'fraction_non_alphanumeric': 0.14397649363369247, 'fraction_numerical': 0.02187397975840679, 'mean_word_length': 4.695167286245353, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3816332', 'n_tokens_mistral': 1294, 'n_tokens_neox': 1273, 'n_words': 321}