<commit_before>global encode
global decode

default rel

%ifidn __OUTPUT_FORMAT__, win64 ; Windows calling convention
	%define outputarray rcx
	%define inputarray rdx
	%define inputsize r8
%elifidn __OUTPUT_FORMAT__, elf64 ; Linux calling convention
	%define outputarray rdi
	%define inputarray rsi
	%define inputsize rdx
%endif

section .text
align 16
encode: ; encode(outputarray, inputarray, inputsize) returns the size of the output
	push r12 ; calling convention demands saving various registers
	push r13
	push r14
	push r15
	push rdi
	push rbx
	push rbp
	push rsi

	mov r9, outputarray ; Memory address of outputarray, will use this to get the size of the output later
	mov r11, 4 ; The maximum length of each line is 128 characters, 4 iterations will result in adequate results. 4*32=128

	%ifidn __OUTPUT_FORMAT__, win64 ; Windows calling convention
		mov rdi, outputarray ; Need to keep the address to the output memory location in RDI because of maskmovdqu
		%define outputarray rdi
	%endif
	

align 16
.encodeset:
	vmovaps ymm0, [inputarray] ; Read 16 bytes from memory
	vpaddb ymm0, ymm0, [const1] ; +42 as per yEnc spec

	vpcmpeqb ymm1, ymm0, [specialNull] ; 0x00
	vpcmpeqb ymm2, ymm0, [specialEqual] ; 0x3D
	vpcmpeqb ymm3, ymm0, [specialLF] ; 0x0A
	vpcmpeqb ymm4, ymm0, [specialCR] ; 0x0D
	vpor ymm1, ymm2 ; Merge compare results
	vpor ymm1, ymm3
	vpor ymm1, ymm4

	vptest ymm1, ymm1
	jnz .encodeillegalcharacters

	vmovdqu [outputarray], ymm0 ; Write to memory, no additional processing needed
	add outputarray, 32
	add inputarray, 32
	sub inputsize, 32
	jbe .done ; Yay! It's done!
	sub r11, 1
	jz .newline
	jmp .encodeset

align 16
.newline:
	mov word [outputarray], 0x0A0D ; \r\n
	add outputarray, 2 ; increase output array pointer
	mov r11, 4 ; Reset counter
	jmp .encodeset

align 16
.encodeillegalcharacters:
	pxor xmm10, xmm10
	pxor xmm11, xmm11
	pxor xmm12, xmm12
	
	vpand ymm9, ymm1, [specialdecode4]
	vpaddb ymm0, ymm0, ymm9

	vmovaps ymm7, [specialNull]
	vmovaps ymm5, [specialEqual] ; Escape character
	vpunpcklbw ymm5, ymm5, ymm0 ; Unpack lower half of data
	vpunpcklbw ymm7, ymm7, ymm1 ; Unpack lower half of mask
	vpsrldq ymm7, ymm7 ,1 ; Shift data register 1 byte to the right to align with ymm6
	vpor ymm6, ymm7, [writebytes] ; Add masks together

	;--------------------------- first split<commit_msg>Stuff?<commit_after>global encode
global decode

default rel

%ifidn __OUTPUT_FORMAT__, win64 ; Windows calling convention
	%define outputarray rcx
	%define inputarray rdx
	%define inputsize r8
%elifidn __OUTPUT_FORMAT__, elf64 ; Linux calling convention
	%define outputarray rdi
	%define inputarray rsi
	%define inputsize rdx
%endif

section .text
align 16
encode: ; encode(outputarray, inputarray, inputsize) returns the size of the output
	push r12 ; calling convention demands saving various registers
	push r13
	push r14
	push r15
	push rdi
	push rbx
	push rbp
	push rsi

	mov r9, outputarray ; Memory address of outputarray, will use this to get the size of the output later
	mov r11, 4 ; The maximum length of each line is 128 characters, 4 iterations will result in adequate results. 4*32=128

	%ifidn __OUTPUT_FORMAT__, win64 ; Windows calling convention
		mov rdi, outputarray ; Need to keep the address to the output memory location in RDI because of maskmovdqu
		%define outputarray rdi
	%endif
	

align 16
.encodeset:
	vmovaps ymm0, [inputarray] ; Read 32 bytes from memory
	vpaddb ymm0, ymm0, [const1] ; +42 as per yEnc spec

	vpcmpeqb ymm1, ymm0, [specialNull] ; 0x00
	vpcmpeqb ymm2, ymm0, [specialEqual] ; 0x3D
	vpcmpeqb ymm3, ymm0, [specialLF] ; 0x0A
	vpcmpeqb ymm4, ymm0, [specialCR] ; 0x0D
	vpor ymm1, ymm2 ; Merge compare results
	vpor ymm1, ymm3
	vpor ymm1, ymm4

	vptest ymm1, ymm1
	jnz .encodeillegalcharacters

	vmovdqu [outputarray], ymm0 ; Write to memory, no additional processing needed
	add outputarray, 32
	add inputarray, 32
	sub inputsize, 32
	jbe .done ; Yay! It's done!
	sub r11, 1
	jz .newline
	jmp .encodeset

align 16
.newline:
	mov word [outputarray], 0x0A0D ; \r\n
	add outputarray, 2 ; increase output array pointer
	mov r11, 4 ; Reset counter
	jmp .encodeset

align 16
.encodeillegalcharacters:
	pxor xmm10, xmm10
	pxor xmm11, xmm11
	pxor xmm12, xmm12
	
	vpand ymm9, ymm1, [specialdecode4]
	vpaddb ymm0, ymm0, ymm9

	vmovaps ymm7, [specialNull]
	vmovaps ymm5, [specialEqual] ; Escape character
	vpunpcklbw ymm5, ymm5, ymm0 ; Unpack lower half of data
	vpunpcklbw ymm7, ymm7, ymm1 ; Unpack lower half of mask
	vpsrldq ymm7, ymm7 ,1 ; Shift data register 1 byte to the right to align with [writebytes]
	vpor ymm6, ymm7, [writebytes] ; Add masks together

	;--------------------------- first split
{'fraction_non_alphanumeric': 0.07217819661597773, 'fraction_numerical': 0.051188691368601415, 'mean_word_length': 4.092693565976009, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5847138', 'n_tokens_mistral': 1852, 'n_tokens_neox': 1711, 'n_words': 639}