<commit_before>// #define NO_ARRAY_BOUNDS_CHECK

#include "common.cpp"
#include "timings.cpp"
#include "tokenizer.cpp"
#include "big_int.cpp"
#include "exact_value.cpp"
#include "build_settings.cpp"


gb_global Timings global_timings = {0};

#if defined(LLVM_BACKEND_SUPPORT)
#include "llvm-c/Types.h"
#endif

#include "parser.hpp"
#include "checker.hpp"

#include "parser.cpp"
#include "checker.cpp"
#include "docs.cpp"


#if defined(LLVM_BACKEND_SUPPORT)
#include "llvm_backend.cpp"
#endif

#include "ir.cpp"
#include "ir_opt.cpp"
#include "ir_print.cpp"
#include "query_data.cpp"


#if defined(GB_SYSTEM_WINDOWS)
// NOTE(IC): In order to find Visual C++ paths without relying on environment variables.
#include "microsoft_craziness.h"
#endif


// NOTE(bill): 'name' is used in debugging and profiling modes
i32 system_exec_command_line_app(char const *name, char const *fmt, ...) {
#if defined(GB_SYSTEM_WINDOWS)
	STARTUPINFOW start_info = {gb_size_of(STARTUPINFOW)};
	PROCESS_INFORMATION pi = {0};
	isize cmd_len = 0;
	isize const cmd_cap = 4096;
	char cmd_line[cmd_cap] = {};
	va_list va;
	gbTempArenaMemory tmp;
	String16 cmd;
	i32 exit_code = 0;


	start_info.dwFlags = STARTF_USESTDHANDLES | STARTF_USESHOWWINDOW;
	start_info.wShowWindow = SW_SHOW;
	start_info.hStdInput   = GetStdHandle(STD_INPUT_HANDLE);
	start_info.hStdOutput  = GetStdHandle(STD_OUTPUT_HANDLE);
	start_info.hStdError   = GetStdHandle(STD_ERROR_HANDLE);

	va_start(va, fmt);
	cmd_len = gb_snprintf_va(cmd_line, cmd_cap-1, fmt, va);
	va_end(va);

	if (build_context.show_system_calls) {
		gb_printf_err("[SYSTEM CALL] %s\n", name);
		gb_printf_err("%.*s\n\n", cast(int)(cmd_len-1), cmd_line);
	}

	tmp = gb_temp_arena_memory_begin(&string_buffer_arena);
	defer (gb_temp_arena_memory_end(tmp));

	cmd = string_to_string16(string_buffer_allocator, make_string(cast(u8 *)cmd_line, cmd_len-1));
	if (CreateProcessW(nullptr, cmd.text,
	                   nullptr, nullptr, true, 0, nullptr, nullptr,
	                   &start_info, &pi)) {
		WaitForSingleObject(pi.hProcess, INFINITE);
		GetExitCodeProcess(pi.hProcess, cast(DWORD *)&exit_code);

		CloseHandle(pi.hProcess);
		CloseHandle(pi.hThread);
	} else {
		// NOTE(bill): failed to create process
		gb_printf_err("Failed to execute command:\n\t%s\n", cmd_line);
		exit_code = -1;
	}

	return exit_code;

#elif defined(GB_SYSTEM_OSX) || defined(GB_SYSTEM_UNIX)

	char cmd_line[4096] = {0};
	isize cmd_len;
	va_list va;
	String cmd;
	i32 exit_code = 0;

	va_start(va, fmt);
	cmd_len = gb_snprintf_va(cmd_line, gb_size_of(cmd_line), fmt, va);
	va_end(va);
	cmd = make_string(cast(u8 *)&cmd_line, cmd_len-1);

	if (build_context.show_system_calls) {
		gb_printf_err("[SYSTEM CALL] %s\n", name);
		gb_printf_err("%s\n\n", cmd_line);
	}
	exit_code = system(cmd_line);

	// pid_t pid = fork();
	// int status = 0;

	// if(pid == 0) {
	// 	// in child, pid == 0.
	// 	int ret = execvp(cmd.text, (char* const*) cmd.text);

	// 	if(ret == -1) {
	// 		gb_printf_err("Failed to execute command:\n\t%s\n", cmd_line);

	// 		// we're in the child, so returning won't do us any good -- just quit.
	// 		exit(-1);
	// 	}

	// 	// unreachable
	// 	abort();
	// } else {
	// 	// wait for child to finish, then we can continue cleanup

	// 	int s = 0;
	// 	waitpid(pid, &s, 0);

	// 	status = WEXITSTATUS(s);
	// }

	// exit_code = status;

	return exit_code;
#endif
}




#if defined(LLVM_BACKEND_SUPPORT)
i32 linker_stage(lbGenerator *gen) {
	i32 exit_code = 0;

	Timings *timings = &global_timings;

	String output_base = gen->output_base;

	if (build_context.metrics.os == TargetOs_js) {
		timings_start_section(timings, str_lit("wasm-ld"));
		exit_code = system_exec_command_line_app("wasm-ld",
			"\"%.*s\\bin\\wasm-ld\" \"%.*s.wasm-obj\" -o \"%.*s.wasm\" %.*s %.*s",
			LIT(build_context.ODIN_ROOT),
			LIT(output_base), LIT(output_base), LIT(build_context.link_flags), LIT(build_context.extra_linker_flags));
		if (exit_code != 0) {
			return exit_code;
		}
		return exit_code;
	}

	if (build_context.cross_compiling && selected_target_metrics->metrics == &target_essence_amd64) {
#ifdef GB_SYSTEM_UNIX
		system_exec_command_line_app("linker", "x86_64-essence-gcc \"%.*s.o\" -o \"%.*s\" %.*s %.*s",
				LIT(output_base), LIT(output_base), LIT(build_context.link_flags), LIT(build_context.extra_linker_flags));
#else
		gb_printf_err("Linking for cross compilation for this platform is not yet supported (%.*s %.*s)\n",
			LIT(target_os_names[build_context.metrics.os]),
			LIT(target_arch_names[build_context.metrics.arch])
		);
#endif
	} else if (build_context.cross_compiling && build_context.different_os) {
		gb_printf_err("Linking for cross compilation for this platform is not yet supported (%.*s %.*s)\n",
			LIT(target_os_names[build_context.metrics.os]),
			LIT(target_arch_names[build_context.metrics.arch])
		);
		build_context.keep_object_files = true;
	} else {
	#if defined(GB_SYSTEM_WINDOWS)
		timings_start_section(timings, str_lit("msvc-link"));

		gbString lib_str = gb_string_make(heap_allocator(), "");
		defer (gb_string_free(lib_str));
		char lib_str_buf[1024] = {0};

		char const *output_ext = "exe";
		gbString link_settings = gb_string_make_reserve(heap_allocator(), 256);
		defer (gb_string_free(link_settings));


		// NOTE(ic): It would be nice to extend this so that we could specify the Visual Studio version that we want instead of defaulting to the latest.
		Find_Result_Utf8 find_result = find_visual_studio_and_windows_sdk_utf8();

		if (find_result.windows_sdk_version == 0) {
			gb_printf_err("Windows SDK not found.\n");
			return 1;
		}

		if (build_context.ignore_microsoft_magic) {
			find_result = {};
		}

		// Add library search paths.
		if (find_result.vs_library_path.len > 0) {
			GB_ASSERT(find_result.windows_sdk_um_library_path.len > 0);
			GB_ASSERT(find_result.windows_sdk_ucrt_library_path.len > 0);

			String path = {};
			auto add_path = [&](String path) {
				if (path[path.len-1] == '\\') {
					path.len -= 1;
				}
				link_settings = gb_string_append_fmt(link_settings, " /LIBPATH:\"%.*s\"", LIT(path));
			};
			add_path(find_result.windows_sdk_um_library_path);
			add_path(find_result.windows_sdk_ucrt_library_path);
			add_path(find_result.vs_library_path);
		}

		for_array(i, gen->module.foreign_library_paths) {
			String lib = gen->module.foreign_library_paths[i];
			GB_ASSERT(lib.len < gb_count_of(lib_str_buf)-1);
			isize len = gb_snprintf(lib_str_buf, gb_size_of(lib_str_buf),
			                        " \"%.*s\"", LIT(lib));
			lib_str = gb_string_appendc(lib_str, lib_str_buf);
		}



		if (build_context.build_mode == BuildMode_DynamicLibrary) {
			output_ext = "dll";
			link_settings = gb_string_append_fmt(link_settings, " /DLL");
		} else {
			link_settings = gb_string_append_fmt(link_settings, " /ENTRY:mainCRTStartup");
		}

		if (build_context.pdb_filepath != "") {
			link_settings = gb_string_append_fmt(link_settings, " /PDB:%.*s", LIT(build_context.pdb_filepath));
		}

		if (build_context.no_crt) {
			link_settings = gb_string_append_fmt(link_settings, " /nodefaultlib");
		} else {
			link_settings = gb_string_append_fmt(link_settings, " /defaultlib:libcmt");
		}

		if (build_context.ODIN_DEBUG) {
			link_settings = gb_string_append_fmt(link_settings, " /DEBUG");
		}

		gbString object_files = gb_string_make(heap_allocator(), "");
		defer (gb_string_free(object_files));
		for_array(i, gen->output_object_paths) {
			String object_path = gen->output_object_paths[i];
			object_files = gb_string_append_fmt(object_files, "\"%.*s\" ", LIT(object_path));
		}

		char const *subsystem_str = build_context.use_subsystem_windows ? "WINDOWS" : "CONSOLE";
		if (!build_context.use_lld) { // msvc
			if (build_context.has_resource) {
				exit_code = system_exec_command_line_app("msvc-link",
					"\"rc.exe\" /nologo /fo \"%.*s.res\" \"%.*s.rc\"",
					LIT(output_base),
					LIT(build_context.resource_filepath)
				);

	            if (exit_code != 0) {
					return exit_code;
				}

				exit_code = system_exec_command_line_app("msvc-link",
					"\"%.*slink.exe\" %s \"%.*s.res\" -OUT:\"%.*s.%s\" %s "
					"/nologo /incremental:no /opt:ref /subsystem:%s "
					" %.*s "
					" %.*s "
					" %s "
					"",
					LIT(find_result.vs_exe_path), object_files, LIT(output_base), LIT(output_base), output_ext,
					link_settings,
					subsystem_str,
					LIT(build_context.link_flags),
					LIT(build_context.extra_linker_flags),
					lib_str
				);
			} else {
				exit_code = system_exec_command_line_app("msvc-link",
					"\"%.*slink.exe\" %s -OUT:\"%.*s.%s\" %s "
					"/nologo /incremental:no /opt:ref /subsystem:%s "
					" %.*s "
					" %.*s "
					" %s "
					"",
					LIT(find_result.vs_exe_path), object_files, LIT(output_base), output_ext,
					link_settings,
					subsystem_str,
					LIT(build_context.link_flags),
					LIT(build_context.extra_linker_flags),
					lib_str
				);
			}
		} else { // lld
			exit_code = system_exec_command_line_app("msvc-link",
				"\"%.*s\\bin\\lld-link\" %s -OUT:\"%.*s.%s\" %s "
				"/nologo /incremental:no /opt:ref /subsystem:%s "
				" %.*s "
				" %.*s "
				" %s "
				"",
				LIT(build_context.ODIN_ROOT),
				LIT(output_base), object_files, output_ext,
				link_settings,
				subsystem_str,
				LIT(build_context.link_flags),
				LIT(build_context.extra_linker_flags),
				lib_str
			);
		}
	#else
		timings_start_section(timings, str_lit("ld-link"));

		// NOTE(vassvik): get cwd, for used for local shared libs linking, since those have to be relative to the exe
		char cwd[256];
		getcwd(&cwd[0], 256);
		//printf("%s\n", cwd);

		// NOTE(vassvik): needs to add the root to the library search paths, so that the full filenames of the library
		//                files can be passed with -l:
		gbString lib_str = gb_string_make(heap_allocator(), "-L/");
		defer (gb_string_free(lib_str));

		for_array(i, gen->module.foreign_library_paths) {
			String lib = gen->module.foreign_library_paths[i];

			// NOTE(zangent): Sometimes, you have to use -framework on MacOS.
			//   This allows you to specify '-f' in a #foreign_system_library,
			//   without having to implement any new syntax specifically for MacOS.
			#if defined(GB_SYSTEM_OSX)
				if (string_ends_with(lib, str_lit(".framework"))) {
					// framework thingie<commit_msg>Fix up `system_exec_command_line_app` exit code code<commit_after>// #define NO_ARRAY_BOUNDS_CHECK

#include "common.cpp"
#include "timings.cpp"
#include "tokenizer.cpp"
#include "big_int.cpp"
#include "exact_value.cpp"
#include "build_settings.cpp"


gb_global Timings global_timings = {0};

#if defined(LLVM_BACKEND_SUPPORT)
#include "llvm-c/Types.h"
#endif

#include "parser.hpp"
#include "checker.hpp"

#include "parser.cpp"
#include "checker.cpp"
#include "docs.cpp"


#if defined(LLVM_BACKEND_SUPPORT)
#include "llvm_backend.cpp"
#endif

#include "ir.cpp"
#include "ir_opt.cpp"
#include "ir_print.cpp"
#include "query_data.cpp"


#if defined(GB_SYSTEM_WINDOWS)
// NOTE(IC): In order to find Visual C++ paths without relying on environment variables.
#include "microsoft_craziness.h"
#endif


// NOTE(bill): 'name' is used in debugging and profiling modes
i32 system_exec_command_line_app(char const *name, char const *fmt, ...) {
#if defined(GB_SYSTEM_WINDOWS)
	STARTUPINFOW start_info = {gb_size_of(STARTUPINFOW)};
	PROCESS_INFORMATION pi = {0};
	isize cmd_len = 0;
	isize const cmd_cap = 4096;
	char cmd_line[cmd_cap] = {};
	va_list va;
	gbTempArenaMemory tmp;
	String16 cmd;
	i32 exit_code = 0;


	start_info.dwFlags = STARTF_USESTDHANDLES | STARTF_USESHOWWINDOW;
	start_info.wShowWindow = SW_SHOW;
	start_info.hStdInput   = GetStdHandle(STD_INPUT_HANDLE);
	start_info.hStdOutput  = GetStdHandle(STD_OUTPUT_HANDLE);
	start_info.hStdError   = GetStdHandle(STD_ERROR_HANDLE);

	va_start(va, fmt);
	cmd_len = gb_snprintf_va(cmd_line, cmd_cap-1, fmt, va);
	va_end(va);

	if (build_context.show_system_calls) {
		gb_printf_err("[SYSTEM CALL] %s\n", name);
		gb_printf_err("%.*s\n\n", cast(int)(cmd_len-1), cmd_line);
	}

	tmp = gb_temp_arena_memory_begin(&string_buffer_arena);
	defer (gb_temp_arena_memory_end(tmp));

	cmd = string_to_string16(string_buffer_allocator, make_string(cast(u8 *)cmd_line, cmd_len-1));
	if (CreateProcessW(nullptr, cmd.text,
	                   nullptr, nullptr, true, 0, nullptr, nullptr,
	                   &start_info, &pi)) {
		WaitForSingleObject(pi.hProcess, INFINITE);
		GetExitCodeProcess(pi.hProcess, cast(DWORD *)&exit_code);

		CloseHandle(pi.hProcess);
		CloseHandle(pi.hThread);
	} else {
		// NOTE(bill): failed to create process
		gb_printf_err("Failed to execute command:\n\t%s\n", cmd_line);
		exit_code = -1;
	}

	return exit_code;

#elif defined(GB_SYSTEM_OSX) || defined(GB_SYSTEM_UNIX)

	char cmd_line[4096] = {0};
	isize cmd_len;
	va_list va;
	String cmd;
	i32 exit_code = 0;

	va_start(va, fmt);
	cmd_len = gb_snprintf_va(cmd_line, gb_size_of(cmd_line), fmt, va);
	va_end(va);
	cmd = make_string(cast(u8 *)&cmd_line, cmd_len-1);

	if (build_context.show_system_calls) {
		gb_printf_err("[SYSTEM CALL] %s\n", name);
		gb_printf_err("%s\n\n", cmd_line);
	}
	exit_code = system(cmd_line);

	// pid_t pid = fork();
	// int status = 0;

	// if(pid == 0) {
	// 	// in child, pid == 0.
	// 	int ret = execvp(cmd.text, (char* const*) cmd.text);

	// 	if(ret == -1) {
	// 		gb_printf_err("Failed to execute command:\n\t%s\n", cmd_line);

	// 		// we're in the child, so returning won't do us any good -- just quit.
	// 		exit(-1);
	// 	}

	// 	// unreachable
	// 	abort();
	// } else {
	// 	// wait for child to finish, then we can continue cleanup

	// 	int s = 0;
	// 	waitpid(pid, &s, 0);

	// 	status = WEXITSTATUS(s);
	// }

	// exit_code = status;

	return exit_code;
#endif
}




#if defined(LLVM_BACKEND_SUPPORT)
i32 linker_stage(lbGenerator *gen) {
	i32 exit_code = 0;

	Timings *timings = &global_timings;

	String output_base = gen->output_base;

	if (build_context.metrics.os == TargetOs_js) {
		timings_start_section(timings, str_lit("wasm-ld"));
		exit_code = system_exec_command_line_app("wasm-ld",
			"\"%.*s\\bin\\wasm-ld\" \"%.*s.wasm-obj\" -o \"%.*s.wasm\" %.*s %.*s",
			LIT(build_context.ODIN_ROOT),
			LIT(output_base), LIT(output_base), LIT(build_context.link_flags), LIT(build_context.extra_linker_flags));
		if (exit_code != 0) {
			return exit_code;
		}
		return exit_code;
	}

	if (build_context.cross_compiling && selected_target_metrics->metrics == &target_essence_amd64) {
#ifdef GB_SYSTEM_UNIX
		exit_code = system_exec_command_line_app("linker", "x86_64-essence-gcc \"%.*s.o\" -o \"%.*s\" %.*s %.*s",
			LIT(output_base), LIT(output_base), LIT(build_context.link_flags), LIT(build_context.extra_linker_flags));
		if (exit_code != 0) {
			return exit_code;
		}
#else
		gb_printf_err("Linking for cross compilation for this platform is not yet supported (%.*s %.*s)\n",
			LIT(target_os_names[build_context.metrics.os]),
			LIT(target_arch_names[build_context.metrics.arch])
		);
#endif
	} else if (build_context.cross_compiling && build_context.different_os) {
		gb_printf_err("Linking for cross compilation for this platform is not yet supported (%.*s %.*s)\n",
			LIT(target_os_names[build_context.metrics.os]),
			LIT(target_arch_names[build_context.metrics.arch])
		);
		build_context.keep_object_files = true;
	} else {
	#if defined(GB_SYSTEM_WINDOWS)
		timings_start_section(timings, str_lit("msvc-link"));

		gbString lib_str = gb_string_make(heap_allocator(), "");
		defer (gb_string_free(lib_str));
		char lib_str_buf[1024] = {0};

		char const *output_ext = "exe";
		gbString link_settings = gb_string_make_reserve(heap_allocator(), 256);
		defer (gb_string_free(link_settings));


		// NOTE(ic): It would be nice to extend this so that we could specify the Visual Studio version that we want instead of defaulting to the latest.
		Find_Result_Utf8 find_result = find_visual_studio_and_windows_sdk_utf8();

		if (find_result.windows_sdk_version == 0) {
			gb_printf_err("Windows SDK not found.\n");
			return 1;
		}

		if (build_context.ignore_microsoft_magic) {
			find_result = {};
		}

		// Add library search paths.
		if (find_result.vs_library_path.len > 0) {
			GB_ASSERT(find_result.windows_sdk_um_library_path.len > 0);
			GB_ASSERT(find_result.windows_sdk_ucrt_library_path.len > 0);

			String path = {};
			auto add_path = [&](String path) {
				if (path[path.len-1] == '\\') {
					path.len -= 1;
				}
				link_settings = gb_string_append_fmt(link_settings, " /LIBPATH:\"%.*s\"", LIT(path));
			};
			add_path(find_result.windows_sdk_um_library_path);
			add_path(find_result.windows_sdk_ucrt_library_path);
			add_path(find_result.vs_library_path);
		}

		for_array(i, gen->module.foreign_library_paths) {
			String lib = gen->module.foreign_library_paths[i];
			GB_ASSERT(lib.len < gb_count_of(lib_str_buf)-1);
			isize len = gb_snprintf(lib_str_buf, gb_size_of(lib_str_buf),
			                        " \"%.*s\"", LIT(lib));
			lib_str = gb_string_appendc(lib_str, lib_str_buf);
		}



		if (build_context.build_mode == BuildMode_DynamicLibrary) {
			output_ext = "dll";
			link_settings = gb_string_append_fmt(link_settings, " /DLL");
		} else {
			link_settings = gb_string_append_fmt(link_settings, " /ENTRY:mainCRTStartup");
		}

		if (build_context.pdb_filepath != "") {
			link_settings = gb_string_append_fmt(link_settings, " /PDB:%.*s", LIT(build_context.pdb_filepath));
		}

		if (build_context.no_crt) {
			link_settings = gb_string_append_fmt(link_settings, " /nodefaultlib");
		} else {
			link_settings = gb_string_append_fmt(link_settings, " /defaultlib:libcmt");
		}

		if (build_context.ODIN_DEBUG) {
			link_settings = gb_string_append_fmt(link_settings, " /DEBUG");
		}

		gbString object_files = gb_string_make(heap_allocator(), "");
		defer (gb_string_free(object_files));
		for_array(i, gen->output_object_paths) {
			String object_path = gen->output_object_paths[i];
			object_files = gb_string_append_fmt(object_files, "\"%.*s\" ", LIT(object_path));
		}

		char const *subsystem_str = build_context.use_subsystem_windows ? "WINDOWS" : "CONSOLE";
		if (!build_context.use_lld) { // msvc
			if (build_context.has_resource) {
				exit_code = system_exec_command_line_app("msvc-link",
					"\"rc.exe\" /nologo /fo \"%.*s.res\" \"%.*s.rc\"",
					LIT(output_base),
					LIT(build_context.resource_filepath)
				);

				if (exit_code != 0) {
					return exit_code;
				}

				exit_code = system_exec_command_line_app("msvc-link",
					"\"%.*slink.exe\" %s \"%.*s.res\" -OUT:\"%.*s.%s\" %s "
					"/nologo /incremental:no /opt:ref /subsystem:%s "
					" %.*s "
					" %.*s "
					" %s "
					"",
					LIT(find_result.vs_exe_path), object_files, LIT(output_base), LIT(output_base), output_ext,
					link_settings,
					subsystem_str,
					LIT(build_context.link_flags),
					LIT(build_context.extra_linker_flags),
					lib_str
				);

				if (exit_code != 0) {
					return exit_code;
				}
			} else {
				exit_code = system_exec_command_line_app("msvc-link",
					"\"%.*slink.exe\" %s -OUT:\"%.*s.%s\" %s "
					"/nologo /incremental:no /opt:ref /subsystem:%s "
					" %.*s "
					" %.*s "
					" %s "
					"",
					LIT(find_result.vs_exe_path), object_files, LIT(output_base), output_ext,
					link_settings,
					subsystem_str,
					LIT(build_context.link_flags),
					LIT(build_context.extra_linker_flags),
					lib_str
				);

				if (exit_code != 0) {
					return exit_code;
				}			}
		} else { // lld
			exit_code = system_exec_command_line_app("msvc-link",
				"\"%.*s\\bin\\lld-link\" %s -OUT:\"%.*s.%s\" %s "
				"/nologo /incremental:no /opt:ref /subsystem:%s "
				" %.*s "
				" %.*s "
				" %s "
				"",
				LIT(build_context.ODIN_ROOT),
				LIT(output_base), object_files, output_ext,
				link_settings,
				subsystem_str,
				LIT(build_context.link_flags),
				LIT(build_context.extra_linker_flags),
				lib_str
			);

			if (exit_code != 0) {
				return exit_code;
			}
		}
	#else
		timings_start_section(timings, str_lit("ld-link"));

		// NOTE(vassvik): get cwd, for used for local shared libs linking, since those have to be relative to the exe
		char cwd[256];
		getcwd(&cwd[0], 256);
		//printf("%s\n", cwd);

		// NOTE(vassvik): needs to add the root to the library search paths, so that the full filenames of the library
		//                files can be passed with -l:
		gbString lib_str = gb_string_make(heap_allocator(), "-L/");
		defer (gb_string_free(lib_str));

		for_array(i, gen->module.foreign_library_paths) {
			String lib = gen->module.foreign_library_paths[i];

			// NOTE(zangent): Sometimes, you have to use -framework on MacOS.
			//   This allows you to specify '-f' in a #foreign_system_library,
			//   without having to implement any new syntax specifically for MacOS.
			#if defined(GB_SYSTEM_OSX)
				if (string_ends_with(lib, str_lit(".framework"))) {
					// framework thingie
{'fraction_non_alphanumeric': 0.14678502879078695, 'fraction_numerical': 0.007581573896353167, 'mean_word_length': 3.942138961346929, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6665316', 'n_tokens_mistral': 8968, 'n_tokens_neox': 7943, 'n_words': 1469}