<commit_before>-module(zml).

-compile(export_all).


compile(InFile) ->
  AST = zml_hand_parser:parse(zml_tokenizer:tokenize_file(InFile)),
  SourceDir = filename:dirname(filename:absname(InFile)),
  StagingDirName = erlang:phash2(make_ref()),
  % TODO: you are here
  AST2 = run_specialized_handlers(AST, SourceDir, 
  translate_ast_item(AST, []).

translate_ast_item([], Acc) ->
  lists:reverse(Acc);
translate_ast_item([String | [Next | _] = T], Acc)
    when is_list(String) and is_list(Next) ->
  translate_ast_item(T, [" " | [String | Acc]]);
translate_ast_item([String | T], Acc) when is_list(String) ->
  translate_ast_item(T, [String | Acc]);
translate_ast_item([{Code,code,[],Children} | T], Acc) ->
  ToAppend = ["!!CODE!!",
    string:join(Code, " "),
    "!!",
    translate_ast_item(Children, []),
    "!!END!!"],
  translate_ast_item(T, [ToAppend | Acc]);
translate_ast_item([{Name,Type,Attributes,[]} | T], Acc) -><commit_msg>Started to set up zml for special handlers<commit_after>-module(zml).

-compile(export_all).


compile(InFile) ->
  AST = zml_hand_parser:parse(zml_tokenizer:tokenize_file(InFile)),
  SourceDir = filename:dirname(filename:absname(InFile)),
  StagingDirName = integer_to_list(erlang:phash2(make_ref())),
  {ok, CurrDir} = file:get_cwd(),
  StagingDir = filename:join([CurrDir, StagingDirName]),
  file:make_dir(StagingDir),
  AST2 = run_specialized_handlers(AST, SourceDir, StagingDir),
  Out = translate_ast_item(AST2, []),
  file:write_file(filename:join([StagingDir, "output.html"]), Out),
  io:format("~s", [Out]),
  StagingDir.

run_specialized_handlers(AST, SourceDir, StagingDir) ->
  run_spec_handlers_inner(AST, SourceDir, StagingDir, AST).

run_spec_handlers_inner([{Name, special, Attr, Children} | T], DSource, DStage, FullAST) ->
  % Call handler TODO
  nyi;
run_spec_handler_inner([H|T],DSource,DStage,FullAST) ->
  run_spec_handler_inner(T, DSource, DStage, FullAST).
  

translate_ast_item([], Acc) ->
  lists:reverse(Acc);
translate_ast_item([String | [Next | _] = T], Acc)
    when is_list(String) and is_list(Next) ->
  translate_ast_item(T, [" " | [String | Acc]]);
translate_ast_item([String | T], Acc) when is_list(String) ->
  translate_ast_item(T, [String | Acc]);
translate_ast_item([{Code,code,[],Children} | T], Acc) ->
  ToAppend = ["!!CODE!!",
    string:join(Code, " "),
    "!!",
    translate_ast_item(Children, []),
    "!!END!!"],
  translate_ast_item(T, [ToAppend | Acc]);
translate_ast_item([{Name,Type,Attributes,[]} | T], Acc) ->
{'fraction_non_alphanumeric': 0.1789557592666401, 'fraction_numerical': 0.0019928258270227183, 'mean_word_length': 6.011173184357542, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5027758', 'n_tokens_mistral': 942, 'n_tokens_neox': 930, 'n_words': 148}