<commit_before><?php

/* PARSER SCRIPT FOR TAGUI FRAMEWORK ~ TEBEL.ORG */

// check flow filename for .tagui or .js or .txt or no extension
$script = $argv[1]; if ($script=="") die("ERROR - specify flow filename as first parameter\n");
if (strpos(pathinfo($script, PATHINFO_BASENAME), '.') !== false) // check if file has extension
if ((pathinfo($script, PATHINFO_EXTENSION)!="gui") and (pathinfo($script, PATHINFO_EXTENSION)!="txt") and (pathinfo($script, PATHINFO_EXTENSION)!="js") and (pathinfo($script, PATHINFO_EXTENSION)!="tagui"))
die("ERROR - use .tagui .js .txt or no extension for flow filename\n");

// make sure required files are available and can be opened
if (!file_exists($script)) die("ERROR - cannot find " . $script . "\n");
$input_file = fopen($script,'r') or die("ERROR - cannot open " . $script . "\n");
$output_file = fopen($script . '.js','w') or die("ERROR - cannot open " . $script . '.js' . "\n");
$config_file = fopen('tagui_config.txt','r') or die("ERROR - cannot open tagui_config.txt" . "\n");
$header_file = fopen('tagui_header.js','r') or die("ERROR - cannot open tagui_header.js" . "\n");
$footer_file = fopen('tagui_footer.js','r') or die("ERROR - cannot open tagui_footer.js" . "\n");

$repo_count = 0; if (file_exists(getenv('custom_csv_file'))) { // load repository file for objects and keywords
$repo_file = fopen(getenv('custom_csv_file'),'r') or die("ERROR - cannot open " . getenv('custom_csv_file') . "\n");
while (!feof($repo_file)) {$repo_data[$repo_count] = fgetcsv($repo_file);
if (count($repo_data[$repo_count]) == 0) die("ERROR - empty row found in " . getenv('custom_csv_file') . "\n");
$repo_count++;} fclose($repo_file); $repo_count-=1; //-1 for header, for EOF need to check flexibly using below line
if (count($repo_data[$repo_count]) == 1) $repo_count-=1;} //-1 for EOF (Windows files don't end with newline character)

$local_repo_location = str_replace("\\","/",dirname($script)) . '/tagui_local.csv';
if (file_exists($local_repo_location)) { // load local repository file if it exists for objects and keywords
$local_repo_file = fopen($local_repo_location,'r') or die("ERROR - cannot open " . 'tagui_local.csv' . "\n");
$repo_count++; fgetcsv($local_repo_file); // increase repo_count to prep for read, discard header record
while (!feof($local_repo_file)) {$repo_data[$repo_count] = fgetcsv($local_repo_file);
if (count($repo_data[$repo_count]) == 0) die("ERROR - empty row found in " . 'tagui_local.csv' . "\n");
if (count($repo_data[$repo_count]) != 1) // pad the empty columns when local repository is used with datatable
{$repo_data[$repo_count] = array_pad($repo_data[$repo_count], count($repo_data[0]), $repo_data[$repo_count][1]);}
$repo_count++;} fclose($local_repo_file); $repo_count-=1; if (count($repo_data[$repo_count]) == 1) $repo_count-=1;}

if (file_exists('tagui_global.csv')) { // load global repository file if it exists for objects and keywords
$global_repo_file = fopen('tagui_global.csv','r') or die("ERROR - cannot open " . 'tagui_global.csv' . "\n");
$repo_count++; fgetcsv($global_repo_file); // increase repo_count to prep for read, discard header record
while (!feof($global_repo_file)) {$repo_data[$repo_count] = fgetcsv($global_repo_file);
if (count($repo_data[$repo_count]) == 0) die("ERROR - empty row found in " . 'tagui_global.csv' . "\n");
if (count($repo_data[$repo_count]) != 1) // pad the empty columns when global repository is used with datatable
{$repo_data[$repo_count] = array_pad($repo_data[$repo_count], count($repo_data[0]), $repo_data[$repo_count][1]);}
$repo_count++;} fclose($global_repo_file); $repo_count-=1; if (count($repo_data[$repo_count]) == 1) $repo_count-=1;}

$tagui_web_browser = "this"; // set the web browser to be used base on tagui_web_browser environment variable
if ((getenv('tagui_web_browser')=='headless') or (getenv('tagui_web_browser')=='chrome')) $tagui_web_browser = 'chrome';

$inside_code_block = 0; // track if step or code is inside user-defined code block
$inside_while_loop = 0; // track if step is in while loop and avoid async wait
$for_loop_tracker = ""; // track for loop to implement IIFE pattern
$code_block_tracker = ""; // track code blocks defined in flow
$line_number = 0; // track flow line number for error message
$real_line_number = 0; // line number excluding comments and blank lines
$test_automation = 0; // to determine casperjs script structure
$url_provided = false; // to detect if url is provided in user-script

// track begin-finish blocks for integrations eg - py, r, run, vision, js, dom
$inside_py_block = 0; $inside_r_block = 0; $inside_run_block = 0;
$inside_vision_block = 0; $inside_js_block = 0; $inside_dom_block = 0;

// series of loops to create casperjs script from header, user flow, footer files

// create header of casperjs script using tagui config and header template
fwrite($output_file,"/* OUTPUT CASPERJS SCRIPT FOR TAGUI FRAMEWORK ~ TEBEL.ORG */\n\n");
fwrite($output_file,"var casper = require('casper').create();\n"); // opening lines
while(!feof($config_file)) {fwrite($output_file,fgets($config_file));} fclose($config_file);
while(!feof($header_file)) {fwrite($output_file,fgets($header_file));} fclose($header_file);

// append global functions file created by user if the file exists
if (file_exists('tagui_global.js')) {
$global_functions_file = fopen('tagui_global.js','r') or die("ERROR - cannot open tagui_global.js" . "\n");
while(!feof($global_functions_file)) {fwrite($output_file,fgets($global_functions_file));}
fwrite($output_file,"\n"); fclose($global_functions_file);}

// append local functions file created by user if the file exists
$local_functions_location = str_replace("\\","/",dirname($script)) . '/tagui_local.js';
if (file_exists($local_functions_location)) {
$local_functions_file = fopen($local_functions_location,'r') or die("ERROR - cannot open tagui_local.js" . "\n");
while(!feof($local_functions_file)) {fwrite($output_file,fgets($local_functions_file));}
fwrite($output_file,"\n"); fclose($local_functions_file);}

// append comment on flow path variable in casperjs script
fwrite($output_file,"// flow path for save_text and snap_image\n");

// save flow path in casperjs script to be used by save_text and snap_image
// casperjs/phantomjs do not seem to support \ for windows paths, replace with / to work
// below marker for appending url_intent when flow file does not start with comments or url
$marker_for_opening_url = "var flow_path = '" . str_replace("\\","/",dirname($script)) . "';\n\n";
fwrite($output_file,$marker_for_opening_url);

// section to handle calling of other TagUI automation scripts for reusability
$temp_output_file = fopen($script . '.raw','w') or die("ERROR - cannot open " . $script . '.raw' . "\n");
while(!feof($input_file)) {fwrite($temp_output_file,expand_intent(fgets($input_file)));} fclose($input_file);
fclose($temp_output_file); // generate temp output file of expanded intents (if any) before reopening as input
$input_file = fopen($script . '.raw','r') or die("ERROR - cannot open " . $script . '.raw' . "\n");

if (strpos(strtolower(file_get_contents('tagui_config.txt')),"var tagui_language = 'english';")==false)
{ // section which includes translation engine for handling flows in other languages
$temp_argv1 = $argv[1]; $temp_argv2 = $argv[2]; $temp_argv3 = $argv[3];
$argv[1] = 'tagui_parse.php'; $argv[2] = 'from';
$temp_tagui_config = strtolower(file_get_contents('tagui_config.txt'));
$temp_tagui_config_start = strpos($temp_tagui_config,'var tagui_language');
$temp_tagui_config_end = strpos($temp_tagui_config,"\n",$temp_tagui_config_start);
$temp_tagui_config = substr($temp_tagui_config,$temp_tagui_config_start,$temp_tagui_config_end-$temp_tagui_config_start);
$temp_tagui_config = str_replace('var tagui_language','',$temp_tagui_config);
$temp_tagui_config = str_replace('"','',$temp_tagui_config); $temp_tagui_config = str_replace("'",'',$temp_tagui_config);
$temp_tagui_config = str_replace('=','',$temp_tagui_config); $temp_tagui_config = str_replace(';','',$temp_tagui_config);
$argv[3] = trim($temp_tagui_config); require 'translate.php'; // set parameters to load translation engine
$argv[1] = $temp_argv1; $argv[2] = $temp_argv2; $argv[3] = $temp_argv3; $translated_raw_flow = "";
while(!feof($input_file)) {$translated_raw_flow .= translate_intent(fgets($input_file));} fclose($input_file);
file_put_contents($script . '.raw', $translated_raw_flow); // save translated output before reopening as input
$input_file = fopen($script . '.raw','r') or die("ERROR - cannot open " . $script . '.raw' . "\n");}

// section to do required pre-processing on expanded .raw flow file
$padded_raw_flow = ""; $previous_line_is_condition = false;
while(!feof($input_file)) {$padded_raw_flow_line = ltrim(fgets($input_file));
// track whether line is inside integrations begin-finish code blocks
if (strtolower(trim($padded_raw_flow_line)) == "js begin") $inside_js_block = 1;
else if (strtolower(trim($padded_raw_flow_line)) == "js finish") $inside_js_block = 0;
else if (strtolower(trim($padded_raw_flow_line)) == "py begin") $inside_py_block = 1;
else if (strtolower(trim($padded_raw_flow_line)) == "py finish") $inside_py_block = 0;
else if (strtolower(trim($padded_raw_flow_line)) == "r begin") $inside_r_block = 1; 
else if (strtolower(trim($padded_raw_flow_line)) == "r finish") $inside_r_block = 0;
else if (strtolower(trim($padded_raw_flow_line)) == "dom begin") $inside_dom_block = 1; 
else if (strtolower(trim($padded_raw_flow_line)) == "dom finish") $inside_dom_block = 0;
else if (strtolower(trim($padded_raw_flow_line)) == "run begin") $inside_run_block = 1; 
else if (strtolower(trim($padded_raw_flow_line)) == "run finish") $inside_run_block = 0;
else if (strtolower(trim($padded_raw_flow_line)) == "vision begin") $inside_vision_block = 1; 
else if (strtolower(trim($padded_raw_flow_line)) == "vision finish") $inside_vision_block = 0;
if (($inside_js_block + $inside_py_block + $inside_r_block + $inside_dom_block + $inside_run_block + $inside_vision_block)
> 0) {$padded_raw_flow .= $padded_raw_flow_line; continue;} // auto-padding not relevant in integrations code blocks
// rewrite JS function definitions to work in scope within CasperJS blocks
if ((substr($padded_raw_flow_line,0,9)=="function ") or (substr($padded_raw_flow_line,0,12)=="js function "))
if (strpos($padded_raw_flow_line,"(")!==false) {$js_function_name_startpos = strpos($padded_raw_flow_line,"function ")+9;
$js_function_name_endpos = strpos($padded_raw_flow_line,"(",$js_function_name_startpos);
$padded_raw_flow_line = trim(substr($padded_raw_flow_line,$js_function_name_startpos,$js_function_name_endpos -
$js_function_name_startpos)) . ' = function ' . trim(substr($padded_raw_flow_line,$js_function_name_endpos))."\n";}
else die("ERROR - missing brackets () for ".$padded_raw_flow_line);
// pad { and } blocks for conditions, to keep JavaScript syntax correct
if ((substr($padded_raw_flow_line,0,3)=="if ") or (substr($padded_raw_flow_line,0,8)=="else if ")
or (substr($padded_raw_flow_line,0,4)=="for ") or (substr($padded_raw_flow_line,0,6)=="while ") or
(substr($padded_raw_flow_line,0,6)=="popup ") or (substr($padded_raw_flow_line,0,6)=="frame ") or
(trim($padded_raw_flow_line)=="else")) $current_line_is_condition = true; else $current_line_is_condition = false;
if (($previous_line_is_condition == true) and ($current_line_is_condition == true))
die("ERROR - for nested conditions, loops, popup, frame, set { and } explicitly\n".
"ERROR - add { before this line and add } accordingly - ".$padded_raw_flow_line);
if (($previous_line_is_condition == true) and (substr($padded_raw_flow_line,0,1)!="{"))
$padded_raw_flow .= "{\n".trim($padded_raw_flow_line)."\n}\n"; else $padded_raw_flow .= $padded_raw_flow_line;
$previous_line_is_condition = $current_line_is_condition; // prepare for next line
} fclose($input_file); file_put_contents($script . '.raw', $padded_raw_flow);
// generate temp output file with padded { and } (if any) before reopening as input
$input_file = fopen($script . '.raw','r') or die("ERROR - cannot open " . $script . '.raw' . "\n");
// re-initialize trackers for begin-finish blocks of integrations
$inside_py_block = 0; $inside_r_block = 0; $inside_run_block = 0;
$inside_vision_block = 0; $inside_js_block = 0; $inside_dom_block = 0;

// main loop to parse intents in flow file for conversion into javascript code
while(!feof($input_file)) {
$cleaned_intent = clean_intent(fgets($input_file));
if ($cleaned_intent == "") continue;
$intent_type = get_intent($cleaned_intent);
if ($intent_type == "block") {
$GLOBALS['block'] = $GLOBALS['block'] . $cleaned_intent ."\\n";}
else {switch($cleaned_intent) {
//Note: need to replace \\n with single \n if the output is not being sent as string outside of casperJS.
//\\n is needed for py, r, sikuli as the output multiline string needs to have \n escaped to work properly in JavaScript.
case("js finish"): {$cleaned_intent = str_replace("\\n", "\n", $GLOBALS['block']); $GLOBALS['inside_js_block'] = 0; break;}
case("py finish"): {$cleaned_intent = $GLOBALS['block']; $GLOBALS['inside_py_block'] = 0; break;}
case("r finish"): {$cleaned_intent = $GLOBALS['block']; $GLOBALS['inside_r_block'] = 0; break;}
case("dom finish"): {$cleaned_intent = str_replace("\\n", "\n", $GLOBALS['block']); $GLOBALS['inside_dom_block'] = 0; break;}
case("vision finish"): {$cleaned_intent = $GLOBALS['block']; $GLOBALS['inside_vision_block'] = 0; break;}}
$output_line = parse_intent($intent_type, $cleaned_intent);
fwrite($output_file, $output_line);}}
fclose($input_file);

// create footer of casperjs script using footer template and do post-processing 
while(!feof($footer_file)) {fwrite($output_file,fgets($footer_file));} fclose($footer_file); fclose($output_file);
chmod ($script . '.js',0600); // append url opening block below instead of throwing error
if (!$url_provided) { // echo "ERROR - first line of " . $script . " not URL or comments\n";
$GLOBALS['real_line_number'] = 1; $generated_js_file_contents = file_get_contents($script . '.js');
$generated_js_file_contents = str_replace($marker_for_opening_url, $marker_for_opening_url . 
url_intent('about:blank'), $generated_js_file_contents); file_put_contents($script . '.js',$generated_js_file_contents);}
if ($inside_code_block != 0) echo "ERROR - number of step { does not tally with with }\n";

// post-processing to clean up artifacts from translating human language to JavaScript
$script_content = file_get_contents($script . '.js');
$script_content = str_replace("});\n\ncasper.then(function() {else\n","\nelse\n\n",$script_content);
$script_content = str_replace("});\n\ncasper.then(function() {else if ","\nelse if ",$script_content);
$script_content = str_replace("}); // end of JS code\n\ncasper.then(function() { // start of JS code\n",
"",$script_content); // above collapse separate js step CasperJS blocks into one continuous block
$script_content = preg_replace('/^casper\.then\(function\(\) {{(.*) \/\/ beg_tx while loop marker/m',
'$1',$script_content); // above regular expression replace lines having beg_tx while loop markers
$script_content = preg_replace('/^casper\.then\(function\(\) {(.*) \/\/ end_fi while loop marker(.*)}\);/m',
'$1$2',$script_content); // above regular expression replace lines having end_fi while loop markers<commit_msg>Trim extra newlines at the end of buffer string used. Change r blocks to use ; to separate lines of code instead of newlines.<commit_after><?php

/* PARSER SCRIPT FOR TAGUI FRAMEWORK ~ TEBEL.ORG */

// check flow filename for .tagui or .js or .txt or no extension
$script = $argv[1]; if ($script=="") die("ERROR - specify flow filename as first parameter\n");
if (strpos(pathinfo($script, PATHINFO_BASENAME), '.') !== false) // check if file has extension
if ((pathinfo($script, PATHINFO_EXTENSION)!="gui") and (pathinfo($script, PATHINFO_EXTENSION)!="txt") and (pathinfo($script, PATHINFO_EXTENSION)!="js") and (pathinfo($script, PATHINFO_EXTENSION)!="tagui"))
die("ERROR - use .tagui .js .txt or no extension for flow filename\n");

// make sure required files are available and can be opened
if (!file_exists($script)) die("ERROR - cannot find " . $script . "\n");
$input_file = fopen($script,'r') or die("ERROR - cannot open " . $script . "\n");
$output_file = fopen($script . '.js','w') or die("ERROR - cannot open " . $script . '.js' . "\n");
$config_file = fopen('tagui_config.txt','r') or die("ERROR - cannot open tagui_config.txt" . "\n");
$header_file = fopen('tagui_header.js','r') or die("ERROR - cannot open tagui_header.js" . "\n");
$footer_file = fopen('tagui_footer.js','r') or die("ERROR - cannot open tagui_footer.js" . "\n");

$repo_count = 0; if (file_exists(getenv('custom_csv_file'))) { // load repository file for objects and keywords
$repo_file = fopen(getenv('custom_csv_file'),'r') or die("ERROR - cannot open " . getenv('custom_csv_file') . "\n");
while (!feof($repo_file)) {$repo_data[$repo_count] = fgetcsv($repo_file);
if (count($repo_data[$repo_count]) == 0) die("ERROR - empty row found in " . getenv('custom_csv_file') . "\n");
$repo_count++;} fclose($repo_file); $repo_count-=1; //-1 for header, for EOF need to check flexibly using below line
if (count($repo_data[$repo_count]) == 1) $repo_count-=1;} //-1 for EOF (Windows files don't end with newline character)

$local_repo_location = str_replace("\\","/",dirname($script)) . '/tagui_local.csv';
if (file_exists($local_repo_location)) { // load local repository file if it exists for objects and keywords
$local_repo_file = fopen($local_repo_location,'r') or die("ERROR - cannot open " . 'tagui_local.csv' . "\n");
$repo_count++; fgetcsv($local_repo_file); // increase repo_count to prep for read, discard header record
while (!feof($local_repo_file)) {$repo_data[$repo_count] = fgetcsv($local_repo_file);
if (count($repo_data[$repo_count]) == 0) die("ERROR - empty row found in " . 'tagui_local.csv' . "\n");
if (count($repo_data[$repo_count]) != 1) // pad the empty columns when local repository is used with datatable
{$repo_data[$repo_count] = array_pad($repo_data[$repo_count], count($repo_data[0]), $repo_data[$repo_count][1]);}
$repo_count++;} fclose($local_repo_file); $repo_count-=1; if (count($repo_data[$repo_count]) == 1) $repo_count-=1;}

if (file_exists('tagui_global.csv')) { // load global repository file if it exists for objects and keywords
$global_repo_file = fopen('tagui_global.csv','r') or die("ERROR - cannot open " . 'tagui_global.csv' . "\n");
$repo_count++; fgetcsv($global_repo_file); // increase repo_count to prep for read, discard header record
while (!feof($global_repo_file)) {$repo_data[$repo_count] = fgetcsv($global_repo_file);
if (count($repo_data[$repo_count]) == 0) die("ERROR - empty row found in " . 'tagui_global.csv' . "\n");
if (count($repo_data[$repo_count]) != 1) // pad the empty columns when global repository is used with datatable
{$repo_data[$repo_count] = array_pad($repo_data[$repo_count], count($repo_data[0]), $repo_data[$repo_count][1]);}
$repo_count++;} fclose($global_repo_file); $repo_count-=1; if (count($repo_data[$repo_count]) == 1) $repo_count-=1;}

$tagui_web_browser = "this"; // set the web browser to be used base on tagui_web_browser environment variable
if ((getenv('tagui_web_browser')=='headless') or (getenv('tagui_web_browser')=='chrome')) $tagui_web_browser = 'chrome';

$inside_code_block = 0; // track if step or code is inside user-defined code block
$inside_while_loop = 0; // track if step is in while loop and avoid async wait
$for_loop_tracker = ""; // track for loop to implement IIFE pattern
$code_block_tracker = ""; // track code blocks defined in flow
$line_number = 0; // track flow line number for error message
$real_line_number = 0; // line number excluding comments and blank lines
$test_automation = 0; // to determine casperjs script structure
$url_provided = false; // to detect if url is provided in user-script

// track begin-finish blocks for integrations eg - py, r, run, vision, js, dom
$inside_py_block = 0; $inside_r_block = 0; $inside_run_block = 0;
$inside_vision_block = 0; $inside_js_block = 0; $inside_dom_block = 0;

// series of loops to create casperjs script from header, user flow, footer files

// create header of casperjs script using tagui config and header template
fwrite($output_file,"/* OUTPUT CASPERJS SCRIPT FOR TAGUI FRAMEWORK ~ TEBEL.ORG */\n\n");
fwrite($output_file,"var casper = require('casper').create();\n"); // opening lines
while(!feof($config_file)) {fwrite($output_file,fgets($config_file));} fclose($config_file);
while(!feof($header_file)) {fwrite($output_file,fgets($header_file));} fclose($header_file);

// append global functions file created by user if the file exists
if (file_exists('tagui_global.js')) {
$global_functions_file = fopen('tagui_global.js','r') or die("ERROR - cannot open tagui_global.js" . "\n");
while(!feof($global_functions_file)) {fwrite($output_file,fgets($global_functions_file));}
fwrite($output_file,"\n"); fclose($global_functions_file);}

// append local functions file created by user if the file exists
$local_functions_location = str_replace("\\","/",dirname($script)) . '/tagui_local.js';
if (file_exists($local_functions_location)) {
$local_functions_file = fopen($local_functions_location,'r') or die("ERROR - cannot open tagui_local.js" . "\n");
while(!feof($local_functions_file)) {fwrite($output_file,fgets($local_functions_file));}
fwrite($output_file,"\n"); fclose($local_functions_file);}

// append comment on flow path variable in casperjs script
fwrite($output_file,"// flow path for save_text and snap_image\n");

// save flow path in casperjs script to be used by save_text and snap_image
// casperjs/phantomjs do not seem to support \ for windows paths, replace with / to work
// below marker for appending url_intent when flow file does not start with comments or url
$marker_for_opening_url = "var flow_path = '" . str_replace("\\","/",dirname($script)) . "';\n\n";
fwrite($output_file,$marker_for_opening_url);

// section to handle calling of other TagUI automation scripts for reusability
$temp_output_file = fopen($script . '.raw','w') or die("ERROR - cannot open " . $script . '.raw' . "\n");
while(!feof($input_file)) {fwrite($temp_output_file,expand_intent(fgets($input_file)));} fclose($input_file);
fclose($temp_output_file); // generate temp output file of expanded intents (if any) before reopening as input
$input_file = fopen($script . '.raw','r') or die("ERROR - cannot open " . $script . '.raw' . "\n");

if (strpos(strtolower(file_get_contents('tagui_config.txt')),"var tagui_language = 'english';")==false)
{ // section which includes translation engine for handling flows in other languages
$temp_argv1 = $argv[1]; $temp_argv2 = $argv[2]; $temp_argv3 = $argv[3];
$argv[1] = 'tagui_parse.php'; $argv[2] = 'from';
$temp_tagui_config = strtolower(file_get_contents('tagui_config.txt'));
$temp_tagui_config_start = strpos($temp_tagui_config,'var tagui_language');
$temp_tagui_config_end = strpos($temp_tagui_config,"\n",$temp_tagui_config_start);
$temp_tagui_config = substr($temp_tagui_config,$temp_tagui_config_start,$temp_tagui_config_end-$temp_tagui_config_start);
$temp_tagui_config = str_replace('var tagui_language','',$temp_tagui_config);
$temp_tagui_config = str_replace('"','',$temp_tagui_config); $temp_tagui_config = str_replace("'",'',$temp_tagui_config);
$temp_tagui_config = str_replace('=','',$temp_tagui_config); $temp_tagui_config = str_replace(';','',$temp_tagui_config);
$argv[3] = trim($temp_tagui_config); require 'translate.php'; // set parameters to load translation engine
$argv[1] = $temp_argv1; $argv[2] = $temp_argv2; $argv[3] = $temp_argv3; $translated_raw_flow = "";
while(!feof($input_file)) {$translated_raw_flow .= translate_intent(fgets($input_file));} fclose($input_file);
file_put_contents($script . '.raw', $translated_raw_flow); // save translated output before reopening as input
$input_file = fopen($script . '.raw','r') or die("ERROR - cannot open " . $script . '.raw' . "\n");}

// section to do required pre-processing on expanded .raw flow file
$padded_raw_flow = ""; $previous_line_is_condition = false;
while(!feof($input_file)) {$padded_raw_flow_line = ltrim(fgets($input_file));
// track whether line is inside integrations begin-finish code blocks
if (strtolower(trim($padded_raw_flow_line)) == "js begin") $inside_js_block = 1;
else if (strtolower(trim($padded_raw_flow_line)) == "js finish") $inside_js_block = 0;
else if (strtolower(trim($padded_raw_flow_line)) == "py begin") $inside_py_block = 1;
else if (strtolower(trim($padded_raw_flow_line)) == "py finish") $inside_py_block = 0;
else if (strtolower(trim($padded_raw_flow_line)) == "r begin") $inside_r_block = 1; 
else if (strtolower(trim($padded_raw_flow_line)) == "r finish") $inside_r_block = 0;
else if (strtolower(trim($padded_raw_flow_line)) == "dom begin") $inside_dom_block = 1; 
else if (strtolower(trim($padded_raw_flow_line)) == "dom finish") $inside_dom_block = 0;
else if (strtolower(trim($padded_raw_flow_line)) == "run begin") $inside_run_block = 1; 
else if (strtolower(trim($padded_raw_flow_line)) == "run finish") $inside_run_block = 0;
else if (strtolower(trim($padded_raw_flow_line)) == "vision begin") $inside_vision_block = 1; 
else if (strtolower(trim($padded_raw_flow_line)) == "vision finish") $inside_vision_block = 0;
if (($inside_js_block + $inside_py_block + $inside_r_block + $inside_dom_block + $inside_run_block + $inside_vision_block)
> 0) {$padded_raw_flow .= $padded_raw_flow_line; continue;} // auto-padding not relevant in integrations code blocks
// rewrite JS function definitions to work in scope within CasperJS blocks
if ((substr($padded_raw_flow_line,0,9)=="function ") or (substr($padded_raw_flow_line,0,12)=="js function "))
if (strpos($padded_raw_flow_line,"(")!==false) {$js_function_name_startpos = strpos($padded_raw_flow_line,"function ")+9;
$js_function_name_endpos = strpos($padded_raw_flow_line,"(",$js_function_name_startpos);
$padded_raw_flow_line = trim(substr($padded_raw_flow_line,$js_function_name_startpos,$js_function_name_endpos -
$js_function_name_startpos)) . ' = function ' . trim(substr($padded_raw_flow_line,$js_function_name_endpos))."\n";}
else die("ERROR - missing brackets () for ".$padded_raw_flow_line);
// pad { and } blocks for conditions, to keep JavaScript syntax correct
if ((substr($padded_raw_flow_line,0,3)=="if ") or (substr($padded_raw_flow_line,0,8)=="else if ")
or (substr($padded_raw_flow_line,0,4)=="for ") or (substr($padded_raw_flow_line,0,6)=="while ") or
(substr($padded_raw_flow_line,0,6)=="popup ") or (substr($padded_raw_flow_line,0,6)=="frame ") or
(trim($padded_raw_flow_line)=="else")) $current_line_is_condition = true; else $current_line_is_condition = false;
if (($previous_line_is_condition == true) and ($current_line_is_condition == true))
die("ERROR - for nested conditions, loops, popup, frame, set { and } explicitly\n".
"ERROR - add { before this line and add } accordingly - ".$padded_raw_flow_line);
if (($previous_line_is_condition == true) and (substr($padded_raw_flow_line,0,1)!="{"))
$padded_raw_flow .= "{\n".trim($padded_raw_flow_line)."\n}\n"; else $padded_raw_flow .= $padded_raw_flow_line;
$previous_line_is_condition = $current_line_is_condition; // prepare for next line
} fclose($input_file); file_put_contents($script . '.raw', $padded_raw_flow);
// generate temp output file with padded { and } (if any) before reopening as input
$input_file = fopen($script . '.raw','r') or die("ERROR - cannot open " . $script . '.raw' . "\n");
// re-initialize trackers for begin-finish blocks of integrations
$inside_py_block = 0; $inside_r_block = 0; $inside_run_block = 0;
$inside_vision_block = 0; $inside_js_block = 0; $inside_dom_block = 0;

// main loop to parse intents in flow file for conversion into javascript code
while(!feof($input_file)) {
$cleaned_intent = clean_intent(fgets($input_file));
if ($cleaned_intent == "") continue;
$intent_type = get_intent($cleaned_intent);
if ($intent_type == "block") {
$GLOBALS['block'] = $GLOBALS['block'] . $cleaned_intent ."\\n";}
else {switch($cleaned_intent) {
//Note: need to replace \\n with single \n if the output is not being sent as string outside of casperJS.
//\\n is needed for py, sikuli as the output multiline string needs to have \n escaped to work properly in JavaScript.
case("js finish"): {$cleaned_intent = str_replace("\\n", "\n", $GLOBALS['block']); $GLOBALS['inside_js_block'] = 0; break;}
case("py finish"): {$cleaned_intent = substr($GLOBALS['block'],0,-2); $GLOBALS['inside_py_block'] = 0; break;}
case("r finish"): {$cleaned_intent = substr($GLOBALS['block'],0,-2); $GLOBALS['inside_r_block'] = 0;
$cleaned_intent = str_replace(";; ","; ",str_replace("\\n","; ",$GLOBALS['block'])); break;}
case("dom finish"): {$cleaned_intent = str_replace("\\n", "\n", $GLOBALS['block']); $GLOBALS['inside_dom_block'] = 0; break;}
case("vision finish"): {$cleaned_intent = $GLOBALS['block']; $GLOBALS['inside_vision_block'] = 0; break;}}
$output_line = parse_intent($intent_type, $cleaned_intent);
fwrite($output_file, $output_line);}}
fclose($input_file);

// create footer of casperjs script using footer template and do post-processing 
while(!feof($footer_file)) {fwrite($output_file,fgets($footer_file));} fclose($footer_file); fclose($output_file);
chmod ($script . '.js',0600); // append url opening block below instead of throwing error
if (!$url_provided) { // echo "ERROR - first line of " . $script . " not URL or comments\n";
$GLOBALS['real_line_number'] = 1; $generated_js_file_contents = file_get_contents($script . '.js');
$generated_js_file_contents = str_replace($marker_for_opening_url, $marker_for_opening_url . 
url_intent('about:blank'), $generated_js_file_contents); file_put_contents($script . '.js',$generated_js_file_contents);}
if ($inside_code_block != 0) echo "ERROR - number of step { does not tally with with }\n";

// post-processing to clean up artifacts from translating human language to JavaScript
$script_content = file_get_contents($script . '.js');
$script_content = str_replace("});\n\ncasper.then(function() {else\n","\nelse\n\n",$script_content);
$script_content = str_replace("});\n\ncasper.then(function() {else if ","\nelse if ",$script_content);
$script_content = str_replace("}); // end of JS code\n\ncasper.then(function() { // start of JS code\n",
"",$script_content); // above collapse separate js step CasperJS blocks into one continuous block
$script_content = preg_replace('/^casper\.then\(function\(\) {{(.*) \/\/ beg_tx while loop marker/m',
'$1',$script_content); // above regular expression replace lines having beg_tx while loop markers
$script_content = preg_replace('/^casper\.then\(function\(\) {(.*) \/\/ end_fi while loop marker(.*)}\);/m',
'$1$2',$script_content); // above regular expression replace lines having end_fi while loop markers
{'fraction_non_alphanumeric': 0.16275460717749757, 'fraction_numerical': 0.006660200452634982, 'mean_word_length': 6.4947904046522895, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7419321', 'n_tokens_mistral': 10521, 'n_tokens_neox': 10237, 'n_words': 2614}