This task has been flagged for clarification. Code on this page in its current state may be flagged incorrect once this task has been clarified. See this page's Talk page for discussion.
Often data is produced by one program, in the wrong format for later use by another program or person. In these situations another program can be written to parse and transform the original data into a format useful to the other. The term "Data Munging" is often used in programming circles for this task.

A request on the comp.lang.awk newsgroup led to a typical data munging task:

I have to analyse data files that have the following format:
Each row corresponds to 1 day and the field logic is: $1 is the date,
followed by 24 value/flag pairs, representing measurements at 01:00,
02:00 ... 24:00 of the respective day. In short:

<date> <val1> <flag1> <val2> <flag2> ...  <val24> <flag24>

Some test data is available at: 
... (nolonger available at original location)

I have to sum up the values (per day and only valid data, i.e. with
flag>0) in order to calculate the mean. That's not too difficult.
However, I also need to know what the "maximum data gap" is, i.e. the
longest period with successive invalid measurements (i.e values with
flag<=0)
The data is free to download and use and is of this format:

Data is no longer available at that link. Zipped mirror available here (offsite mirror).

1991-03-30	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1
1991-03-31	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	20.000	1	20.000	1	20.000	1	35.000	1	50.000	1	60.000	1	40.000	1	30.000	1	30.000	1	30.000	1	25.000	1	20.000	1	20.000	1	20.000	1	20.000	1	20.000	1	35.000	1
1991-03-31	40.000	1	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2
1991-04-01	0.000	-2	13.000	1	16.000	1	21.000	1	24.000	1	22.000	1	20.000	1	18.000	1	29.000	1	44.000	1	50.000	1	43.000	1	38.000	1	27.000	1	27.000	1	24.000	1	23.000	1	18.000	1	12.000	1	13.000	1	14.000	1	15.000	1	13.000	1	10.000	1
1991-04-02	8.000	1	9.000	1	11.000	1	12.000	1	12.000	1	12.000	1	27.000	1	26.000	1	27.000	1	33.000	1	32.000	1	31.000	1	29.000	1	31.000	1	25.000	1	25.000	1	24.000	1	21.000	1	17.000	1	14.000	1	15.000	1	12.000	1	12.000	1	10.000	1
1991-04-03	10.000	1	9.000	1	10.000	1	10.000	1	9.000	1	10.000	1	15.000	1	24.000	1	28.000	1	24.000	1	18.000	1	14.000	1	12.000	1	13.000	1	14.000	1	15.000	1	14.000	1	15.000	1	13.000	1	13.000	1	13.000	1	12.000	1	10.000	1	10.000	1

Only a sample of the data showing its format is given above. The full example file may be downloaded here.

Structure your program to show statistics for each line of the file, (similar to the original Python, Perl, and AWK examples below), followed by summary statistics for the file. When showing example output just show a few line statistics and the full end summary.

let input_line ic =
  try Some(input_line ic)
  with End_of_file -> None
 
let fold_input f ini ic =
  let rec fold ac =
    match input_line ic with
    | Some line -> fold (f ac line)
    | None -> ac
  in
  fold ini
 
let ic = open_in "readings.txt"
 
let scan line =
  Scanf.sscanf line "%s\
    \t%f\t%d\t%f\t%d\t%f\t%d\t%f\t%d\t%f\t%d\t%f\t%d\
    \t%f\t%d\t%f\t%d\t%f\t%d\t%f\t%d\t%f\t%d\t%f\t%d\
    \t%f\t%d\t%f\t%d\t%f\t%d\t%f\t%d\t%f\t%d\t%f\t%d\
    \t%f\t%d\t%f\t%d\t%f\t%d\t%f\t%d\t%f\t%d\t%f\t%d"
    (fun date
         v1  f1  v2  f2  v3  f3  v4  f4  v5  f5  v6  f6
         v7  f7  v8  f8  v9  f9  v10 f10 v11 f11 v12 f12
         v13 f13 v14 f14 v15 f15 v16 f16 v17 f17 v18 f18
         v19 f19 v20 f20 v21 f21 v22 f22 v23 f23 v24 f24 ->
      (date),
      [ (v1,  f1 ); (v2,  f2 ); (v3,  f3 ); (v4,  f4 ); (v5,  f5 ); (v6,  f6 );
        (v7,  f7 ); (v8,  f8 ); (v9,  f9 ); (v10, f10); (v11, f11); (v12, f12);
        (v13, f13); (v14, f14); (v15, f15); (v16, f16); (v17, f17); (v18, f18);
        (v19, f19); (v20, f20); (v21, f21); (v22, f22); (v23, f23); (v24, f24); ])
 
let tot_file, num_file, _, nodata_max, nodata_maxline =
  fold_input
    (fun (tot_file, num_file, nodata, nodata_max, nodata_maxline) line ->
       let date, datas = scan line in
       let _datas = List.filter (fun (_, flag) -> flag > 0) datas in
       let ok = List.length _datas in
       let tot = List.fold_left (fun ac (value, _) -> ac +. value) 0.0 _datas in
       let nodata, nodata_max, nodata_maxline =
         List.fold_left
             (fun (nodata, nodata_max, nodata_maxline) (_, flag) ->
                if flag <= 0
                then (succ nodata, nodata_max, nodata_maxline)
                else
                  if nodata_max = nodata && nodata > 0
                  then (0, nodata_max, date::nodata_maxline)
                  else if nodata_max < nodata && nodata > 0
                  then (0, nodata, [date])
                  else (0, nodata_max, nodata_maxline)
             )
             (nodata, nodata_max, nodata_maxline) datas in
       Printf.printf "Line: %s" date;
       Printf.printf "  Reject: %2d  Accept: %2d" (24 - ok) ok;
       Printf.printf "\tLine_tot: %8.3f" tot;
       Printf.printf "\tLine_avg: %8.3f\n" (tot /. float ok);
       (tot_file +. tot, num_file + ok, nodata, nodata_max, nodata_maxline))
    (0.0, 0, 0, 0, [])
    ic ;;
 
close_in ic ;;
 
Printf.printf "Total    = %f\n" tot_file;
Printf.printf "Readings = %d\n" num_file;
Printf.printf "Average  = %f\n" (tot_file /. float num_file);
Printf.printf "Maximum run(s) of %d consecutive false readings \
               ends at line starting with date(s): %s\n"
               nodata_max (String.concat ", " nodata_maxline);