The following task concerns data that came from a pollution monitoring station with twenty-four instruments monitoring twenty-four aspects of pollution in the air. Periodically a record is added to the file, each record being a line of 49 fields separated by white-space, which can be one or more space or tab characters.

The fields (from the left) are:

 DATESTAMP [ VALUEn FLAGn ] * 24

i.e. a datestamp followed by twenty-four repetitions of a floating-point instrument value and that instrument's associated integer flag. Flag values are >= 1 if the instrument is working and < 1 if there is some problem with it, in which case that instrument's value should be ignored.

A sample from the full data file readings.txt, which is also used in the Text processing/1 task, follows:

Data is no longer available at that link. Zipped mirror available here

1991-03-30	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1
1991-03-31	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	20.000	1	20.000	1	20.000	1	35.000	1	50.000	1	60.000	1	40.000	1	30.000	1	30.000	1	30.000	1	25.000	1	20.000	1	20.000	1	20.000	1	20.000	1	20.000	1	35.000	1
1991-03-31	40.000	1	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2
1991-04-01	0.000	-2	13.000	1	16.000	1	21.000	1	24.000	1	22.000	1	20.000	1	18.000	1	29.000	1	44.000	1	50.000	1	43.000	1	38.000	1	27.000	1	27.000	1	24.000	1	23.000	1	18.000	1	12.000	1	13.000	1	14.000	1	15.000	1	13.000	1	10.000	1
1991-04-02	8.000	1	9.000	1	11.000	1	12.000	1	12.000	1	12.000	1	27.000	1	26.000	1	27.000	1	33.000	1	32.000	1	31.000	1	29.000	1	31.000	1	25.000	1	25.000	1	24.000	1	21.000	1	17.000	1	14.000	1	15.000	1	12.000	1	12.000	1	10.000	1
1991-04-03	10.000	1	9.000	1	10.000	1	10.000	1	9.000	1	10.000	1	15.000	1	24.000	1	28.000	1	24.000	1	18.000	1	14.000	1	12.000	1	13.000	1	14.000	1	15.000	1	14.000	1	15.000	1	13.000	1	13.000	1	13.000	1	12.000	1	10.000	1	10.000	1

Task

 Confirm the general field format of the file.
 Identify any DATESTAMPs that are duplicated.
 Report the number of records that have good readings for all instruments.


#load "str.cma"
open Str
 
let strip_cr str =
  let last = pred (String.length str) in
  if str.[last] <> '\r' then str else String.sub str 0 last
 
let map_records =
  let rec aux acc = function
    | value::flag::tail ->
        let e = (float_of_string value, int_of_string flag) in
        aux (e::acc) tail
    | [_] -> invalid_arg "invalid data"
    | [] -> List.rev acc
  in
  aux [] ;;
 
let duplicated_dates =
  let same_date (d1,_) (d2,_) = (d1 = d2) in
  let date (d,_) = d in
  let rec aux acc = function
    | a::b::tl when same_date a b ->
        aux (date a::acc) tl
    | _::tl ->
        aux acc tl
    | [] ->
        List.rev acc
  in
  aux [] ;;
 
let record_ok (_,record) =
  let is_ok (_,v) = v >= 1 in
  let sum_ok =
    List.fold_left (fun sum this ->
      if is_ok this then succ sum else sum) 0 record
  in
  sum_ok = 24
 
let num_good_records =
  List.fold_left  (fun sum record ->
    if record_ok record then succ sum else sum) 0 ;;
 
let parse_line line =
  let li = split (regexp "[ \t]+") line in
  let records = map_records (List.tl li)
  and date = List.hd li in
  (date, records)
 
let () =
  let ic = open_in "readings.txt" in
  let rec read_loop acc =
    let line_opt = try Some (strip_cr (input_line ic))
                   with End_of_file -> None
    in
    match line_opt with
      None -> close_in ic; List.rev acc
    | Some line -> read_loop (parse_line line :: acc)
  in
  let inputs = read_loop [] in
 
  Printf.printf "%d total lines\n" (List.length inputs);
 
  Printf.printf "duplicated dates:\n";
  let dups = duplicated_dates inputs in
  List.iter print_endline dups;
 
  Printf.printf "number of good records: %d\n" (num_good_records inputs);
;;