The following task concerns data that came from a pollution monitoring station with twenty-four instruments monitoring twenty-four aspects of pollution in the air. Periodically a record is added to the file, each record being a line of 49 fields separated by white-space, which can be one or more space or tab characters.

The fields (from the left) are:

 DATESTAMP [ VALUEn FLAGn ] * 24

i.e. a datestamp followed by twenty-four repetitions of a floating-point instrument value and that instrument's associated integer flag. Flag values are >= 1 if the instrument is working and < 1 if there is some problem with it, in which case that instrument's value should be ignored.

A sample from the full data file readings.txt, which is also used in the Text processing/1 task, follows:

Data is no longer available at that link. Zipped mirror available here

1991-03-30	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1
1991-03-31	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	10.000	1	20.000	1	20.000	1	20.000	1	35.000	1	50.000	1	60.000	1	40.000	1	30.000	1	30.000	1	30.000	1	25.000	1	20.000	1	20.000	1	20.000	1	20.000	1	20.000	1	35.000	1
1991-03-31	40.000	1	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2	0.000	-2
1991-04-01	0.000	-2	13.000	1	16.000	1	21.000	1	24.000	1	22.000	1	20.000	1	18.000	1	29.000	1	44.000	1	50.000	1	43.000	1	38.000	1	27.000	1	27.000	1	24.000	1	23.000	1	18.000	1	12.000	1	13.000	1	14.000	1	15.000	1	13.000	1	10.000	1
1991-04-02	8.000	1	9.000	1	11.000	1	12.000	1	12.000	1	12.000	1	27.000	1	26.000	1	27.000	1	33.000	1	32.000	1	31.000	1	29.000	1	31.000	1	25.000	1	25.000	1	24.000	1	21.000	1	17.000	1	14.000	1	15.000	1	12.000	1	12.000	1	10.000	1
1991-04-03	10.000	1	9.000	1	10.000	1	10.000	1	9.000	1	10.000	1	15.000	1	24.000	1	28.000	1	24.000	1	18.000	1	14.000	1	12.000	1	13.000	1	14.000	1	15.000	1	14.000	1	15.000	1	13.000	1	13.000	1	13.000	1	12.000	1	10.000	1	10.000	1

Task

 Confirm the general field format of the file.
 Identify any DATESTAMPs that are duplicated.
 Report the number of records that have good readings for all instruments.


import strutils, tables
 
const NumFields = 49
const DateField = 0
const FlagGoodValue = 1
 
var badRecords: int       # Number of records that have invalid formatted values.
var totalRecords: int     # Total number of records in the file.
var badInstruments: int   # Total number of records that have at least one instrument showing error.
var seenDates: Table[string, bool]  # Table to keep track of what dates we have seen.
 
proc checkFloats(floats: seq[string]): bool =
  ## Ensure we can parse all records as floats (except the date stamp).
  for index in 1..<NumFields:
    try:
      # We're assuming all instrument flags are floats not integers.
      discard parseFloat(floats[index])
    except ValueError:
      return false
  true
 
proc areAllFlagsOk(instruments: seq[string]): bool =
  ## Ensure that all sensor flags are ok.
 
  # Flags start at index 2, and occur every 2 fields.
  for index in countup(2, NumFields, 2):
    # We're assuming all instrument flags are floats not integers
    var flag = parseFloat(instruments[index])
    if flag < FlagGoodValue: return false
  true
 
 
# Note: we're not checking the format of the date stamp.
 
# Main.
 
var currentLine = 0
for line in "readings.txt".lines:
  currentLine.inc
  if line.len == 0: continue    # Empty lines don't count as records.
 
  var tokens = line.split({' ', '\t'})
  totalRecords.inc
 
  if tokens.len != NumFields:
    badRecords.inc
    continue
 
  if not checkFloats(tokens):
    badRecords.inc
    continue
 
  if not areAllFlagsOk(tokens):
    badInstruments.inc
 
  if seenDates.hasKeyOrPut(tokens[DateField], true):
    echo tokens[DateField], " duplicated on line ", currentLine
 
let goodRecords = totalRecords - badRecords
let goodInstruments = goodRecords - badInstruments
 
echo "Total Records: ", totalRecords
echo "Records with wrong format: ", badRecords
echo "Records where all instruments were OK: ", goodInstruments