#!/bin/sh
#
# $Id: t_ropp2ropp.sh 5117 2016-12-15 15:04:41Z idculv $
#
#****s* Tools/t_ropp2ropp *
#
# NAME
#    t_ropp2ropp - Test the ropp2ropp program.
#
# SYNOPSIS
#    t_ropp2ropp <options>
#
# DESCRIPTION
#    This shell script tests some (but not all) of the ROPP functionality by
#    performing several conversions using the ropp2ropp program. It uses
#    example data located in the ../data directory, and creates .cdl
#    representation of the created netCDF files. The latter (or text files
#    generated by ropp2ropp) can then be compared with master versions of
#    these files (also created by this script) using standard Unix tools like
#    diff.
#
# OPTIONS
#    -r       Run range check tests
#    -n       Run netCDF -> netCDF singlefile tests
#    -m       Run netCDF -> netCDF multifile  tests
#    -t       Run netCDF -> netCDF 2D file test
#
# USES
#   test2ropp
#   ropp2ropp
#   ncdump
#
# FILES
#   The tests use reference files from the ropp_io/data directory:
#      ../data/ropp_test.nc      -  ROPP netCDF master file (single profile)
#      ../data/ropp_testm.nc     -  ROPP netCDF multifile (>1 profile)
#   It is essential that these reference files are self-consistent
#   (e.g. the multifile was generated from the master file (2x)).
#
# DESCRIPTION
#    The following tests are run:
#
#    -r
#      1a. MISSING data netCDF  -> MISSING netCDF
#      1b. INVALID data netCDF  -> MISSING netCDF
#      2.  VALID   data netCDF  -> VALID   netCDF
#
#    -n
#      3. netCDF singlefile     ->  netCDF singlefile
#
#    -m
#      4. netCDF singlefile     ->  netCDF multifile
#      5. netCDF multifile      ->  netCDF singlefiles
#
#    -t
#      6. netCDF singlefile (2D data) -> netCDF singlefile (2D)
#
#    Generated files should ideally be identical with the reference files
#    (or dumps from them) Note, however, that .cdl files contain the name
#    of the data file in the first line, so that a simple textual comparison
#    will give at least one difference between these. Additional differences
#    might occur because of rounding or in the file processing date/times.
#
# EXAMPLE
#    To run all tests, do
#
#     > t_ropp2ropp -r -n -m -t
#
# REFERENCES
#   ROPP User Guide. Ref: SAF/ROM/METO/UG/ROPP/002
#
# AUTHOR
#   Met Office, Exeter, UK.
#   Any comments on this software should be given via the ROM SAF
#   Helpdesk at http://www.romsaf.org
#
# COPYRIGHT
#   (c) EUMETSAT. All rights reserved.
#   For further details please refer to the file COPYRIGHT
#   which you should have received as part of this distribution.
#
#****

#-------------------------------------------------------------------------------
# 1. Usage help
#-------------------------------------------------------------------------------

usage() {
      echo "Usage: $0 <options>"
      echo "Possible options:"
      echo "       -r    Run range check       tests"
      echo "       -n    Run netCDF singlefile tests"
      echo "       -m    Run netCDF multifile  tests"
      echo "       -t    Run netCDF 2d singlefile test"
      echo "       -h    This help"
      exit
}
#
#-------------------------------------------------------------------------------
# 2. Command line arguments
#-------------------------------------------------------------------------------
#
if [ $# -lt 1 ]; then
   usage
fi

while getopts hrtnm opt; do
   case $opt in
     h)  usage ;;
     r)  DO_ranchk=1 ;;
     n)  DO_netcdf=1 ;;
     m)  DO_netcdf_m=1 ;;
     t)  DO_netcdf_d=1 ;;
     *)  echo "Unknown option: $opt." ; usage ;;
   esac
done
shift `expr $OPTIND - 1`

bindir=../tools

unset NCDUMP
if which ncdump >/dev/null 2>&1; then
  export NCDUMP=`which ncdump`
  echo \$NCDUMP = $NCDUMP
else
  if which $ROPP_ROOT/*/bin/ncdump >/dev/null 2>&1; then
    export NCDUMP=`ls -ut $ROPP_ROOT/*/bin/ncdump |head -1`
    echo \$NCDUMP = $NCDUMP
  else
    echo 'ncdump not found in $PATH or under $ROPP_ROOT/*/bin'
    echo 'Will be unable to carry out comparisons of netCDF data'
  fi
fi

#
#-------------------------------------------------------------------------------
# 3. Ensure any old files are wiped
#-------------------------------------------------------------------------------
#
rm -f ropp_test_*.* oc_*.nc > /dev/null 2>&1
#
ntest=0
nfail=0

if [ $DO_ranchk ]; then
#
#-------------------------------------------------------------------------------
# 4. Range check 1: (a) generate a file containing only missing data values;
#    run through ropp2ropp; output should result in only a header block
#    (with all values except processing time set missing) and no Level 1/2
#    data at all (all Npoints values set zero)
#    (b) repeat for a file of random, out-of-range (invalid, but not missing)
#    values; output should be same as for (a) apart from a slightly later
#    processing time in the header.
#-------------------------------------------------------------------------------
#
  echo
  echo "Test 1: Testing range checks - missing & invalid data --> missing data"
  echo "======================================================================"
  ((ntest=ntest+1))
  $bindir/test2ropp MISSING -o ropp_test_1m.nc -n 100
  if [ ! -f ropp_test_1m.nc ]; then
    echo
    echo "*** Failed to generate MISSING test file (in): FAIL"
    exit 1
  fi

  $bindir/ropp2ropp ropp_test_1m.nc -o ropp_test_1m.nc -u
  if [ ! -f ropp_test_1m.nc ]; then
    echo
    echo "*** Failed to generate MISSING test file (out): FAIL"
    exit 1
  fi
  $NCDUMP -p 5,5 ropp_test_1m.nc > ropp_test_1m.cdl

  $bindir/test2ropp INVALID -o ropp_test_1i.nc -n 100
  if [ ! -f ropp_test_1m.nc ]; then
    echo
    echo "*** Failed to generate INVALID test file (in): FAIL"
    exit 1
  fi

  $bindir/ropp2ropp ropp_test_1i.nc -o ropp_test_1i.nc -u
  if [ ! -f ropp_test_1i.nc ]; then
    echo
    echo "*** Failed to generate INVALID test file (out): FAIL"
    exit 1
  fi
  $NCDUMP -p 5,5 ropp_test_1i.nc > ropp_test_1i.cdl

  echo
  echo "To check results, compare ropp_test_1m.cdl and ropp_test_1i.cdl"
  echo "(Ignore differences in file names and processing_date)"
  echo
  grep -v "^netcdf .*{" ropp_test_1m.cdl | grep -v "processing_date" > x.cdl
  grep -v "^netcdf .*{" ropp_test_1i.cdl | grep -v "processing_date" > y.cdl
  if cmp -s x.cdl y.cdl; then
    echo
    echo "****************************************"
    echo "*** Test 1. Range INVALID test: PASS ***"
    echo "****************************************"
    echo
  else
    echo
    echo "****************************************"
    echo "*** Test 1. Range INVALID test: FAIL ***"
    echo "****************************************"
    echo
    diff -c ropp_test_1m.cdl ropp_test_1i.cdl > ropp_test_1i.dif
    echo "*** Differences found; please review ropp_test_1i.dif to check significance"
    echo
    ((nfail=nfail+1))
  fi
  rm -f x.cdl y.cdl
#
#-------------------------------------------------------------------------------
# 5. Range check 2: generate a file containing random, but valid, values;
#    run through ropp2ropp; output should be the same as the input
#-------------------------------------------------------------------------------
#
  echo
  echo "Test 2: Testing range checks - valid data --> valid data"
  echo "========================================================"
  ((ntest=ntest+1))
  $bindir/test2ropp VALID -o ropp_test_2v.nc -n 100
  if [ ! -f ropp_test_2v.nc ]; then
    echo
    echo "*** Failed to generate VALID test file (in): FAIL"
    exit
  fi
  $NCDUMP -p 5,5 ropp_test_2v.nc > ropp_test_2v.cdl

  $bindir/ropp2ropp ropp_test_2v.nc -o ropp_test_2o.nc -u
  if [ ! -f ropp_test_2o.nc ]; then
    echo
    echo "*** Failed to generate VALID test file (out): FAIL"
    exit
  fi
  $NCDUMP -p 5,5 ropp_test_2o.nc > ropp_test_2o.cdl

  echo
  echo "To check results, compare ropp_test_2v.cdl and ropp_test_2o.cdl"
  echo "(Ignore differences in file names and processing_date)"
  echo
  grep -v "^netcdf .*{" ropp_test_2v.cdl | grep -v "processing_date" > x.cdl
  grep -v "^netcdf .*{" ropp_test_2o.cdl | grep -v "processing_date" > y.cdl
  sed s/" \[v.*\]"//g x.cdl > x1.cdl; mv x1.cdl x.cdl
  sed s/" \[v.*\]"//g y.cdl > y1.cdl; mv y1.cdl y.cdl
  if cmp -s x.cdl y.cdl; then
    echo
    echo "**************************************"
    echo "*** Test 2. Range VALID test: PASS ***"
    echo "**************************************"
    echo
  else
    echo
    echo "**************************************"
    echo "*** Test 2. Range VALID test: FAIL ***"
    echo "**************************************"
    echo
    diff -c ropp_test_2v.cdl ropp_test_2o.cdl > ropp_test_2o.dif
    echo "*** Differences found; please review ropp_test_2o.dif to check significance"
    echo
    ((nfail=nfail+1))
  fi
  rm -f x.cdl y.cdl
fi
#
#-------------------------------------------------------------------------------
# 6. Read the reference ROPP file (netCDF) and 'convert' to netCDF
#-------------------------------------------------------------------------------
#
if [ $DO_netcdf ]; then
  echo
  echo "Test 3: Testing ROPP (netCDF) --> ROPP (netCDF)"
  echo "==============================================="
  ((ntest=ntest+1))
  $NCDUMP -p 5,5 ../data/ropp_test.nc > ropp_test_4n.cdl
  $bindir/ropp2ropp ../data/ropp_test.nc -o ropp_test_4o.nc
  if [ ! -f ropp_test_4o.nc ]; then
    echo
    echo "*** Failed to generate netCDF test file: FAIL"
    exit
  fi
  $NCDUMP -p 5,5 ropp_test_4o.nc > ropp_test_4o.cdl

  echo
  echo "To check results, compare ropp_test_4n.cdl and ropp_test_4o.cdl"
  echo "(Ignore differences in file names & thin_method)"
  echo
  grep -v "^netcdf .*{" ropp_test_4n.cdl | grep -v "thin_method" > x.cdl
  grep -v "^netcdf .*{" ropp_test_4o.cdl | grep -v "thin_method" > y.cdl
  if cmp -s x.cdl y.cdl; then
    echo
    echo "*******************************************"
    echo "*** Test 3. ROPP netCDF -> netCDF: PASS ***"
    echo "*******************************************"
    echo
  else
    echo
    echo "*******************************************"
    echo "*** Test 3. ROPP netCDF -> netCDF: FAIL ***"
    echo "*******************************************"
    echo
    diff -c ropp_test_4n.cdl ropp_test_4o.cdl > ropp_test_4o.dif
    echo "*** Differences found; please review ropp_test_4o.dif to check significance"
    echo
    ((nfail=nfail+1))
  fi
  rm -f x.cdl y.cdl
fi
#
#-------------------------------------------------------------------------------
# 7. Read ROPP singlefiles (netCDF) and convert to one multifile
#-------------------------------------------------------------------------------
#
if [ $DO_netcdf_m ]; then
  echo
  echo "Test 4: Testing ROPP singlefiles (netCDF) --> ROPP multifile (netCDF)"
  echo "====================================================================="
  ((ntest=ntest+1))
  $NCDUMP -p 5,5 ../data/ropp_testm.nc > ropp_test_5m.cdl
  $bindir/ropp2ropp ../data/ropp_test.nc \
                    ../data/ropp_test.nc \
                     -m -o ropp_test_5o.nc
  if [ ! -f ropp_test_5o.nc ]; then
    echo
    echo "*** Failed to generate netCDF multi-file: FAIL"
    exit
  fi
  $NCDUMP -p 5,5 ropp_test_5o.nc > ropp_test_5o.cdl

  echo
  echo "To check results, compare ropp_test_5m.cdl and ropp_test_5o.cdl"
  echo "(Ignore difference in file names & thin_method))"
  echo
  grep -v "^netcdf .*{" ropp_test_5m.cdl | grep -v "thin_method" > x.cdl
  grep -v "^netcdf .*{" ropp_test_5o.cdl | grep -v "thin_method" > y.cdl
  if cmp -s x.cdl y.cdl; then
    echo
    echo "**************************************************"
    echo "*** Test 4. ROPP singlefile -> multifile: PASS ***"
    echo "**************************************************"
    echo
  else
    echo
    echo "**************************************************"
    echo "*** Test 4. ROPP singlefile -> multifile: FAIL ***"
    echo "**************************************************"
    echo
    diff -c ropp_test_5m.cdl ropp_test_5o.cdl > ropp_test_5o.dif
    echo "*** Differences found; please review ropp_test_5o.dif to check significance"
    echo
    ((nfail=nfail+1))
  fi
  rm -f x.cdl y.cdl
#
#-------------------------------------------------------------------------------
# 8. Read ROPP multifile (netCDF) and convert to individual netCDFs
#    NB: only one output singlefile; it will be overwritten multiple
#    times!
#-------------------------------------------------------------------------------
#
  echo
  echo "Test 5: Testing ROPP multifile (netCDF) --> ROPP singlefiles (netCDF)"
  echo "====================================================================="
  ((ntest=ntest+1))
  $NCDUMP -p 5,5 ../data/ropp_test.nc > ropp_test_6m.cdl
  $bindir/ropp2ropp ropp_test_5o.nc -o ropp_test_6o.nc
  if [ ! -f ropp_test_6o.nc ]; then
    echo
    echo "*** Failed to generate netCDF single-file: FAIL"
    exit
  fi
  $NCDUMP -p 5,5 ropp_test_6o.nc > ropp_test_6o.cdl

  echo
  echo "To check results, compare ropp_test_6m.cdl and ropp_test_6o.cdl"
  echo "(Ignore differences in file names & thin_method)"
  echo
  grep -v "^netcdf .*{" ropp_test_6m.cdl | grep -v "thin_method" > x.cdl
  grep -v "^netcdf .*{" ropp_test_6o.cdl | grep -v "thin_method" > y.cdl
  if cmp -s x.cdl y.cdl; then
    echo
    echo "**************************************************"
    echo "*** Test 5. ROPP multifile -> singlefile: PASS ***"
    echo "**************************************************"
    echo
  else
    echo
    echo "**************************************************"
    echo "*** Test 5. ROPP multifile -> singlefile: FAIL ***"
    echo "**************************************************"
    echo
    diff -c ropp_test_6m.cdl ropp_test_6o.cdl > ropp_test_6o.dif
    echo "*** Differences found; please review ropp_test_6o.dif to check significance"
    ((nfail=nfail+1))
    echo
  fi
  rm -f x.cdl y.cdl
fi
#
#-------------------------------------------------------------------------------
# 9. Read the ROPP 2D file (netCDF) and convert to netCDF
#-------------------------------------------------------------------------------
#
if [ $DO_netcdf_d ]; then
  echo
  echo "Test 6: Testing ROPP 2D (netCDF) --> ROPP 2D (netCDF)"
  echo "====================================================="
  ((ntest=ntest+1))
  $NCDUMP -p 5,5 ../data/ropp_test2d.nc > ropp_test_7n.cdl
  $bindir/ropp2ropp ../data/ropp_test2d.nc -o ropp_test_7o.nc -t
  if [ ! -f ropp_test_7o.nc ]; then
    echo
    echo "*** Failed to generate netCDF 2d test file: FAIL"
    exit
  fi
  $NCDUMP -p 5,5 ropp_test_7o.nc > ropp_test_7o.cdl

  echo
  echo "To check results, compare ropp_test_7n.cdl and ropp_test_7o.cdl"
  echo "(Ignore differences in file names & thin_method)"
  echo
  grep -v "^netcdf .*{" ropp_test_7n.cdl | grep -v "thin_method" > x.cdl
  grep -v "^netcdf .*{" ropp_test_7o.cdl | grep -v "thin_method" > y.cdl
  if cmp -s x.cdl y.cdl; then
    echo
    echo "*************************************************"
    echo "*** Test 6. ROPP 2d netCDF -> 2d netCDF: PASS ***"
    echo "*************************************************"
    echo
  else
    echo
    echo "*************************************************"
    echo "*** Test 6. ROPP 2d netCDF -> 2d netCDF: FAIL ***"
    echo "*************************************************"
    echo
    diff -c ropp_test_7n.cdl ropp_test_7o.cdl > ropp_test_7o.dif
    echo "*** Differences found; please review ropp_test_7o.dif to check significance"
    echo
    ((nfail=nfail+1))
  fi
  rm -f x.cdl y.cdl
fi
#
#
#-------------------------------------------------------------------------------
# 10. Summary of results
#-------------------------------------------------------------------------------
#
((npass=ntest-nfail))
#echo "PASS: $npass  FAIL: $nfail of $ntest tests"  # This screws up the test folder
echo "PASS: $npass  NOT PASS: $nfail  TOTAL: $ntest tests"
echo
status=0
if [ $nfail -gt 0 ]; then status=1; fi
exit $status