Merge branch 'develop' of ssh://software.ecmwf.int:7999/eccodes/eccodes into develop

CMakeLists.txt

@@ -336,12 +336,12 @@ ecbuild_install_project( NAME ${CMAKE_PROJECT_NAME} )
 
 ecbuild_print_summary()
 
-message("")
+message(STATUS "")
-message("   +---------------------------+")
+message(STATUS "   +---------------------------+")
-message("   |  ecCodes version ${ECCODES_VERSION}   |")
+message(STATUS "   |  ecCodes version ${ECCODES_VERSION}   |")
-message("   |  Configuration completed  |")
+message(STATUS "   |  Configuration completed  |")
-message("   +---------------------------+")
+message(STATUS "   +---------------------------+")
-message("")
+message(STATUS "")
-message("   You can now do 'make' to compile the package, 'ctest' to test it and 'make install' to install it afterwards.")
+message(STATUS "   You can now do 'make' to compile the package, 'ctest' to test it and 'make install' to install it afterwards.")
-message("")
+message(STATUS "")
-message("")
+message(STATUS "")

data/grib_data_files.txt

@@ -84,3 +84,4 @@ timeRangeIndicator_10.grib
 timeRangeIndicator_5.grib
 tp_ecmwf.grib
 v.grib2
+msl.octa.glob.grib1

definitions/grib2/products_uerra.def

@@ -59,20 +59,6 @@ concept marsType {
      typeOfProcessedData = 2;
    }
 
-   cf = {
-     typeOfProcessedData = 3;
-   }
-   "10" = {
-     typeOfProcessedData = 3;
-   }
-
-   pf = {
-     typeOfProcessedData = 4;
-   }
-   "11" = {
-     typeOfProcessedData = 4;
-   }
-
    an = {
      typeOfProcessedData = 0;
    }
@@ -86,26 +72,23 @@ concept marsType {
 }
 
 # See GRIB-205 re no_copy
+# Cannot use typeOfProcessedData for stream. See GRIB-911
 concept marsStream {
 
     oper = {
-      typeOfProcessedData = 0;
+      productDefinitionTemplateNumber = 8;
     }
 
     oper = {
-      typeOfProcessedData = 2;
+      productDefinitionTemplateNumber = 0;
     }
 
     enda = {
-      typeOfProcessedData = 3;
+      productDefinitionTemplateNumber = 11;
     }
 
     enda = {
-      typeOfProcessedData = 4;
+      productDefinitionTemplateNumber = 1;
-    }
-
-    enda = {
-      typeOfProcessedData = 8;
     }
 
     "default" = {

examples/C/grib_index.c

@@ -27,7 +27,6 @@ int main(int argc,char* argv[])
     grib_index* index=NULL;
     grib_handle* h=NULL;
     char* infile=NULL;
-    char* outfile=NULL;
     long *steps,*levels,*numbers; /* arrays */
     char** shortName=NULL;
     int i,j,k,l;
@@ -39,7 +38,6 @@ int main(int argc,char* argv[])
 
     if (argc != 2) usage(argv[0]);
     infile=argv[1];
-    outfile=argv[2];
 
     printf("indexing...\n");
 

src/grib_parse_utils.c

@@ -551,6 +551,8 @@ int grib_yyerror(const char* msg)
 {
     grib_context_log(grib_parser_context, GRIB_LOG_ERROR,
             "grib_parser: %s at line %d of %s", msg, grib_yylineno + 1,parse_file);
+    grib_context_log(grib_parser_context, GRIB_LOG_ERROR,
+            "ecCodes Version: %s", ECCODES_VERSION_STR);
     error = 1;
     return 1;
 }

tests/CMakeLists.txt

@@ -36,6 +36,7 @@ list( APPEND tests1
     calendar
     unit_tests
     md5
+    uerra
 )
 list( APPEND tests2
     bufr_dump
@@ -62,6 +63,7 @@ list( APPEND tests2
     multi
     budg
     gridType
+    octahedral
     global
     concept
     decimalPrecision

tests/Makefile.am

@@ -24,6 +24,7 @@ TESTS = definitions.sh \
         multi.sh \
         budg.sh \
         gridType.sh \
+        octahedral.sh \
         global.sh \
         concept.sh \
         decimalPrecision.sh \
@@ -45,6 +46,7 @@ TESTS = definitions.sh \
         julian.sh \
         statistics.sh \
         tigge.sh \
+        uerra.sh \
         tigge_conversions.sh \
         read_any.sh \
         padding.sh \

tests/calendar.sh

@@ -18,27 +18,27 @@ sample=$ECCODES_SAMPLES_PATH/GRIB2.tmpl
 
 
 ${tools_dir}grib_set -s tablesVersion=13,setCalendarId=1,typeOfCalendar=0 $sample $temp1
-check_key_equals $temp1 calendarIdPresent 1
+grib_check_key_equals $temp1 calendarIdPresent 1
-check_key_equals $temp1 section1Length    24
+grib_check_key_equals $temp1 section1Length    24
-check_key_equals $temp1 calendarIdentificationTemplateNumber 0
+grib_check_key_equals $temp1 calendarIdentificationTemplateNumber 0
-check_key_equals $temp1 typeOfCalendar    0
+grib_check_key_equals $temp1 typeOfCalendar    0
 
 ${tools_dir}grib_set -s tablesVersion=13,setCalendarId=1,calendarIdentificationTemplateNumber=1 $sample $temp1
-check_key_equals $temp1 calendarIdPresent 1
+grib_check_key_equals $temp1 calendarIdPresent 1
-check_key_equals $temp1 section1Length    25
+grib_check_key_equals $temp1 section1Length    25
-check_key_equals $temp1 calendarIdentificationTemplateNumber 1
+grib_check_key_equals $temp1 calendarIdentificationTemplateNumber 1
-check_key_equals $temp1 numberOfTensOfThousandsOfYearsOfOffset MISSING
+grib_check_key_equals $temp1 numberOfTensOfThousandsOfYearsOfOffset MISSING
 
 ${tools_dir}grib_set -s tablesVersion=13,setCalendarId=1,calendarIdentificationTemplateNumber=2 $sample $temp1
-check_key_equals $temp1 calendarIdPresent 1
+grib_check_key_equals $temp1 calendarIdPresent 1
-check_key_equals $temp1 section1Length    26
+grib_check_key_equals $temp1 section1Length    26
-check_key_equals $temp1 calendarIdentificationTemplateNumber 2
+grib_check_key_equals $temp1 calendarIdentificationTemplateNumber 2
-check_key_equals $temp1 typeOfCalendar    255
+grib_check_key_equals $temp1 typeOfCalendar    255
-check_key_equals $temp1 numberOfTensOfThousandsOfYearsOfOffset MISSING
+grib_check_key_equals $temp1 numberOfTensOfThousandsOfYearsOfOffset MISSING
 
 # Remove the calendar section
 ${tools_dir}grib_set -s deleteCalendarId=1 $temp1 $temp2
-check_key_equals $temp2 section1Length    21
+grib_check_key_equals $temp2 section1Length    21
-check_key_equals $temp2 calendarIdPresent 0
+grib_check_key_equals $temp2 calendarIdPresent 0
 
 rm -f $temp1 $temp2

tests/check_gaussian_grids.sh

@@ -13,5 +13,5 @@
 samples_dir=$ECCODES_SAMPLES_PATH
 for gg in ${samples_dir}/reduced_gg_* ${samples_dir}/regular_gg_*; do
    ${tools_dir}grib_check_gaussian_grid $gg >/dev/null
-   check_key_equals $gg "global" 1
+   grib_check_key_equals $gg "global" 1
 done

tests/global.sh

@@ -42,10 +42,10 @@ files_subarea="
 
 for gg in $files_global; do
     filepath=${data_dir}/$gg
-    check_key_equals $filepath "global" 1
+    grib_check_key_equals $filepath "global" 1
 done
 
 for gg in $files_subarea; do
     filepath=${data_dir}/$gg
-    check_key_equals $filepath "global" 0
+    grib_check_key_equals $filepath "global" 0
 done

tests/index.c

@@ -19,7 +19,6 @@ int main(int argc,char* argv[])
     grib_index* index=NULL;
     grib_handle* h=NULL;
     char* infile=NULL;
-    char* outfile=NULL;
     long *steps,*levels,*numbers; /* arrays */
     char** shortName=NULL;
     int i,j,k,l;
@@ -31,7 +30,6 @@ int main(int argc,char* argv[])
 
     if (argc != 2) usage(argv[0]);
     infile=argv[1];
-    outfile=argv[2];
 
     printf("indexing...\n");
 

tests/neg_fctime.sh

@@ -28,38 +28,38 @@ done
 ############################
 
 # check input time and date
-check_key_equals $input 'dataDate,dataTime' '20080206 1200'
+grib_check_key_equals $input 'dataDate,dataTime' '20080206 1200'
 
 # Negative step in same day
 # --------------------------
 ${tools_dir}grib_set -s forecastTime=-2 $input $temp
-check_key_equals $temp 'validityDate,validityTime' '20080206 1000'
+grib_check_key_equals $temp 'validityDate,validityTime' '20080206 1000'
 
 ${tools_dir}grib_set -s forecastTime=-11 $input $temp
-check_key_equals $temp 'validityDate,validityTime' '20080206 100'
+grib_check_key_equals $temp 'validityDate,validityTime' '20080206 100'
 
 ${tools_dir}grib_set -s forecastTime=-12 $input $temp
-check_key_equals $temp 'validityDate,validityTime' '20080206 0'
+grib_check_key_equals $temp 'validityDate,validityTime' '20080206 0'
 
 
 # Negative step previous day
 # --------------------------
 ${tools_dir}grib_set -s forecastTime=-13 $input $temp
-check_key_equals $temp 'validityDate,validityTime' '20080205 2300'
+grib_check_key_equals $temp 'validityDate,validityTime' '20080205 2300'
 
 ${tools_dir}grib_set -s forecastTime=-24 $input $temp
-check_key_equals $temp 'validityDate,validityTime' '20080205 1200'
+grib_check_key_equals $temp 'validityDate,validityTime' '20080205 1200'
 
 ${tools_dir}grib_set -s forecastTime=-33 $input $temp
-check_key_equals $temp 'validityDate,validityTime' '20080205 300'
+grib_check_key_equals $temp 'validityDate,validityTime' '20080205 300'
 
 # Negative step further back
 # --------------------------
 ${tools_dir}grib_set -s forecastTime=-38 $input $temp
-check_key_equals $temp 'validityDate,validityTime' '20080204 2200'
+grib_check_key_equals $temp 'validityDate,validityTime' '20080204 2200'
 
 ${tools_dir}grib_set -s forecastTime=-185 $input $temp
-check_key_equals $temp 'validityDate,validityTime' '20080129 1900'
+grib_check_key_equals $temp 'validityDate,validityTime' '20080129 1900'
 
 
 # Clean up

tests/octahedral.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+# Copyright 2005-2016 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# 
+# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
+# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
+#
+
+. ./include.sh
+
+REDIRECT=/dev/null
+
+this_test="octa_grid.test"
+temp=temp.$this_test
+
+input=${data_dir}/msl.octa.glob.grib1
+
+# Check 'global' key
+grib_check_key_equals $input "global,isOctahedral" "1 1"
+
+# Check numberOfDataPoints
+grib_check_key_equals $input "numberOfDataPoints,numberOfCodedValues" "6599680 6599680"
+
+# Iterator
+${tools_dir}grib_get_data $input > $temp
+numlines=`wc -l $temp | awk '{print $1}'`
+[ "$numlines" = "6599681" ]     # 1 + numberOfDataPoints
+
+rm -f $temp

tests/read_index.c

@@ -15,12 +15,10 @@ void usage(char* prog) {
   exit(1);
 }
 
-int main(int argc,char* argv[]) {
+int main(int argc,char* argv[])
-  
+{
   grib_index* index=NULL;
   grib_handle* h=NULL;
-  char* infile=NULL;
-  char* outfile=NULL;
   long *step,*level,*number;
   char** shortName=NULL;
   int i,j,k,l;
@@ -31,8 +29,6 @@ int main(int argc,char* argv[]) {
   int ret=0,count=0;
 
   if (argc != 2) usage(argv[0]);
-  infile=argv[1];
-  outfile=argv[2];
 
   printf("indexing...\n");
 

tests/uerra.sh

@@ -0,0 +1,69 @@
+#!/bin/sh
+# Copyright 2005-2016 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# 
+# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
+# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
+#
+
+. ./include.sh
+
+REDIRECT=/dev/null
+
+#set -eax
+grib2_sample=$ECCODES_SAMPLES_PATH/GRIB2.tmpl
+this_test=ecCodes-uerra
+tempSample=tempSample.${this_test}.grib2
+temp1=temp1.${this_test}.grib2
+temp2=temp2.${this_test}.grib2
+
+test_stream_and_type()
+{
+    utype=$1
+
+    if [ "$utype" = "oper" ]; then
+        pspd=8
+    elif [ "$utype" = "test" ]; then
+        pspd=9
+    fi
+
+    ${tools_dir}grib_set -s tablesVersion=15,productionStatusOfProcessedData=$pspd \
+                           $grib2_sample $tempSample
+
+    grib_check_key_equals $tempSample 'stream,mars.stream' 'oper oper'
+    
+    # Check mars model is not set
+    model=`${tools_dir}grib_get -f -p mars.model $tempSample`
+    [ "$model" = "not_found" ]
+
+    ${tools_dir}grib_set -s typeOfProcessedData=0 $tempSample $temp1
+    grib_check_key_equals $temp1 'mars.type' 'an'
+    ${tools_dir}grib_set -s typeOfProcessedData=2 $tempSample $temp1
+    grib_check_key_equals $temp1 'mars.type' 'fc'
+
+    ${tools_dir}grib_set -s marsStream=enda $tempSample $temp1    # ENSEMBLE
+    grib_check_key_exists $temp1 'perturbationNumber'
+
+    # All combinations
+    for mt in an fc; do
+        for ms in oper enda; do
+            echo "Testing stream=$ms type=$mt ..."
+            ${tools_dir}grib_set -s marsStream=$ms,marsType=$mt $tempSample $temp1
+            grib_check_key_equals $temp1 'mars.stream,mars.type' "$ms $mt"
+
+            # Param 228228 (accum) and 167 (instant)
+            ${tools_dir}grib_set -s stepType=accum,paramId=228228 $temp1 $temp2
+            grib_check_key_equals $temp2 'stream,type' "$ms $mt"
+
+            ${tools_dir}grib_set -s paramId=167                   $temp1 $temp2
+            grib_check_key_equals $temp2 'stream,type' "$ms $mt"
+        done
+    done
+}
+
+test_stream_and_type 'oper'
+test_stream_and_type 'test'
+
+rm -f $temp1 $temp2 $tempSample

tests/utils.sh

@@ -15,7 +15,7 @@ set -ea
 # Check the key(s) in the given grib file have the expected value(s)
 # Assumes the file has only ONE message otherwise output goes on to
 # several lines
-check_key_equals()
+grib_check_key_equals()
 {
    file=$1
    key=$2
@@ -24,3 +24,10 @@ check_key_equals()
    [ "$result" = "$val" ]
 }
 
+grib_check_key_exists()
+{
+   file=$1
+   key=$2
+   # grib_get will fail if the key is not found
+   $tools_dir/grib_get -p $key $file >/dev/null
+}

tools/bufr_compare.c

@@ -558,7 +558,6 @@ static int compare_values(grib_runtime_options* options,grib_handle* h1,grib_han
     unsigned char *uval1 = NULL,*uval2 = NULL;
     double *dval1 = NULL, *dval2 = NULL;
     long *lval1 = NULL, *lval2 = NULL;
-    int failed=0;
     double maxdiff=0;
     double packingError1=0,packingError2=0;
     double value_tolerance=0;
@@ -861,7 +860,6 @@ static int compare_values(grib_runtime_options* options,grib_handle* h1,grib_han
             if (verbose) printf("  (%d values) tolerance=%g\n",(int)len1,value_tolerance);
             for(i = 0; i < len1; i++) {
                 if((diff=compare_double(pv1++,pv2++,&value_tolerance))!=0) {
-                    failed=1;
                     countdiff++;
                     if (maxdiff < diff) {maxdiff=diff;imaxdiff=i;}
                     err1 = GRIB_VALUE_MISMATCH;

tools/metar_ls.c

@@ -80,7 +80,7 @@ int grib_tool_init(grib_runtime_options* options) {
   size_t size=4;
   int ret=0;
   double min=0,max=0;
-  int i=0,idx=0;
+  int i=0;
   char* p=NULL;
   if (grib_options_on("j")) {
 	options->verbose=0;
@@ -142,7 +142,7 @@ int grib_tool_init(grib_runtime_options* options) {
     options->latlon_idx=-1;
     max=options->distances[0];
     for (i=0;i<4;i++)
-      if (max<options->distances[i]) {max=options->distances[i];idx=i;}
+      if (max<options->distances[i]) {max=options->distances[i];}
     min=max;
     for (i=0;i<4;i++) {
       if ((min >= options->distances[i]) && (options->mask_values[i] >= 0.5)) {