From 0cc400b219f2349c2ee9456f91828a4cb40ca857 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Fri, 1 Jun 2018 17:25:33 +0100 Subject: [PATCH 1/6] EFAS: fix for mars.date and anoffset calculation from validity date --- definitions/grib2/local.98.41.def | 12 +++++------- definitions/mars/efas.anoffset.def | 16 ++++++++++++++++ definitions/mars/grib.efas.an.def | 13 +++++++------ definitions/mars/grib.efas.fu.def | 1 + definitions/mars/grib.efas.go.def | 13 +++++++------ 5 files changed, 36 insertions(+), 19 deletions(-) create mode 100644 definitions/mars/efas.anoffset.def diff --git a/definitions/grib2/local.98.41.def b/definitions/grib2/local.98.41.def index f015f6995..1c98a051b 100644 --- a/definitions/grib2/local.98.41.def +++ b/definitions/grib2/local.98.41.def @@ -27,14 +27,12 @@ meta timeOfForecast time (hourOfForecast,minuteOfForecast,secondOfForecast) : # Calculate the Julian number for the forecast date and time. # This will be a floating point number with units of 'day' meta julianForecastDay julian_day(dateOfForecast,hourOfForecast,minuteOfForecast,secondOfForecast): hidden; -# Calculate the difference between the forecast date and reference date -transient diffInDays = (julianForecastDay - julianDay) : hidden; # float -# Now convert this to hours. First convert to minutes then round up -transient diffInHours = (diffInDays * 1440 + 0.5)/60 : hidden; -meta _anoffset round(diffInHours, 10): dump,long_type; -transient anoffset = _anoffset; # needed to force anoffset to be integer -alias local.anoffset = anoffset; +# Note: The 'anoffset' key is the difference between the forecast date and the validity date, +# but we cannot calculate it here (in section 2) because the keys validityDate & validityTime +# are defined later in section 4. +# Therefore that calculation is done when we load the mars config files mars/grib.efas.*.def. +# See mars/efas.anoffset.def # ECC-662 unsigned[2] anoffsetFirst = missing(): dump, can_be_missing; diff --git a/definitions/mars/efas.anoffset.def b/definitions/mars/efas.anoffset.def new file mode 100644 index 000000000..e81de38af --- /dev/null +++ b/definitions/mars/efas.anoffset.def @@ -0,0 +1,16 @@ +# This is applicable to EFAS GRIB2 data which uses local definition 41 +# stream must be 'efas' + +if (localDefinitionNumber == 41) { + # Calculate the difference between the forecast date and the validity date + meta the_valTime scale(validityTime,oneConstant,hundred) : read_only, long_type; + meta julianValidDay julian_day(validityDate, the_valTime, zero, zero): hidden, no_copy; + + transient diffInDays = (julianForecastDay - julianValidDay) : hidden; # float + + # Now convert this to hours. First convert to minutes then round up + transient diffInHours = (diffInDays * 1440 + 0.5)/60 : hidden; + meta _anoffset round(diffInHours, 10): dump,long_type; + transient anoffset = _anoffset; # needed to force anoffset to be integer + alias local.anoffset = anoffset; +} diff --git a/definitions/mars/grib.efas.an.def b/definitions/mars/grib.efas.an.def index 10da90a03..8545afdef 100644 --- a/definitions/mars/grib.efas.an.def +++ b/definitions/mars/grib.efas.an.def @@ -2,16 +2,17 @@ alias mars.step = startStep; # See ECC-683 -#alias mars.date = dateOfForecast; -#alias mars.time = timeOfForecast; -meta valDate validity_date(dataDate,dataTime,endStep,stepUnits): no_copy; -meta valTime validity_time(dataDate,dataTime,endStep,stepUnits): no_copy; -alias mars.date = valDate; -alias mars.time = valTime; +alias mars.date = dateOfForecast; +alias mars.time = timeOfForecast; +#meta valDate validity_date(dataDate,dataTime,endStep,stepUnits): no_copy; +#meta valTime validity_time(dataDate,dataTime,endStep,stepUnits): no_copy; +#alias mars.date = valDate; +#alias mars.time = valTime; #meta efas_suite_name sprintf("%s_%s_%s", inputOriginatingCentre, efas_forecast, efas_post_proc) : no_copy; alias mars.origin = inputOriginatingCentre; +include "mars/efas.anoffset.def" alias mars.anoffset=anoffset; # We need this because 'efas_post_proc' is defined later diff --git a/definitions/mars/grib.efas.fu.def b/definitions/mars/grib.efas.fu.def index 6dea1033d..be1af1dbe 100644 --- a/definitions/mars/grib.efas.fu.def +++ b/definitions/mars/grib.efas.fu.def @@ -7,6 +7,7 @@ alias mars.time = timeOfForecast; #meta efas_suite_name sprintf("%s_%s", inputOriginatingCentre, efas_post_proc) : no_copy; alias mars.origin = inputOriginatingCentre; +include "mars/efas.anoffset.def" alias mars.anoffset=anoffset; # We need this because 'efas_post_proc' is defined later diff --git a/definitions/mars/grib.efas.go.def b/definitions/mars/grib.efas.go.def index 6cfa297c6..1164e5516 100644 --- a/definitions/mars/grib.efas.go.def +++ b/definitions/mars/grib.efas.go.def @@ -2,15 +2,16 @@ alias mars.step = startStep; # See ECC-683 -#alias mars.date = dateOfForecast; -#alias mars.time = timeOfForecast; -meta valDate validity_date(dataDate,dataTime,endStep,stepUnits): no_copy; -meta valTime validity_time(dataDate,dataTime,endStep,stepUnits): no_copy; -alias mars.date = valDate; -alias mars.time = valTime; +alias mars.date = dateOfForecast; +alias mars.time = timeOfForecast; +#meta valDate validity_date(dataDate,dataTime,endStep,stepUnits): no_copy; +#meta valTime validity_time(dataDate,dataTime,endStep,stepUnits): no_copy; +#alias mars.date = valDate; +#alias mars.time = valTime; alias mars.origin = inputOriginatingCentre; +include "mars/efas.anoffset.def" alias mars.anoffset=anoffset; # We need this because 'efas_post_proc' is defined later From 4156fb1c5957d1391853a39e5362fc31c325593b Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Fri, 15 Jun 2018 11:21:11 +0100 Subject: [PATCH 2/6] ECC-701: latest MARS implementation for EFAS. Fix anoffset calculation --- definitions/grib2/local.98.41.def | 12 +++++++----- definitions/mars/efas.anoffset.def | 16 ---------------- definitions/mars/grib.efas.an.def | 1 - definitions/mars/grib.efas.fu.def | 1 - definitions/mars/grib.efas.go.def | 1 - 5 files changed, 7 insertions(+), 24 deletions(-) delete mode 100644 definitions/mars/efas.anoffset.def diff --git a/definitions/grib2/local.98.41.def b/definitions/grib2/local.98.41.def index 1c98a051b..f015f6995 100644 --- a/definitions/grib2/local.98.41.def +++ b/definitions/grib2/local.98.41.def @@ -27,12 +27,14 @@ meta timeOfForecast time (hourOfForecast,minuteOfForecast,secondOfForecast) : # Calculate the Julian number for the forecast date and time. # This will be a floating point number with units of 'day' meta julianForecastDay julian_day(dateOfForecast,hourOfForecast,minuteOfForecast,secondOfForecast): hidden; +# Calculate the difference between the forecast date and reference date +transient diffInDays = (julianForecastDay - julianDay) : hidden; # float -# Note: The 'anoffset' key is the difference between the forecast date and the validity date, -# but we cannot calculate it here (in section 2) because the keys validityDate & validityTime -# are defined later in section 4. -# Therefore that calculation is done when we load the mars config files mars/grib.efas.*.def. -# See mars/efas.anoffset.def +# Now convert this to hours. First convert to minutes then round up +transient diffInHours = (diffInDays * 1440 + 0.5)/60 : hidden; +meta _anoffset round(diffInHours, 10): dump,long_type; +transient anoffset = _anoffset; # needed to force anoffset to be integer +alias local.anoffset = anoffset; # ECC-662 unsigned[2] anoffsetFirst = missing(): dump, can_be_missing; diff --git a/definitions/mars/efas.anoffset.def b/definitions/mars/efas.anoffset.def deleted file mode 100644 index e81de38af..000000000 --- a/definitions/mars/efas.anoffset.def +++ /dev/null @@ -1,16 +0,0 @@ -# This is applicable to EFAS GRIB2 data which uses local definition 41 -# stream must be 'efas' - -if (localDefinitionNumber == 41) { - # Calculate the difference between the forecast date and the validity date - meta the_valTime scale(validityTime,oneConstant,hundred) : read_only, long_type; - meta julianValidDay julian_day(validityDate, the_valTime, zero, zero): hidden, no_copy; - - transient diffInDays = (julianForecastDay - julianValidDay) : hidden; # float - - # Now convert this to hours. First convert to minutes then round up - transient diffInHours = (diffInDays * 1440 + 0.5)/60 : hidden; - meta _anoffset round(diffInHours, 10): dump,long_type; - transient anoffset = _anoffset; # needed to force anoffset to be integer - alias local.anoffset = anoffset; -} diff --git a/definitions/mars/grib.efas.an.def b/definitions/mars/grib.efas.an.def index 8545afdef..d47a2f909 100644 --- a/definitions/mars/grib.efas.an.def +++ b/definitions/mars/grib.efas.an.def @@ -12,7 +12,6 @@ alias mars.time = timeOfForecast; #meta efas_suite_name sprintf("%s_%s_%s", inputOriginatingCentre, efas_forecast, efas_post_proc) : no_copy; alias mars.origin = inputOriginatingCentre; -include "mars/efas.anoffset.def" alias mars.anoffset=anoffset; # We need this because 'efas_post_proc' is defined later diff --git a/definitions/mars/grib.efas.fu.def b/definitions/mars/grib.efas.fu.def index be1af1dbe..6dea1033d 100644 --- a/definitions/mars/grib.efas.fu.def +++ b/definitions/mars/grib.efas.fu.def @@ -7,7 +7,6 @@ alias mars.time = timeOfForecast; #meta efas_suite_name sprintf("%s_%s", inputOriginatingCentre, efas_post_proc) : no_copy; alias mars.origin = inputOriginatingCentre; -include "mars/efas.anoffset.def" alias mars.anoffset=anoffset; # We need this because 'efas_post_proc' is defined later diff --git a/definitions/mars/grib.efas.go.def b/definitions/mars/grib.efas.go.def index 1164e5516..3c2dba0e9 100644 --- a/definitions/mars/grib.efas.go.def +++ b/definitions/mars/grib.efas.go.def @@ -11,7 +11,6 @@ alias mars.time = timeOfForecast; alias mars.origin = inputOriginatingCentre; -include "mars/efas.anoffset.def" alias mars.anoffset=anoffset; # We need this because 'efas_post_proc' is defined later From 73e60a070c422f155c4c9b2ae6bb4fe7e74cafeb Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Fri, 15 Jun 2018 11:28:02 +0100 Subject: [PATCH 3/6] ECC-701: latest MARS implementation for EFAS. Use endStep for an,go and fu --- definitions/mars/grib.efas.an.def | 5 +++-- definitions/mars/grib.efas.fu.def | 6 ++++-- definitions/mars/grib.efas.go.def | 5 +++-- tests/grib_efas.sh | 4 ++-- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/definitions/mars/grib.efas.an.def b/definitions/mars/grib.efas.an.def index d47a2f909..1c5cb6012 100644 --- a/definitions/mars/grib.efas.an.def +++ b/definitions/mars/grib.efas.an.def @@ -1,7 +1,8 @@ # EFAS Analysis (Water Balance) -alias mars.step = startStep; -# See ECC-683 +# ECC-701 +alias mars.step = endStep; + alias mars.date = dateOfForecast; alias mars.time = timeOfForecast; #meta valDate validity_date(dataDate,dataTime,endStep,stepUnits): no_copy; diff --git a/definitions/mars/grib.efas.fu.def b/definitions/mars/grib.efas.fu.def index 6dea1033d..7d8f81eb4 100644 --- a/definitions/mars/grib.efas.fu.def +++ b/definitions/mars/grib.efas.fu.def @@ -1,5 +1,7 @@ -# EFAS Fullup -alias mars.step = startStep; +# EFAS Fill-up + +# ECC-701 +alias mars.step = endStep; alias mars.date = dateOfForecast; alias mars.time = timeOfForecast; diff --git a/definitions/mars/grib.efas.go.def b/definitions/mars/grib.efas.go.def index 3c2dba0e9..1bbc15194 100644 --- a/definitions/mars/grib.efas.go.def +++ b/definitions/mars/grib.efas.go.def @@ -1,7 +1,8 @@ # EFAS Gridded observations -alias mars.step = startStep; -# See ECC-683 +# ECC-701 +alias mars.step = endStep; + alias mars.date = dateOfForecast; alias mars.time = timeOfForecast; #meta valDate validity_date(dataDate,dataTime,endStep,stepUnits): no_copy; diff --git a/tests/grib_efas.sh b/tests/grib_efas.sh index 62ebc2677..d54426a58 100755 --- a/tests/grib_efas.sh +++ b/tests/grib_efas.sh @@ -65,12 +65,12 @@ ${tools_dir}/grib_set -s setLocalDefinition=1,localDefinitionNumber=41,yearOfFor grib_check_key_equals $temp1 anoffset 25 grib_check_key_equals $temp1 anoffsetFirst,anoffsetLast,anoffsetFrequency "MISSING MISSING MISSING" -# ECC-663: MARS step +# MARS step types="an fu go" for t in $types; do ${tools_dir}/grib_set -s setLocalDefinition=1,localDefinitionNumber=41,type=$t,stepType=accum,stepRange=12-36,paramId=260268 \ $sample $temp1 - grib_check_key_equals $temp1 mars.step 12 # start step + grib_check_key_equals $temp1 mars.step 36 # end step (ECC-701) #${tools_dir}/grib_dump -Da $temp1 | grep mars.step done From 1e188a902d155f374ede8e6d5e49e376b39be4d6 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Fri, 15 Jun 2018 11:48:55 +0100 Subject: [PATCH 4/6] ECC-701: latest MARS implementation for EFAS. Use new type 'sia' (may be renamed later) instead of analysis --- definitions/mars/{grib.efas.an.def => grib.efas.sia.def} | 2 +- definitions/mars/type.table | 3 ++- src/grib_accessor_class_g2_mars_labeling.c | 3 ++- tests/grib_efas.sh | 3 ++- 4 files changed, 7 insertions(+), 4 deletions(-) rename definitions/mars/{grib.efas.an.def => grib.efas.sia.def} (93%) diff --git a/definitions/mars/grib.efas.an.def b/definitions/mars/grib.efas.sia.def similarity index 93% rename from definitions/mars/grib.efas.an.def rename to definitions/mars/grib.efas.sia.def index 1c5cb6012..0d8be9c8e 100644 --- a/definitions/mars/grib.efas.an.def +++ b/definitions/mars/grib.efas.sia.def @@ -1,4 +1,4 @@ -# EFAS Analysis (Water Balance) +# EFAS Simulated Analysis (Water Balance) # ECC-701 alias mars.step = endStep; diff --git a/definitions/mars/type.table b/definitions/mars/type.table index 27b84bc29..e7a3b60bd 100644 --- a/definitions/mars/type.table +++ b/definitions/mars/type.table @@ -55,7 +55,8 @@ 65 cv Calibration/Validation forecast 70 or Ocean reanalysis 71 fx Flux forcing -72 fu EFAS Fillup +72 fu Fill-up +73 sia EFAS simulated Analysis 80 fcmean Forecast mean 81 fcmax Forecast maximum 82 fcmin Forecast minimum diff --git a/src/grib_accessor_class_g2_mars_labeling.c b/src/grib_accessor_class_g2_mars_labeling.c index cac136be6..b80148547 100644 --- a/src/grib_accessor_class_g2_mars_labeling.c +++ b/src/grib_accessor_class_g2_mars_labeling.c @@ -369,7 +369,8 @@ static int extra_set(grib_accessor* a,long val) break; case 70: /* Ocean reanalysis (or) */ case 71: /* Flux forcing (fx) */ - case 72: /* EFAS Fillup (fu) */ + case 72: /* EFAS Fill-up (fu) */ + case 73: /* EFAS Simulated Analysis */ case 80: /* Forecast mean (fcmean) */ case 81: /* Forecast maximum (fcmax) */ case 82: /* Forecast minimum (fcmin) */ diff --git a/tests/grib_efas.sh b/tests/grib_efas.sh index d54426a58..3dca6cdc9 100755 --- a/tests/grib_efas.sh +++ b/tests/grib_efas.sh @@ -66,11 +66,12 @@ grib_check_key_equals $temp1 anoffset 25 grib_check_key_equals $temp1 anoffsetFirst,anoffsetLast,anoffsetFrequency "MISSING MISSING MISSING" # MARS step -types="an fu go" +types="sia fu go" for t in $types; do ${tools_dir}/grib_set -s setLocalDefinition=1,localDefinitionNumber=41,type=$t,stepType=accum,stepRange=12-36,paramId=260268 \ $sample $temp1 grib_check_key_equals $temp1 mars.step 36 # end step (ECC-701) + grib_check_key_exists $temp1 mars.anoffset #${tools_dir}/grib_dump -Da $temp1 | grep mars.step done From a50141462282a5845bc9b7f78961e6a28cccb772 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Mon, 18 Jun 2018 10:42:00 +0100 Subject: [PATCH 5/6] ECC-701: Change type to sfo (Simulation forced with observations) --- definitions/Makefile.am | 2 +- definitions/mars/{grib.efas.sia.def => grib.efas.sfo.def} | 3 ++- definitions/mars/type.table | 2 +- src/grib_accessor_class_g2_mars_labeling.c | 4 ++-- tests/grib_efas.sh | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) rename definitions/mars/{grib.efas.sia.def => grib.efas.sfo.def} (91%) diff --git a/definitions/Makefile.am b/definitions/Makefile.am index 700b8f4c1..06db12299 100644 --- a/definitions/Makefile.am +++ b/definitions/Makefile.am @@ -16306,12 +16306,12 @@ dist_definitionsmars_DATA = \ mars/grib.edmo.cl.def\ mars/grib.edmo.fc.def\ mars/grib.edmo.ssd.def\ - mars/grib.efas.an.def\ mars/grib.efas.cf.def\ mars/grib.efas.fc.def\ mars/grib.efas.fu.def\ mars/grib.efas.go.def\ mars/grib.efas.pf.def\ + mars/grib.efas.sfo.def\ mars/grib.efhc.cf.def\ mars/grib.efhc.icp.def\ mars/grib.efhc.pf.def\ diff --git a/definitions/mars/grib.efas.sia.def b/definitions/mars/grib.efas.sfo.def similarity index 91% rename from definitions/mars/grib.efas.sia.def rename to definitions/mars/grib.efas.sfo.def index 0d8be9c8e..a66d7053d 100644 --- a/definitions/mars/grib.efas.sia.def +++ b/definitions/mars/grib.efas.sfo.def @@ -1,4 +1,5 @@ -# EFAS Simulated Analysis (Water Balance) +# Simulation forced with observations (EFAS) +# Water Balance # ECC-701 alias mars.step = endStep; diff --git a/definitions/mars/type.table b/definitions/mars/type.table index e7a3b60bd..d9f6486b3 100644 --- a/definitions/mars/type.table +++ b/definitions/mars/type.table @@ -56,7 +56,7 @@ 70 or Ocean reanalysis 71 fx Flux forcing 72 fu Fill-up -73 sia EFAS simulated Analysis +73 sfo Simulation forced with observations 80 fcmean Forecast mean 81 fcmax Forecast maximum 82 fcmin Forecast minimum diff --git a/src/grib_accessor_class_g2_mars_labeling.c b/src/grib_accessor_class_g2_mars_labeling.c index b80148547..18492138b 100644 --- a/src/grib_accessor_class_g2_mars_labeling.c +++ b/src/grib_accessor_class_g2_mars_labeling.c @@ -369,8 +369,8 @@ static int extra_set(grib_accessor* a,long val) break; case 70: /* Ocean reanalysis (or) */ case 71: /* Flux forcing (fx) */ - case 72: /* EFAS Fill-up (fu) */ - case 73: /* EFAS Simulated Analysis */ + case 72: /* Fill-up (fu) */ + case 73: /* Simulation forced with observations (sfo) */ case 80: /* Forecast mean (fcmean) */ case 81: /* Forecast maximum (fcmax) */ case 82: /* Forecast minimum (fcmin) */ diff --git a/tests/grib_efas.sh b/tests/grib_efas.sh index 3dca6cdc9..1d4f686f4 100755 --- a/tests/grib_efas.sh +++ b/tests/grib_efas.sh @@ -66,7 +66,7 @@ grib_check_key_equals $temp1 anoffset 25 grib_check_key_equals $temp1 anoffsetFirst,anoffsetLast,anoffsetFrequency "MISSING MISSING MISSING" # MARS step -types="sia fu go" +types="sfo fu go" for t in $types; do ${tools_dir}/grib_set -s setLocalDefinition=1,localDefinitionNumber=41,type=$t,stepType=accum,stepRange=12-36,paramId=260268 \ $sample $temp1 From cc04daf388c2be445bcc62ff6310ac6f13347cd1 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Mon, 25 Jun 2018 17:01:42 +0100 Subject: [PATCH 6/6] ECC-707: Key 'date' printed by grib_ls can be different from 'ls.date' --- definitions/grib2/local.98.41.def | 1 + tools/grib_tools.c | 20 +++++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/definitions/grib2/local.98.41.def b/definitions/grib2/local.98.41.def index f015f6995..4cdc526ca 100644 --- a/definitions/grib2/local.98.41.def +++ b/definitions/grib2/local.98.41.def @@ -43,6 +43,7 @@ unsigned[2] anoffsetFrequency = missing(): dump, can_be_missing; # Boolean transient is_efas = 1; +transient lsdate_bug = 1: hidden; # See ECC-707 # Note: the key typeOfPostProcessing is in the PDTNs 70, 71, 72 and 73 concept efas_post_proc { diff --git a/tools/grib_tools.c b/tools/grib_tools.c index f6fad9b48..07bf9c849 100644 --- a/tools/grib_tools.c +++ b/tools/grib_tools.c @@ -813,6 +813,17 @@ void grib_skip_check(grib_runtime_options* options,grib_handle* h) } } +/* See ECC-707 */ +static int fix_for_lsdate_needed(grib_handle* h) +{ + long lsdate_bug = 0; + int err = grib_get_long(h, "lsdate_bug", &lsdate_bug); + if (!err && lsdate_bug == 1) { + return 1; + } + return 0; +} + void grib_print_key_values(grib_runtime_options* options, grib_handle* h) { int i=0; @@ -864,15 +875,22 @@ void grib_print_key_values(grib_runtime_options* options, grib_handle* h) } } else { /* Other products e.g. GRIB */ + const int fix_lsdate = (fix_for_lsdate_needed(h) && options->name_space && strcmp(options->name_space,"ls")==0); + if (grib_is_missing(h,options->print_keys[i].name,&ret) && ret==GRIB_SUCCESS) { sprintf(value,"MISSING"); } else if ( ret == GRIB_SUCCESS ) { + const char* pName = NULL; if (options->print_keys[i].type == GRIB_TYPE_UNDEFINED) grib_get_native_type(h,options->print_keys[i].name,&(options->print_keys[i].type)); switch (options->print_keys[i].type) { case GRIB_TYPE_STRING: - ret=grib_get_string( h,options->print_keys[i].name,value,&len); + pName = options->print_keys[i].name; + if (fix_lsdate && strcmp(pName, "date")==0) { /* ECC-707 */ + pName = "ls.date"; + } + ret=grib_get_string( h,pName,value,&len); break; case GRIB_TYPE_DOUBLE: ret=grib_get_double( h,options->print_keys[i].name,&dvalue);