ECC-407: Add option to bufr_count/grib_count to count valid messages

tests/bufr_count.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-# Copyright 2005-2017 ECMWF.
+# Copyright 2005-2018 ECMWF.
 #
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
@@ -10,13 +10,32 @@
 
 . ./include.sh
 
-#set -x
-
-#Enter data dir
+# Enter data dir
 cd ${data_dir}/bufr
+input=syno_multi.bufr
 
-# counting messages
-count=`${tools_dir}/bufr_count syno_multi.bufr`
-#[ "$count" = "3" ]
-count=`${tools_dir}/codes_count syno_multi.bufr`
+# Counting valid messages
+# ------------------------
+count=`${tools_dir}/bufr_count $input`
 [ "$count" = "3" ]
+count=`${tools_dir}/codes_count $input`
+[ "$count" = "3" ]
+
+# Files with invalid (unreadable) messages
+# -----------------------------------------
+temp=$input.truncated
+# Remove last 4 bytes of multi message BUFR file
+head --bytes=-4 $input > $temp
+
+set +e
+# Without -f, bufr_count should fail
+${tools_dir}/bufr_count $temp
+status=$?
+set -e
+[ $status -ne 0 ]
+
+# With -f should count the valid messages
+count=`${tools_dir}/bufr_count -f $temp`
+[ "$count" = "2" ]
+
+rm -f $temp

tools/bufr_count.desc

@@ -1,12 +1,11 @@
-
 NAME    bufr_count
 
 DESCRIPTION
         Print the total number of BUFR messages in the given files. 
 
 USAGE 
-        bufr_count [options] file file ...
+        bufr_count [options] bufr_file bufr_file ...
 
 OPTIONS
         -v  Verbose mode. The number of messages is given for each file.
-         
+        -f  Force. Force the execution not to fail on error.

tools/codes_count.c

@@ -1,5 +1,5 @@
 /*
- * Copyright 2005-2017 ECMWF.
+ * Copyright 2005-2018 ECMWF.
  *
  * This software is licensed under the terms of the Apache Licence Version 2.0
  * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
@@ -9,10 +9,11 @@
  */
 
 #include "grib_api_internal.h"
+static int fail_on_error = 1;
 
 static void usage(const char* prog)
 {
-    printf("usage: %s [-v] infile1 infile2 ... \n",prog);
+    printf("Usage: %s [-v] [-f] infile1 infile2 ... \n",prog);
     exit(1);
 }
 
@@ -31,10 +32,30 @@ static int count_messages(FILE* in, int message_type, unsigned long *count)
     if (message_type == CODES_GRIB)        wmo_read=wmo_read_grib_from_file_malloc;
     else if (message_type == CODES_BUFR)   wmo_read=wmo_read_bufr_from_file_malloc;
     else                                   wmo_read=wmo_read_any_from_file_malloc;
-
-    while ( (mesg=wmo_read(in,0,  &size,&offset,&err))!=NULL && err==GRIB_SUCCESS) {
-        grib_context_free(c,mesg);
-        (*count)++;
+    
+    if (fail_on_error)
+    {
+        while ( (mesg=wmo_read(in,0,  &size,&offset,&err))!=NULL && err==GRIB_SUCCESS) {
+            grib_context_free(c,mesg);
+            (*count)++;
+        }
+    }
+    else
+    {
+        int done = 0;
+        while(!done) {
+            mesg = wmo_read(in, 0, &size, &offset, &err);
+            /*printf("Count so far=%ld, mesg=%x, err=%d (%s)\n", *count, mesg, err, grib_get_error_message(err));*/
+            if (!mesg) {
+                if (err == GRIB_END_OF_FILE || err == GRIB_PREMATURE_END_OF_FILE) {
+                    done = 1; /* reached the end */
+                }
+            }
+            if (mesg && !err) {
+                grib_context_free(c,mesg);
+                (*count)++;
+            }
+        }
     }
 
     if (err==GRIB_END_OF_FILE) err=GRIB_SUCCESS;
@@ -47,7 +68,7 @@ int main(int argc,char* argv[])
     FILE* infh = NULL;
     char* filename;
     int i, verbose=0;
-    int err=0;
+    int err=0, files_processed=0;
     unsigned long count_total=0, count_curr=0;
     int message_type = 0; /* GRIB, BUFR etc */
 
@@ -60,7 +81,10 @@ int main(int argc,char* argv[])
     for (i=1;i<argc;i++) {
         if (strcmp(argv[i], "-v")==0) {
             verbose = 1;
-            if (argc <3) usage(argv[0]);
+            continue;
+        }
+        if (strcmp(argv[i], "-f")==0) {
+            fail_on_error = 0;
             continue;
         }
         filename=argv[i];
@@ -69,10 +93,10 @@ int main(int argc,char* argv[])
             perror(filename);
             exit(1);
         }
-
+        files_processed=1; /* At least one file processed */
         count_curr=0;
         err=count_messages(infh, message_type, &count_curr);
-        if (err) {
+        if (err && fail_on_error) {
             fprintf(stderr,"Invalid message(s) found in %s", filename);
             if (count_curr>0) fprintf(stderr," (got as far as %lu)", count_curr);
             fprintf(stderr,"\n");
@@ -88,6 +112,8 @@ int main(int argc,char* argv[])
 
         fclose(infh);
     }
+
+    if (!files_processed) usage(argv[0]);
     if (verbose) {
         printf("%7lu %s\n", count_total, "total");
     } else {

tools/codes_count.desc

@@ -8,4 +8,4 @@ USAGE
 
 OPTIONS
         -v  Verbose mode. The number of messages is given for each file.
-         
+        -f  Force. Force the execution not to fail on error.

tools/grib_count.desc

@@ -8,4 +8,4 @@ USAGE
 
 OPTIONS
         -v  Verbose mode. The number of messages is given for each file.
-         
+        -f  Force. Force the execution not to fail on error.