Merge branch 'develop' into bugfix/memory_leaks

examples/C/bufr_get_keys.sh

@@ -16,7 +16,7 @@ label="bufr_get_keys_test_c"
 
 #Define tmp file
 fTmp=${label}.tmp.txt
-rm -f $fTmp | true
+rm -f $fTmp
 
 #We check "syno_multi.bufr". The path is
 #hardcoded in the example
@@ -24,11 +24,11 @@ rm -f $fTmp | true
 REDIRECT=/dev/null
 
 #Write the key values into a file
-${examples_dir}/c_bufr_get_keys 2> $REDIRECT > $fTmp
+${examples_dir}/c_bufr_get_keys > $fTmp # 2> $REDIRECT
 
 #TODO: check the results
 
-#cat  $fTmp
+#cat $fTmp
 
 #Clean up
-rm -f $fTmp | true
+rm -f $fTmp

src/grib_accessor_class_bufr_data_array.c

@@ -412,7 +412,7 @@ static void init(grib_accessor* a, const long v, grib_arguments* params)
     a->length           = 0;
     self->bitsToEndData = get_length(a) * 8;
     self->unpackMode    = CODES_BUFR_UNPACK_STRUCTURE;
-
+    self->inputBitmap   = NULL;
     /* Assert(a->length>=0); */
 }
 
@@ -461,6 +461,7 @@ static void self_clear(grib_context* c, grib_accessor_bufr_data_array* self)
     self->refValIndex = 0;
     tableB_override_clear(c, self);
     self->set_to_missing_if_out_of_range = 0;
+    if (self->inputBitmap) grib_context_free(c, self->inputBitmap);
 }
 
 static int get_native_type(grib_accessor* a)
@@ -3422,4 +3423,5 @@ static void destroy(grib_context* c, grib_accessor* a)
         grib_sarray_delete_content(c, self->tempStrings);
         grib_sarray_delete(c, self->tempStrings);
     }
+    grib_iarray_delete(self->iss_list);
 }

src/grib_accessor_class_bufrdc_expanded_descriptors.c

@@ -192,6 +192,7 @@ static int unpack_long(grib_accessor* a, long* val, size_t* len)
             val[rlen++] = v[i];
     }
     *len = rlen;
+    grib_context_free(c,v);
 
     return GRIB_SUCCESS;
 }

src/grib_io.c

@@ -399,6 +399,7 @@ static int read_GRIB(reader* r)
 
         default:
             r->seek_from_start(r->read_data, r->offset + 4);
+            grib_buffer_delete(c, buf);
             return GRIB_UNSUPPORTED_EDITION;
             break;
     }
@@ -898,13 +899,25 @@ static int _read_any(reader* r, int grib_ok, int bufr_ok, int hdf5_ok, int wrap_
 
     return err;
 }
+
 static int read_any(reader* r, int grib_ok, int bufr_ok, int hdf5_ok, int wrap_ok)
 {
     int result = 0;
+
+#ifndef ECCODES_EACH_THREAD_OWN_FILE
+    /* If several threads can open the same file, then we need the locks
+     * so each thread gets its own message. Otherwise if threads are passed
+     * different files, then the lock is not needed
+     */
     GRIB_MUTEX_INIT_ONCE(&once, &init);
     GRIB_MUTEX_LOCK(&mutex1);
+#endif
+
     result = _read_any(r, grib_ok, bufr_ok, hdf5_ok, wrap_ok);
+
+#ifndef ECCODES_EACH_THREAD_OWN_FILE
     GRIB_MUTEX_UNLOCK(&mutex1);
+#endif
     return result;
 }
 

src/grib_yacc.h

@@ -1,19 +1,20 @@
-/* A Bison parser, made by GNU Bison 2.7.  */
+/* A Bison parser, made by GNU Bison 3.5.4.  */
 
 /* Bison interface for Yacc-like parsers in C
-   
-      Copyright (C) 1984, 1989-1990, 2000-2012 Free Software Foundation, Inc.
-   
+
+   Copyright (C) 1984, 1989-1990, 2000-2015, 2018-2020 Free Software Foundation,
+   Inc.
+
    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.
-   
+
    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.
-   
+
    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
 
@@ -26,13 +27,16 @@
    special exception, which will cause the skeleton and the resulting
    Bison output files to be licensed under the GNU General Public
    License without this special exception.
-   
+
    This special exception was added by the Free Software Foundation in
    version 2.2 of Bison.  */
 
+/* Undocumented macros, especially those whose name start with YY_,
+   are private implementation details.  Do not rely on them.  */
+
 #ifndef YY_YY_Y_TAB_H_INCLUDED
 # define YY_YY_Y_TAB_H_INCLUDED
-/* Enabling traces.  */
+/* Debug traces.  */
 #ifndef YYDEBUG
 # define YYDEBUG 0
 #endif
@@ -40,137 +44,136 @@
 extern int grib_yydebug;
 #endif
 
-/* Tokens.  */
+/* Token type.  */
 #ifndef YYTOKENTYPE
 # define YYTOKENTYPE
-   /* Put the tokens into the symbol table, so that GDB and other debuggers
-      know about them.  */
-   enum grib_yytokentype {
-     LOWERCASE = 258,
-     IF = 259,
-     IF_TRANSIENT = 260,
-     ELSE = 261,
-     END = 262,
-     CLOSE = 263,
-     UNSIGNED = 264,
-     TEMPLATE = 265,
-     TEMPLATE_NOFAIL = 266,
-     TRIGGER = 267,
-     ASCII = 268,
-     GROUP = 269,
-     NON_ALPHA = 270,
-     KSEC1EXPVER = 271,
-     LABEL = 272,
-     LIST = 273,
-     IS_IN_LIST = 274,
-     IS_IN_DICT = 275,
-     IS_INTEGER = 276,
-     TO_INTEGER = 277,
-     TO_STRING = 278,
-     SEX2DEC = 279,
-     WHILE = 280,
-     IBMFLOAT = 281,
-     SIGNED = 282,
-     UINT8 = 283,
-     INT8 = 284,
-     UINT16 = 285,
-     INT16 = 286,
-     UINT16_LITTLE_ENDIAN = 287,
-     INT16_LITTLE_ENDIAN = 288,
-     UINT32 = 289,
-     INT32 = 290,
-     UINT32_LITTLE_ENDIAN = 291,
-     INT32_LITTLE_ENDIAN = 292,
-     UINT64 = 293,
-     INT64 = 294,
-     UINT64_LITTLE_ENDIAN = 295,
-     INT64_LITTLE_ENDIAN = 296,
-     BLOB = 297,
-     BYTE = 298,
-     CODETABLE = 299,
-     SMART_TABLE = 300,
-     DICTIONARY = 301,
-     COMPLEX_CODETABLE = 302,
-     LOOKUP = 303,
-     ALIAS = 304,
-     UNALIAS = 305,
-     META = 306,
-     POS = 307,
-     INTCONST = 308,
-     TRANS = 309,
-     FLAGBIT = 310,
-     CONCEPT = 311,
-     GETENV = 312,
-     HASH_ARRAY = 313,
-     CONCEPT_NOFAIL = 314,
-     NIL = 315,
-     DUMMY = 316,
-     MODIFY = 317,
-     READ_ONLY = 318,
-     STRING_TYPE = 319,
-     LONG_TYPE = 320,
-     DOUBLE_TYPE = 321,
-     NO_COPY = 322,
-     DUMP = 323,
-     JSON = 324,
-     XML = 325,
-     NO_FAIL = 326,
-     EDITION_SPECIFIC = 327,
-     OVERRIDE = 328,
-     HIDDEN = 329,
-     CAN_BE_MISSING = 330,
-     MISSING = 331,
-     CONSTRAINT = 332,
-     COPY_OK = 333,
-     WHEN = 334,
-     SET = 335,
-     SET_NOFAIL = 336,
-     WRITE = 337,
-     APPEND = 338,
-     PRINT = 339,
-     EXPORT = 340,
-     REMOVE = 341,
-     RENAME = 342,
-     SKIP = 343,
-     PAD = 344,
-     SECTION_PADDING = 345,
-     MESSAGE = 346,
-     MESSAGE_COPY = 347,
-     PADTO = 348,
-     PADTOEVEN = 349,
-     PADTOMULTIPLE = 350,
-     G1_HALF_BYTE = 351,
-     G1_MESSAGE_LENGTH = 352,
-     G1_SECTION4_LENGTH = 353,
-     SECTION_LENGTH = 354,
-     LENGTH = 355,
-     FLAG = 356,
-     ITERATOR = 357,
-     NEAREST = 358,
-     BOX = 359,
-     KSEC = 360,
-     ASSERT = 361,
-     SUBSTR = 362,
-     CASE = 363,
-     SWITCH = 364,
-     DEFAULT = 365,
-     EQ = 366,
-     NE = 367,
-     GE = 368,
-     LE = 369,
-     LT = 370,
-     GT = 371,
-     BIT = 372,
-     BITOFF = 373,
-     AND = 374,
-     OR = 375,
-     NOT = 376,
-     IS = 377,
-     IDENT = 378,
-     STRING = 379,
-     INTEGER = 380,
-     FLOAT = 381
-   };
+  enum grib_yytokentype
+  {
+    LOWERCASE = 258,
+    IF = 259,
+    IF_TRANSIENT = 260,
+    ELSE = 261,
+    END = 262,
+    CLOSE = 263,
+    UNSIGNED = 264,
+    TEMPLATE = 265,
+    TEMPLATE_NOFAIL = 266,
+    TRIGGER = 267,
+    ASCII = 268,
+    GROUP = 269,
+    NON_ALPHA = 270,
+    KSEC1EXPVER = 271,
+    LABEL = 272,
+    LIST = 273,
+    IS_IN_LIST = 274,
+    IS_IN_DICT = 275,
+    IS_INTEGER = 276,
+    TO_INTEGER = 277,
+    TO_STRING = 278,
+    SEX2DEC = 279,
+    WHILE = 280,
+    IBMFLOAT = 281,
+    SIGNED = 282,
+    UINT8 = 283,
+    INT8 = 284,
+    UINT16 = 285,
+    INT16 = 286,
+    UINT16_LITTLE_ENDIAN = 287,
+    INT16_LITTLE_ENDIAN = 288,
+    UINT32 = 289,
+    INT32 = 290,
+    UINT32_LITTLE_ENDIAN = 291,
+    INT32_LITTLE_ENDIAN = 292,
+    UINT64 = 293,
+    INT64 = 294,
+    UINT64_LITTLE_ENDIAN = 295,
+    INT64_LITTLE_ENDIAN = 296,
+    BLOB = 297,
+    BYTE = 298,
+    CODETABLE = 299,
+    SMART_TABLE = 300,
+    DICTIONARY = 301,
+    COMPLEX_CODETABLE = 302,
+    LOOKUP = 303,
+    ALIAS = 304,
+    UNALIAS = 305,
+    META = 306,
+    POS = 307,
+    INTCONST = 308,
+    TRANS = 309,
+    FLAGBIT = 310,
+    CONCEPT = 311,
+    GETENV = 312,
+    HASH_ARRAY = 313,
+    CONCEPT_NOFAIL = 314,
+    NIL = 315,
+    DUMMY = 316,
+    MODIFY = 317,
+    READ_ONLY = 318,
+    STRING_TYPE = 319,
+    LONG_TYPE = 320,
+    DOUBLE_TYPE = 321,
+    NO_COPY = 322,
+    DUMP = 323,
+    JSON = 324,
+    XML = 325,
+    NO_FAIL = 326,
+    EDITION_SPECIFIC = 327,
+    OVERRIDE = 328,
+    HIDDEN = 329,
+    CAN_BE_MISSING = 330,
+    MISSING = 331,
+    CONSTRAINT = 332,
+    COPY_OK = 333,
+    WHEN = 334,
+    SET = 335,
+    SET_NOFAIL = 336,
+    WRITE = 337,
+    APPEND = 338,
+    PRINT = 339,
+    EXPORT = 340,
+    REMOVE = 341,
+    RENAME = 342,
+    SKIP = 343,
+    PAD = 344,
+    SECTION_PADDING = 345,
+    MESSAGE = 346,
+    MESSAGE_COPY = 347,
+    PADTO = 348,
+    PADTOEVEN = 349,
+    PADTOMULTIPLE = 350,
+    G1_HALF_BYTE = 351,
+    G1_MESSAGE_LENGTH = 352,
+    G1_SECTION4_LENGTH = 353,
+    SECTION_LENGTH = 354,
+    LENGTH = 355,
+    FLAG = 356,
+    ITERATOR = 357,
+    NEAREST = 358,
+    BOX = 359,
+    KSEC = 360,
+    ASSERT = 361,
+    SUBSTR = 362,
+    CASE = 363,
+    SWITCH = 364,
+    DEFAULT = 365,
+    EQ = 366,
+    NE = 367,
+    GE = 368,
+    LE = 369,
+    LT = 370,
+    GT = 371,
+    BIT = 372,
+    BITOFF = 373,
+    AND = 374,
+    OR = 375,
+    NOT = 376,
+    IS = 377,
+    IDENT = 378,
+    STRING = 379,
+    INTEGER = 380,
+    FLOAT = 381
+  };
 #endif
 /* Tokens.  */
 #define LOWERCASE 258
@@ -298,12 +301,10 @@ extern int grib_yydebug;
 #define INTEGER 380
 #define FLOAT 381
 
-
-
+/* Value type.  */
 #if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED
-typedef union YYSTYPE
+union YYSTYPE
 {
-/* Line 2058 of yacc.c  */
 #line 39 "griby.y"
 
     char                    *str;
@@ -322,29 +323,17 @@ typedef union YYSTYPE
   grib_rule               *rules;
   grib_rule_entry         *rule_entry;
 
+#line 327 "y.tab.h"
 
-/* Line 2058 of yacc.c  */
-#line 328 "y.tab.h"
-} YYSTYPE;
+};
+typedef union YYSTYPE YYSTYPE;
 # define YYSTYPE_IS_TRIVIAL 1
-# define grib_yystype YYSTYPE /* obsolescent; will be withdrawn */
 # define YYSTYPE_IS_DECLARED 1
 #endif
 
+
 extern YYSTYPE grib_yylval;
 
-#ifdef YYPARSE_PARAM
-#if defined __STDC__ || defined __cplusplus
-int grib_yyparse (void *YYPARSE_PARAM);
-#else
-int grib_yyparse ();
-#endif
-#else /* ! YYPARSE_PARAM */
-#if defined __STDC__ || defined __cplusplus
 int grib_yyparse (void);
-#else
-int grib_yyparse ();
-#endif
-#endif /* ! YYPARSE_PARAM */
 
 #endif /* !YY_YY_Y_TAB_H_INCLUDED  */

src/makeyacc

@@ -1,4 +1,7 @@
 set -xe
+
+# We use flex and bison
+
 export LEX=flex
 export LEX_OUT=gribl.c
 $LEX -o gribl.c gribl.l
@@ -7,6 +10,7 @@ sed 's/fgetc/getc/g' < grib_lex1.c > grib_lex.c
 rm -f grib_lex1.c
 rm -f $LEX_OUT
 
+# This invokes bison
 yacc -v -d griby.y
 sed 's/yy/grib_yy/g' < y.tab.c > grib_yacc1.c
 sed 's/fgetc/getc/g' < grib_yacc1.c > grib_yacc.c

tests/grib_multi_from_message.c

@@ -12,8 +12,7 @@
  * test: reading GRIB2 multi fields messages from memory
  */
 
-#include "grib_api.h"
-#include <sys/stat.h>
+#include "grib_api_internal.h"
 #include <assert.h>
 
 #ifndef ECCODES_ON_WINDOWS
@@ -44,9 +43,11 @@ int main(int argc, char* argv[])
     FILE* f             = NULL;
     long level          = 0;
     grib_context* c     = grib_context_get_default();
+    int multi_support   = 0;
 
     if (argc == 3 && !strcmp(argv[1], "-m")) {
-        grib_multi_support_on(0);
+        grib_multi_support_on(c);
+        multi_support = 1;
         filename = argv[2];
     }
     else if (argc == 2)
@@ -85,6 +86,9 @@ int main(int argc, char* argv[])
         GRIB_CHECK(grib_get_string(h, "shortName", shortName, &len), "shortName");
         GRIB_CHECK(grib_get_long(h, "level", &level), "level");
         printf("%d %s %ld\n", count, shortName, level);
+        if (!multi_support) {
+            grib_context_free(c, h->buffer->data); /* See grib_handle_delete and grib_buffer_delete */
+        }
         grib_handle_delete(h);
     }
 

tests/ieee.sh

@@ -48,12 +48,15 @@ export GRIB_IEEE_PACKING
 ${tools_dir}/grib_filter -o $out32 w.filter $infile 
 ${tools_dir}/grib_filter r.filter $out32 > $out32.txt 
 diff $out32.txt ${data_dir}/ieee_test.good
+grib_check_key_equals $out32 'packingType,precision' 'grid_ieee 1'
 
 GRIB_IEEE_PACKING=64
 export GRIB_IEEE_PACKING
 ${tools_dir}/grib_filter -o $out64 w.filter $infile 
 ${tools_dir}/grib_filter r.filter $out64 > $out64.txt 
 diff $out64.txt ${data_dir}/ieee_test.good
+grib_check_key_equals $out64 'packingType,precision' 'grid_ieee 2'
+
 
 rm -f $outsimple $out32 $out64 $out32.txt $out64.txt
 rm -f ${data_dir}/$outsimple.txt ${data_dir}/$out32.txt ${data_dir}/$out64.txt 

tools/bufr_dump.c

@@ -379,6 +379,7 @@ int grib_tool_new_handle_action(grib_runtime_options* options, grib_handle* h)
 {
     long length = 0;
     int i, err = 0;
+    grib_handle* hclone     = NULL;
     grib_accessor* a        = NULL;
     grib_accessors_list* al = NULL;
     if (grib_get_long(h, "totalLength", &length) != GRIB_SUCCESS)
@@ -422,7 +423,8 @@ int grib_tool_new_handle_action(grib_runtime_options* options, grib_handle* h)
                 new_handle = grib_handle_new_from_message(0, buffer, size);
                 Assert(new_handle);
                 /* Replace handle with the new one which has only one subset */
-                h = new_handle; /*TODO: possible leak!*/
+                h = new_handle;
+                hclone = h2; /* to be deleted later */
             }
         }
         else {
@@ -539,6 +541,7 @@ int grib_tool_new_handle_action(grib_runtime_options* options, grib_handle* h)
         }
     }
 
+    grib_handle_delete(hclone);
     return 0;
 }
 

tools/codes_count.c

@@ -58,9 +58,9 @@ static int count_messages(FILE* in, int message_type, unsigned long* count)
                 }
             }
             if (mesg && !err) {
-                grib_context_free(c, mesg);
                 (*count)++;
             }
+            grib_context_free(c, mesg);
         }
     }