From 97e2d3ddd61a1870f44e0d658649c1ec4fd7dd3a Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Tue, 31 Jul 2018 17:33:22 +0100 Subject: [PATCH] ECC-727: handle all versions of HDF5 Super Block --- definitions/hdf5/boot.def | 44 ++++++++++-- src/grib_io.c | 139 +++++++++++++++++++++++++------------- 2 files changed, 132 insertions(+), 51 deletions(-) diff --git a/definitions/hdf5/boot.def b/definitions/hdf5/boot.def index ba6d79bae..3d4ce84de 100644 --- a/definitions/hdf5/boot.def +++ b/definitions/hdf5/boot.def @@ -4,10 +4,46 @@ constant identifier="HDF5"; ascii[8] signature : dump; uint8 versionNumberOfSuperblock : dump; -if (versionNumberOfSuperblock == 0 || - versionNumberOfSuperblock == 1 || - versionNumberOfSuperblock == 2 || - versionNumberOfSuperblock == 3) +if (versionNumberOfSuperblock == 0 || versionNumberOfSuperblock == 1) +{ + uint8 versionNumOfFilesFreeSpaceStorage : dump; + uint8 versionNumOfRootGroupSymbolTableEntry : dump; + uint8 reserved1; + uint8 versionNumOfSharedHeaderMessageFormat: dump; + uint8 sizeOfOffsets : dump; + uint8 sizeOfLength : dump; + uint8 reserved2; + #uint32_little_endian groupLeafAndInternalNodeK : dump; + #uint32_little_endian fileConsistencyFlags : dump; + + #unsigned[4] groupLeafAndInternalNodeK : dump; + unsigned[2] groupLeafNodeK : dump; + unsigned[2] groupInternalNodeK : dump; + unsigned[4] fileConsistencyFlags : dump; + + if (versionNumberOfSuperblock == 1) { + uint16 indexedStorageInternalNodeK : dump; + uint16 reserved3; + } + + if(sizeOfOffsets == 8) { + uint64_little_endian baseAddress : dump; + uint64_little_endian addressOfFileFreeSpaceInfo : dump; + uint64_little_endian endOfFileAddress : dump; + uint64_little_endian driverInformationBlockAddress : dump; + uint64_little_endian rootGroupSymbolTableEntry: dump; + } + + if(sizeOfOffsets == 4) { + uint32_little_endian baseAddress : dump; + uint32_little_endian addressOfFileFreeSpaceInfo : dump; + uint32_little_endian endOfFileAddress : dump; + uint32_little_endian driverInformationBlockAddress : dump; + uint32_little_endian rootGroupSymbolTableEntry: dump; + } +} + +if (versionNumberOfSuperblock == 2 || versionNumberOfSuperblock == 3) { uint8 sizeOfOffsets : dump; uint8 sizeOfLength : dump; diff --git a/src/grib_io.c b/src/grib_io.c index 67f36527e..8b2fd8f0b 100644 --- a/src/grib_io.c +++ b/src/grib_io.c @@ -491,15 +491,11 @@ static int read_HDF5_offset(reader *r, int length, unsigned long* v, unsigned ch return 0; } -static int check_hdf5_superblock_version(unsigned char version) -{ - if (version==0 || version==1 || version==2 || version==3) return GRIB_SUCCESS; - return GRIB_INVALID_MESSAGE; -} + static int read_HDF5(reader *r) { /* See: http://www.hdfgroup.org/HDF5/doc/H5.format.html#Superblock */ - unsigned char tmp[36]; /* Should be enough */ + unsigned char tmp[49]; /* Should be enough */ unsigned char buf[4]; unsigned char version_of_superblock, size_of_offsets, size_of_lengths, consistency_flags; @@ -533,51 +529,100 @@ static int read_HDF5(reader *r) } tmp[i++] = version_of_superblock; - err = check_hdf5_superblock_version(version_of_superblock); - if(err) { + + if (version_of_superblock == 2 || version_of_superblock == 3) { + if( (r->read(r->read_data, &size_of_offsets, 1, &err) != 1) || err) { + return err; + } + + tmp[i++] = size_of_offsets; + + if(size_of_offsets > 8) { + grib_context_log(c, GRIB_LOG_ERROR,"read_HDF5: invalid size_of_offsets: %ld, only <= 8 is supported", (long)size_of_offsets); + return GRIB_NOT_IMPLEMENTED; + } + + if( (r->read(r->read_data, &size_of_lengths, 1, &err) != 1) || err) { + return err; + } + + tmp[i++] = size_of_lengths; + + if( (r->read(r->read_data, &consistency_flags, 1, &err) != 1) || err) { + return err; + } + + tmp[i++] = consistency_flags; + + err = read_HDF5_offset(r, size_of_offsets, &base_address, tmp, &i); + if(err) { + return err; + } + + err = read_HDF5_offset(r, size_of_offsets, &superblock_extension_address, tmp, &i); + if(err) { + return err; + } + + err = read_HDF5_offset(r, size_of_offsets, &end_of_file_address, tmp, &i); + if(err) { + return err; + } + } else if (version_of_superblock == 0 || version_of_superblock == 1) { + char skip[4]; + unsigned long file_free_space_info; + unsigned char version_of_file_free_space, version_of_root_group_symbol_table, version_number_shared_header, ch; + + if( (r->read(r->read_data, &version_of_file_free_space, 1, &err) != 1) || err) return err; + tmp[i++] = version_of_file_free_space; + + if( (r->read(r->read_data, &version_of_root_group_symbol_table, 1, &err) != 1) || err) return err; + tmp[i++] = version_of_root_group_symbol_table; + + if( (r->read(r->read_data, &ch, 1, &err) != 1) || err) return err; /* reserved */ + tmp[i++] = ch; + + if( (r->read(r->read_data, &version_number_shared_header, 1, &err) != 1) || err) return err; + tmp[i++] = version_number_shared_header; + + if( (r->read(r->read_data, &size_of_offsets, 1, &err) != 1) || err) return err; + tmp[i++] = size_of_offsets; + if (size_of_offsets > 8) { + grib_context_log(c, GRIB_LOG_ERROR,"read_HDF5: invalid size_of_offsets: %ld, only <= 8 is supported", (long)size_of_offsets); + return GRIB_NOT_IMPLEMENTED; + } + + if( (r->read(r->read_data, &size_of_lengths, 1, &err) != 1) || err) return err; + tmp[i++] = size_of_lengths; + + if( (r->read(r->read_data, &ch, 1, &err) != 1) || err) return err; /*reserved*/ + tmp[i++] = ch; + + if( (r->read(r->read_data, &skip, 4, &err) != 4) || err) return err; /* Group Leaf/Internal Node K: 4 bytes */ + tmp[i++] = skip[0]; tmp[i++] = skip[1]; tmp[i++] = skip[2]; tmp[i++] = skip[3]; + + if( (r->read(r->read_data, &skip, 4, &err) != 4) || err) return err; /* consistency_flags: 4 bytes */ + tmp[i++] = skip[0]; tmp[i++] = skip[1]; tmp[i++] = skip[2]; tmp[i++] = skip[3]; + + if (version_of_superblock == 1) { + /* Indexed storage internal node K and reserved: only in version 1 of superblock */ + if( (r->read(r->read_data, &skip, 4, &err) != 4) || err) return err; + tmp[i++] = skip[0]; tmp[i++] = skip[1]; tmp[i++] = skip[2]; tmp[i++] = skip[3]; + } + + err = read_HDF5_offset(r, size_of_offsets, &base_address, tmp, &i); + if (err) return err; + + err = read_HDF5_offset(r, size_of_offsets, &file_free_space_info, tmp, &i); + if (err) return err; + + err = read_HDF5_offset(r, size_of_offsets, &end_of_file_address, tmp, &i); + if (err) return err; + } else { grib_context_log(c, GRIB_LOG_ERROR,"read_HDF5: invalid version of superblock: %ld", (long)version_of_superblock); return GRIB_NOT_IMPLEMENTED; } - if( (r->read(r->read_data, &size_of_offsets, 1, &err) != 1) || err) { - return err; - } - - tmp[i++] = size_of_offsets; - - - if(size_of_offsets > 8) { - grib_context_log(c, GRIB_LOG_ERROR,"read_HDF5: invalid size_of_offsets: %ld, only <= 8 is supported", (long)size_of_offsets); - return GRIB_NOT_IMPLEMENTED; - } - - if( (r->read(r->read_data, &size_of_lengths, 1, &err) != 1) || err) { - return err; - } - - tmp[i++] = size_of_lengths; - - if( (r->read(r->read_data, &consistency_flags, 1, &err) != 1) || err) { - return err; - } - - tmp[i++] = consistency_flags; - - err = read_HDF5_offset(r, size_of_offsets, &base_address, tmp, &i); - if(err) { - return err; - } - - err = read_HDF5_offset(r, size_of_offsets, &superblock_extension_address, tmp, &i); - if(err) { - return err; - } - - err = read_HDF5_offset(r, size_of_offsets, &end_of_file_address, tmp, &i); - if(err) { - return err; - } - Assert(i <= sizeof(tmp)); return read_the_rest(r, end_of_file_address, tmp, i, 0); }