From aebe776fc4a0ff955d6ddf2f659a33eefb4efb1a Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Thu, 3 Aug 2017 14:44:26 +0100 Subject: [PATCH] add string encoding for big endian-platforms (SUP-2121) --- src/grib_api_prototypes.h | 2 +- src/grib_bits.c | 37 +++--- src/grib_bits_any_endian.c | 14 +-- src/grib_bits_fast_big_endian.c | 209 ++++++++++++++++++-------------- 4 files changed, 138 insertions(+), 124 deletions(-) diff --git a/src/grib_api_prototypes.h b/src/grib_api_prototypes.h index 846aea9ce..a05d15778 100644 --- a/src/grib_api_prototypes.h +++ b/src/grib_api_prototypes.h @@ -1503,7 +1503,7 @@ const char *grib_get_git_sha1(void); /* grib_bits_any_endian.c */ int grib_is_all_bits_one(long val, long nbits); -int grib_encode_string(const unsigned char *bitStream, long *bitOffset, size_t numberOfCharacters, char *string); +int grib_encode_string(unsigned char *bitStream, long *bitOffset, size_t numberOfCharacters, const char *string); char *grib_decode_string(const unsigned char *bitStream, long *bitOffset, size_t numberOfCharacters, char *string); unsigned long grib_decode_unsigned_long(const unsigned char *p, long *bitp, long nbits); int grib_encode_unsigned_long(unsigned char *p, unsigned long val, long *bitp, long nbits); diff --git a/src/grib_bits.c b/src/grib_bits.c index d18ffee68..82dc1df13 100644 --- a/src/grib_bits.c +++ b/src/grib_bits.c @@ -101,34 +101,36 @@ int grib_encode_signed_long(unsigned char* p, long val , long o, int l) return GRIB_SUCCESS; } -static void grib_set_bit_on( unsigned char* p, long* bitp){ - +static void grib_set_bit_on( unsigned char* p, long* bitp) +{ p += *bitp/8; *p |= (1u << (7-((*bitp)%8))); (*bitp)++; } -void grib_set_bits_on( unsigned char* p, long* bitp,long nbits){ +void grib_set_bits_on( unsigned char* p, long* bitp,long nbits) +{ int i; for (i=0;i> 3); return (*p&(1<<(7-(bitp%8)))); } -void grib_set_bit( unsigned char* p, long bitp, int val){ - +void grib_set_bit( unsigned char* p, long bitp, int val) +{ if(val == 0) grib_set_bit_off(p,&bitp); else grib_set_bit_on(p,&bitp); } @@ -166,18 +168,11 @@ int grib_encode_signed_longb(unsigned char* p, long val ,long *bitp, long nb) } #if GRIB_IBMPOWER67_OPT - -#include "grib_bits_ibmpow.c" - + #include "grib_bits_ibmpow.c" #else -#if FAST_BIG_ENDIAN - -#include "grib_bits_fast_big_endian.c" - -#else - -#include "grib_bits_any_endian.c" - -#endif - + #if FAST_BIG_ENDIAN + #include "grib_bits_fast_big_endian.c" + #else + #include "grib_bits_any_endian.c" + #endif #endif diff --git a/src/grib_bits_any_endian.c b/src/grib_bits_any_endian.c index 3166b5193..bb5feeb62 100644 --- a/src/grib_bits_any_endian.c +++ b/src/grib_bits_any_endian.c @@ -44,7 +44,7 @@ int grib_is_all_bits_one(long val, long nbits) return bits_all_one.v[nbits]==val; } -int grib_encode_string(const unsigned char* bitStream, long *bitOffset, size_t numberOfCharacters,char* string) +int grib_encode_string(unsigned char *bitStream, long *bitOffset, size_t numberOfCharacters, const char *string) { size_t i; int err=0; @@ -282,15 +282,9 @@ int grib_encode_unsigned_longb(unsigned char* p, unsigned long val ,long *bitp, } #if OMP_PACKING - -#include "grib_bits_any_endian_omp.c" - + #include "grib_bits_any_endian_omp.c" #elif VECTOR - -#include "grib_bits_any_endian_vector.c" - + #include "grib_bits_any_endian_vector.c" #else - -#include "grib_bits_any_endian_simple.c" - + #include "grib_bits_any_endian_simple.c" #endif diff --git a/src/grib_bits_fast_big_endian.c b/src/grib_bits_fast_big_endian.c index bde7dc667..f9918aad3 100644 --- a/src/grib_bits_fast_big_endian.c +++ b/src/grib_bits_fast_big_endian.c @@ -22,138 +22,163 @@ typedef struct bits_all_one_t { static bits_all_one_t bits_all_one={0,0,{0,}}; -static void init_bits_all_one() { - int size=sizeof(long)*8; - long* v=0; - unsigned long cmask=-1; - bits_all_one.size=size; - bits_all_one.inited=1; - v=bits_all_one.v+size; - *v= cmask << size; - while (size>0) *(--v)= ~(cmask << --size); - +static void init_bits_all_one() +{ + int size=sizeof(long)*8; + long* v=0; + unsigned long cmask=-1; + bits_all_one.size=size; + bits_all_one.inited=1; + v=bits_all_one.v+size; + *v= cmask << size; + while (size>0) *(--v)= ~(cmask << --size); } -int grib_is_all_bits_one(long val, long nbits) { - +int grib_is_all_bits_one(long val, long nbits) +{ if (!bits_all_one.inited) init_bits_all_one(); return bits_all_one.v[nbits]==val; - } +int grib_encode_string(unsigned char *bitStream, long *bitOffset, size_t numberOfCharacters, const char *string) +{ + size_t i; + int err=0; + long byteOffset = *bitOffset / 8; + int remainder = *bitOffset % 8; + unsigned char* p; + int remainderComplement=8-remainder; + const char *s=string; + + Assert(numberOfCharacters<512 && (numberOfCharacters == 0 || string)); + + /* if (remainder) byteOffset++; */ + + if (numberOfCharacters > 0) { + unsigned char *restrict p = bitStream+byteOffset; + if ( remainder == 0 ) { + memcpy(p, string, numberOfCharacters); + } else { + unsigned char mask = (unsigned char)(0xFF00 >> remainder); + unsigned char accum = p[0]; + for (size_t i=0; i < numberOfCharacters; ++i) { + accum |= (s[i] >>remainder) & ~mask; + p[i] = accum; + accum = (s[i] << remainderComplement) & mask; + } + p[numberOfCharacters] = accum; + } + } + *bitOffset+=numberOfCharacters*8; + return err; +} char* grib_decode_string(const unsigned char* bitStream, long *bitOffset, size_t numberOfCharacters,char* string) { - size_t i; - long byteOffset = *bitOffset / 8; - int remainder = *bitOffset % 8; - unsigned char c; - unsigned char* p; - char* s=string; - unsigned char mask[] ={ 0, 0x80, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC, 0xFE }; - int remainderComplement=8-remainder; + size_t i; + long byteOffset = *bitOffset / 8; + int remainder = *bitOffset % 8; + unsigned char c; + unsigned char* p; + char* s=string; + unsigned char mask[] ={ 0, 0x80, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC, 0xFE }; + int remainderComplement=8-remainder; - if (numberOfCharacters==0) return string; + if (numberOfCharacters==0) return string; - p=(unsigned char*)bitStream+byteOffset; + p=(unsigned char*)bitStream+byteOffset; - if ( remainder == 0 ) { - memcpy(string,bitStream+byteOffset,numberOfCharacters); + if ( remainder == 0 ) { + memcpy(string,bitStream+byteOffset,numberOfCharacters); + *bitOffset+=numberOfCharacters*8; + return string; + } + + for (i=0;i>remainderComplement ); + s++; + } *bitOffset+=numberOfCharacters*8; return string; - } - - for (i=0;i>remainderComplement ); - s++; - } - *bitOffset+=numberOfCharacters*8; - return string; } unsigned long grib_decode_unsigned_long(const unsigned char* p, long *bitp, long nbits) { - long countOfLeftmostBits=0,leftmostBits=0; - long startBit= *bitp ; - long remainingBits = nbits; - long *pp=(long*)p; - unsigned long val=0; + long countOfLeftmostBits=0,leftmostBits=0; + long startBit= *bitp ; + long remainingBits = nbits; + long *pp=(long*)p; + unsigned long val=0; - if (startBit >= max_nbits) { - pp += startBit/max_nbits; - startBit %= max_nbits; - } + if (startBit >= max_nbits) { + pp += startBit/max_nbits; + startBit %= max_nbits; + } - countOfLeftmostBits = startBit + remainingBits; - if (countOfLeftmostBits > max_nbits) { - countOfLeftmostBits = max_nbits - startBit; - remainingBits -= countOfLeftmostBits; - leftmostBits=(VALUE(*pp,startBit,countOfLeftmostBits)) << remainingBits; - startBit = 0; - pp++; - } else - leftmostBits = 0; + countOfLeftmostBits = startBit + remainingBits; + if (countOfLeftmostBits > max_nbits) { + countOfLeftmostBits = max_nbits - startBit; + remainingBits -= countOfLeftmostBits; + leftmostBits=(VALUE(*pp,startBit,countOfLeftmostBits)) << remainingBits; + startBit = 0; + pp++; + } else + leftmostBits = 0; - val=leftmostBits+(VALUE(*pp,startBit,remainingBits)); + val=leftmostBits+(VALUE(*pp,startBit,remainingBits)); - *bitp += nbits; + *bitp += nbits; - return val; + return val; } int grib_encode_unsigned_long(unsigned char* p, unsigned long val ,long *bitp, long nbits) { - long* destination = (long*)p; - long countOfLeftmostBits=0,nextWord=0,startBit=0,remainingBits=0,rightmostBits=0; + long* destination = (long*)p; + long countOfLeftmostBits=0,nextWord=0,startBit=0,remainingBits=0,rightmostBits=0; - startBit = *bitp; - remainingBits = nbits; + startBit = *bitp; + remainingBits = nbits; - if (startBit >= max_nbits) { - nextWord = startBit / max_nbits; - startBit %= max_nbits; - } else - nextWord = 0; + if (startBit >= max_nbits) { + nextWord = startBit / max_nbits; + startBit %= max_nbits; + } else + nextWord = 0; - countOfLeftmostBits = startBit + remainingBits; - if (countOfLeftmostBits > max_nbits) { - countOfLeftmostBits = max_nbits - startBit; - startBit = max_nbits - remainingBits; - remainingBits -= countOfLeftmostBits; + countOfLeftmostBits = startBit + remainingBits; + if (countOfLeftmostBits > max_nbits) { + countOfLeftmostBits = max_nbits - startBit; + startBit = max_nbits - remainingBits; + remainingBits -= countOfLeftmostBits; + destination[nextWord] = + ((destination[nextWord] >> countOfLeftmostBits) << countOfLeftmostBits) + + (VALUE(val,startBit,countOfLeftmostBits)); + startBit = 0; + nextWord++; + } + + rightmostBits = VALUE(val,max_nbits-remainingBits,remainingBits); destination[nextWord] = - ((destination[nextWord] >> countOfLeftmostBits) << countOfLeftmostBits) - + (VALUE(val,startBit,countOfLeftmostBits)); - startBit = 0; - nextWord++; - } + (destination[nextWord] & ~MASKVALUE(startBit,remainingBits)) + + (rightmostBits << max_nbits-(remainingBits+startBit)); - rightmostBits = VALUE(val,max_nbits-remainingBits,remainingBits); - destination[nextWord] = - (destination[nextWord] & ~MASKVALUE(startBit,remainingBits)) - + (rightmostBits << max_nbits-(remainingBits+startBit)); - - *bitp+=nbits; - return GRIB_SUCCESS; + *bitp+=nbits; + return GRIB_SUCCESS; } int grib_encode_unsigned_longb(unsigned char* p, unsigned long val ,long *bitp, long nbits) { - return grib_encode_unsigned_long(p,val ,bitp, nbits); + return grib_encode_unsigned_long(p,val ,bitp, nbits); } #if VECTOR - -#include "grib_bits_fast_big_endian_vector.c" - + #include "grib_bits_fast_big_endian_vector.c" #elif OMP - -#include "grib_bits_fast_big_endian_omp.c" - + #include "grib_bits_fast_big_endian_omp.c" #else - -#include "grib_bits_fast_big_endian_simple.c" - + #include "grib_bits_fast_big_endian_simple.c" #endif