diff --git a/.gitignore b/.gitignore deleted file mode 100644 index f3e32e270..000000000 --- a/.gitignore +++ /dev/null @@ -1,318 +0,0 @@ -# Generated stuff from libtool, automake etc -Makefile -Makefile.in -*.lo -*.la -.deps/ -/libtool -config.h.in -lex.yy.c -stamp-h1 -src/config.h -src/eccodes_version.h -src/grib_api.h.new -src/grib_api_constants.h.new -src/eccodes_constants.h.new -src/grib_errors.c.new -src/grib_errors.h.new -src/errors.py.new -config/test-driver -autom4te.cache/ -config.log -config.status -configure -grib_api.spec -grib_api.pc -grib_api_f90.pc -perl/GRIB-API/Makefile.PL -rpms/eccodes.pc -rpms/eccodes.spec -rpms/eccodes_f90.pc -aclocal.m4 -src/config.h.in~ - -python/eccodes.py -python/eccodes/__init__.pyc -python/eccodes/eccodes.pyc -python/gribapi.py -python/gribapi/__init__.pyc -python/gribapi/gribapi.pyc -python/gribapi_swig.pyc - -# Generated stuff from builds -src/.libs/ -tools/.libs/ -tools/big2gribex -tools/gaussian -tools/gg_sub_area_check -tools/grib2ppm -tools/grib_compare -tools/grib_copy -tools/grib_count -tools/bufr_count -tools/codes_count -tools/codes_bufr_filter -tools/grib_dump -tools/grib_filter -tools/grib_get -tools/grib_get_data -tools/grib_histogram -tools/grib_index_build -tools/gts_copy -tools/gts_compare -tools/gts_get -tools/metar_compare -tools/gts_dump -tools/gts_filter -tools/gts_ls -tools/codes_info -tools/grib_list_keys -tools/grib_ls -tools/grib_merge -tools/grib_repair -tools/grib_set -tools/grib_to_netcdf -tools/mars_request -tools/codes_parser -tools/xref -tools/grib1to2 -tools/bufr_compare -tools/bufr_copy -tools/bufr_dump -tools/bufr_get -tools/bufr_index_build -tools/bufr_ls -tools/bufr_set -tools/grib_to_json -tools/metar_dump -tools/metar_filter -tools/metar_get -tools/metar_ls -tools/metar_copy -tools/taf_dump -tools/taf_filter -tools/taf_get -tools/taf_ls -examples/C/.libs/ -examples/C/test-suite.log -examples/C/c_box -examples/C/c_bufr_clone -examples/C/c_bufr_missing -examples/C/c_bufr_attributes -examples/C/c_bufr_keys_iterator -examples/C/c_bufr_set_keys -examples/C/c_bufr_expanded -examples/C/c_bufr_read_header -examples/C/c_bufr_read_temp -examples/C/c_bufr_read_synop -examples/C/c_bufr_get_keys -examples/C/c_bufr_read_scatterometer -examples/C/c_bufr_subset -examples/C/c_grib_ensemble_index -examples/C/c_grib_clone -examples/C/c_grib_iterator_bitmap -examples/C/c_new_sample -examples/C/c_grib_get_keys -examples/C/c_grib_get_data -examples/C/c_get_product_kind -examples/C/c_grib_iterator -examples/C/c_grib_index -examples/C/c_grib_keys_iterator -examples/C/c_large_grib1 -examples/C/c_grib_list -examples/C/c_mars_param -examples/C/c_grib_multi -examples/C/c_multi2 -examples/C/c_grib_multi_write -examples/C/c_grib_nearest -examples/C/c_points -examples/C/c_grib_precision -examples/C/c_grib_print_data -examples/C/c_sections_copy -examples/C/c_grib_set_keys -examples/C/c_grib_set_bitmap -examples/C/c_grib_set_data -examples/C/c_set_missing -examples/C/c_grib_set_pv -examples/C/c_values_check -examples/C/*.sh.log -examples/C/*.sh.trs -examples/F90/eccodes_f_bufr_expanded -examples/F90/eccodes_f_bufr_read_synop -examples/F90/eccodes_f_bufr_read_temp -examples/F90/eccodes_f_bufr_attributes -examples/F90/eccodes_f_bufr_set_keys -examples/F90/eccodes_f_bufr_get_keys -examples/F90/eccodes_f_bufr_get_string_array -examples/F90/eccodes_f_bufr_keys_iterator -examples/F90/eccodes_f_bufr_subset -examples/F90/eccodes_f_bufr_clone -examples/F90/eccodes_f_bufr_read_header -examples/F90/eccodes_f_bufr_read_scatterometer -examples/F90/eccodes_f_bufr_read_tropical_cyclone -examples/F90/eccodes_f_grib_clone -examples/F90/eccodes_f_grib_copy_namespace -examples/F90/eccodes_f_grib_count_messages -examples/F90/eccodes_f_grib_count_messages_multi -examples/F90/eccodes_f_grib_copy_message -examples/F90/eccodes_f_copy_namespace -examples/F90/eccodes_f_count_messages -examples/F90/eccodes_f_grib_get_keys -examples/F90/eccodes_f_grib_get_data -examples/F90/eccodes_f_get_product_kind -examples/F90/eccodes_f_get_pl -examples/F90/eccodes_f_get_pv -examples/F90/eccodes_f_get_set_uuid -examples/F90/eccodes_f_grib_index -examples/F90/eccodes_f_grib_keys_iterator -examples/F90/eccodes_f_grib_multi -examples/F90/eccodes_f_grib_multi_write -examples/F90/eccodes_f_grib_nearest -examples/F90/eccodes_f_new_from_file -examples/F90/eccodes_f_grib_precision -examples/F90/eccodes_f_grib_print_data -examples/F90/eccodes_f_grib_print_data_static -examples/F90/eccodes_f_read_from_file -examples/F90/eccodes_f_read_message -examples/F90/eccodes_f_grib_samples -examples/F90/eccodes_f_grib_set_keys -examples/F90/eccodes_f_grib_set_bitmap -examples/F90/eccodes_f_grib_set_gvc -examples/F90/eccodes_f_grib_set_missing -examples/F90/eccodes_f_grib_set_pv -examples/F90/*.sh.log -examples/F90/*.sh.trs -examples/F90/test-suite.log -examples/python/.libs/ -examples/python/my.idx -examples/python/p_count_messages -examples/python/p_grib_count_messages -examples/python/p_grib_iterator -examples/python/p_grib_keys_iterator -examples/python/p_grib_print_data -examples/python/*.sh.log -examples/python/*.sh.trs -examples/python/test-suite.log -fortran/.libs/ -fortran/*.mod -fortran/grib_f90.f90 -fortran/eccodes_f90.f90 -fortran/grib_kinds.h -fortran/grib_types -fortran/same_int_long -fortran/same_int_size_t -python/.libs/ -python/gribapi_swig.py -python/gribapi_swig_wrap.c -python/eccode_swig.py -python/eccode_swig_wrap.c -python/setup.py -tests/.libs/ -tests/*.sh.log -tests/*.sh.trs -tests/test-suite.log -tests/bpv_limit -tests/grib_double_cmp -tests/gauss_sub -tests/gribex_perf -tests/index -tests/jpeg_perf -tests/julian -tests/laplacian -tests/multi_from_message -tests/pack_unpack -tests/packing -tests/packing_check -tests/png_perf -tests/read_any -tests/read_index -tests/so_perf -tests/unit_tests -tests/ccsds_perf -tests/grib_util_set_spec -tigge/.libs/ -tigge/tigge_accumulations -tigge/tigge_check -tigge/tigge_name -tigge/tigge_split - -# IFS samples -ifs_samples/*/*.tmpl - -# compiled source # -################### -*.com -*.dll -*.exe -*.o -*.so - -# Packages # -############ -*.7z -*.dmg -*.gz -*.iso -*.jar -*.rar -*.tar -*.zip - -# Logs and databases # -###################### -*.sql -*.sqlite - -# OS generated files # -###################### -.DS_Store -.DS_Store? -._* -.Spotlight-V100 -.Trashes -Icon? -ehthumbs.db -Thumbs.db -.directory - -src/tags - -data/bufr/*.ref -data/bufr/*.test -data/bufr/*.json -data/bufr/*.no -data/bufr/*.bufr -data/gts/*.DAT -data/gts/*.ref -data/metar/*.txt -data/metar/*.ref -data/.downloaded -data/budg -data/*.grib -data/*.grib2 -data/*.grib1 -data/tigge/tigge*.grib -data/exp/ - -CMakeLists.txt.user* - -#some config -myconfig* -myconfigMem* -myconfigO* -myconfigemos* -myconfigemosgprof* -myconfiggprof* -myconfignopy* -INSTALL -share/ -lib -include - -data/bufr/*diff -data/bufr/*decode -data/bufr/*test - -*.sublime-workspace -*.old - diff --git a/examples/python/high_level_api.py b/examples/python/high_level_api.py index a41db3920..90edea99c 100644 --- a/examples/python/high_level_api.py +++ b/examples/python/high_level_api.py @@ -1,9 +1,9 @@ #!/bin/env python """ -Unit tests for ``PythonicGrib``. +Unit tests for high level Python interface. -Author: Daniel Lee, DWD, 2014 +Author: Daniel Lee, DWD, 2016 """ import os @@ -14,10 +14,11 @@ from eccodes import GribFile from eccodes import GribIndex from eccodes import GribMessage from eccodes.high_level.gribmessage import IndexNotSelectedError - +from eccodes import BufrFile, BufrMessage TESTGRIB = "../../data/high_level_api.grib2" -TEST_OUTPUT = "test-output.grib" +TESTBUFR = "../../data/bufr/syno_multi.bufr" +TEST_OUTPUT = "test-output.codes" TEST_INDEX = "test.index" TEST_KEYS = ("dataDate", "stepRange") TEST_VALUES = 20110225, 0 @@ -26,118 +27,164 @@ for i1 in range(len(TEST_KEYS)): SELECTION_DICTIONARY[TEST_KEYS[i1]] = TEST_VALUES[i1] TEST_INDEX_OUTPUT = TESTGRIB TEST_STEPRANGE = ('0', '12', '18', '24', '6') -# These keys should be available even if new keys become available -KNOWN_GRIB_KEYS = ['globalDomain', 'GRIBEditionNumber', - 'tablesVersionLatest', 'grib2divider', 'angularPrecision', - 'missingValue', 'ieeeFloats', 'isHindcast', - 'section0Length', 'identifier', 'discipline', - 'editionNumber', 'totalLength', 'sectionNumber', - 'section1Length', 'numberOfSection', 'centre', - 'centreDescription', 'subCentre', 'tablesVersion', - 'masterDir', 'localTablesVersion', 'localDir', - 'significanceOfReferenceTime', 'year', 'month', 'day', - 'hour', 'minute', 'second', 'dataDate', 'julianDay', - 'dataTime', 'productionStatusOfProcessedData', - 'typeOfProcessedData', 'md5Section1', - 'selectStepTemplateInterval', 'selectStepTemplateInstant', - 'stepType', 'setCalendarId', 'deleteCalendarId', - 'is_uerra', 'sectionNumber', 'grib2LocalSectionPresent', - 'section2Length', 'numberOfSection', 'addEmptySection2', - 'grib2LocalSectionNumber', 'marsClass', 'marsType', - 'marsStream', 'experimentVersionNumber', 'class', 'type', - 'stream', 'productDefinitionTemplateNumberInternal', - 'localDefinitionNumber', 'eps', 'oceanAtmosphereCoupling', - 'legBaseDate', 'legBaseTime', 'legNumber', - 'referenceDate', 'climateDateFrom', 'climateDateTo', - 'addExtraLocalSection', 'deleteExtraLocalSection', - 'extraLocalSectionPresent', 'section2Padding', - 'sectionNumber', 'gridDescriptionSectionPresent', - 'section3Length', 'numberOfSection', - 'sourceOfGridDefinition', 'numberOfDataPoints', - 'numberOfOctectsForNumberOfPoints', - 'interpretationOfNumberOfPoints', 'PLPresent', +# These keys should be available even if new keys are defined +KNOWN_GRIB_KEYS = ['7777', 'EPS information', 'GRIBEditionNumber', 'N', 'NV', + 'Ni', 'Nj', 'PLPresent', 'PVPresent', + 'Parameter information', 'addEmptySection2', + 'addExtraLocalSection', 'alternativeRowScanning', + 'angleDivisor', 'angleMultiplier', 'angularPrecision', + 'average', 'backgroundProcess', + 'basicAngleOfTheInitialProductionDomain', + 'binaryScaleFactor', 'bitMapIndicator', 'bitmapPresent', + 'bitsPerValue', 'bottomLevel', 'centre', + 'centreDescription', 'cfName', 'cfNameECMF', 'cfVarName', + 'cfVarNameECMF', 'changeDecimalPrecision', 'class', + 'climateDateFrom', 'climateDateTo', 'codedValues', + 'dataDate', 'dataRepresentationTemplateNumber', 'dataTime', + 'day', 'decimalPrecision', 'decimalScaleFactor', + 'deleteCalendarId', 'deleteExtraLocalSection', 'deletePV', + 'discipline', 'distinctLatitudes', 'distinctLongitudes', + 'editionNumber', 'endStep', 'eps', + 'experimentVersionNumber', 'extraLocalSectionPresent', + 'forecastTime', 'g2grid', 'gaussianGridName', + 'genVertHeightCoords', 'generatingProcessIdentifier', + 'getNumberOfValues', 'global', 'globalDomain', + 'grib 2 Section 5 DATA REPRESENTATION SECTION', + 'grib 2 Section 6 BIT-MAP SECTION', 'grib 2 Section 7 data', + 'grib2LocalSectionNumber', 'grib2LocalSectionPresent', + 'grib2divider', 'gridDefinitionDescription', 'gridDefinitionTemplateNumber', - 'gridDefinitionDescription', 'shapeOfTheEarth', - 'scaleFactorOfRadiusOfSphericalEarth', - 'scaledValueOfRadiusOfSphericalEarth', - 'scaleFactorOfEarthMajorAxis', - 'scaledValueOfEarthMajorAxis', - 'scaleFactorOfEarthMinorAxis', - 'scaledValueOfEarthMinorAxis', 'radius', 'Ni', 'Nj', - 'basicAngleOfTheInitialProductionDomain', 'mBasicAngle', - 'angleMultiplier', 'mAngleMultiplier', - 'subdivisionsOfBasicAngle', 'angleDivisor', - 'latitudeOfFirstGridPoint', 'longitudeOfFirstGridPoint', + 'gridDescriptionSectionPresent', 'gridType', 'hour', + 'hoursAfterDataCutoff', 'iDirectionIncrement', + 'iDirectionIncrementGiven', 'iDirectionIncrementInDegrees', + 'iScansNegatively', 'iScansPositively', 'identifier', + 'ieeeFloats', 'ifsParam', 'ijDirectionIncrementGiven', + 'indicatorOfUnitOfTimeRange', + 'interpretationOfNumberOfPoints', 'isConstant', + 'isHindcast', 'isOctahedral', 'is_uerra', + 'jDirectionIncrementGiven', 'jPointsAreConsecutive', + 'jScansPositively', 'julianDay', 'kurtosis', 'latLonValues', + 'latitudeOfFirstGridPoint', + 'latitudeOfFirstGridPointInDegrees', + 'latitudeOfLastGridPoint', + 'latitudeOfLastGridPointInDegrees', 'latitudes', + 'legBaseDate', 'legBaseTime', 'legNumber', + 'lengthOfHeaders', 'level', 'localDefinitionNumber', + 'localDir', 'localTablesVersion', + 'longitudeOfFirstGridPoint', + 'longitudeOfFirstGridPointInDegrees', + 'longitudeOfLastGridPoint', + 'longitudeOfLastGridPointInDegrees', 'longitudes', + 'mAngleMultiplier', 'mBasicAngle', 'marsClass', + 'marsStream', 'marsType', 'masterDir', 'maximum', + 'md5Headers', 'md5Section1', 'md5Section3', 'md5Section4', + 'md5Section5', 'md5Section6', 'md5Section7', 'minimum', + 'minute', 'minutesAfterDataCutoff', 'missingValue', + 'modelName', 'month', 'name', 'nameECMF', + 'nameOfFirstFixedSurface', 'nameOfSecondFixedSurface', + 'neitherPresent', 'numberOfDataPoints', + 'numberOfForecastsInEnsemble', 'numberOfMissing', + 'numberOfOctectsForNumberOfPoints', 'numberOfSection', + 'numberOfValues', 'oceanAtmosphereCoupling', + 'offsetValuesBy', 'optimizeScaleFactor', 'packingError', + 'packingType', 'paramId', 'paramIdECMF', + 'parameterCategory', 'parameterName', 'parameterNumber', + 'parameterUnits', 'perturbationNumber', 'pressureUnits', + 'productDefinitionTemplateNumber', + 'productDefinitionTemplateNumberInternal', 'productType', + 'productionStatusOfProcessedData', 'radius', + 'referenceDate', 'referenceValue', 'referenceValueError', 'resolutionAndComponentFlags', 'resolutionAndComponentFlags1', 'resolutionAndComponentFlags2', - 'iDirectionIncrementGiven', 'jDirectionIncrementGiven', - 'uvRelativeToGrid', 'resolutionAndComponentFlags6', + 'resolutionAndComponentFlags6', 'resolutionAndComponentFlags7', 'resolutionAndComponentFlags8', - 'ijDirectionIncrementGiven', 'latitudeOfLastGridPoint', - 'longitudeOfLastGridPoint', 'iDirectionIncrement', 'N', - 'scanningMode', 'iScansNegatively', 'jScansPositively', - 'jPointsAreConsecutive', 'alternativeRowScanning', - 'iScansPositively', 'scanningMode5', 'scanningMode6', - 'scanningMode7', 'scanningMode8', 'g2grid', - 'latitudeOfFirstGridPointInDegrees', - 'longitudeOfFirstGridPointInDegrees', - 'latitudeOfLastGridPointInDegrees', - 'longitudeOfLastGridPointInDegrees', - 'iDirectionIncrementInDegrees', 'global', 'latLonValues', - 'latitudes', 'longitudes', 'distinctLatitudes', - 'distinctLongitudes', 'isOctahedral', 'gaussianGridName', - 'section3Padding', 'gridType', 'md5Section3', - 'sectionNumber', 'section4Length', 'numberOfSection', - 'NV', 'neitherPresent', 'productDefinitionTemplateNumber', - 'genVertHeightCoords', 'Parameter information', - 'parameterCategory', 'parameterNumber', 'parameterUnits', - 'parameterName', 'typeOfGeneratingProcess', - 'backgroundProcess', 'generatingProcessIdentifier', - 'hoursAfterDataCutoff', 'minutesAfterDataCutoff', - 'indicatorOfUnitOfTimeRange', 'x', 'stepUnits', - 'forecastTime', 'startStep', 'endStep', 'stepRange', - 'stepTypeInternal', 'validityDate', 'validityTime', - 'typeOfFirstFixedSurface', 'unitsOfFirstFixedSurface', - 'nameOfFirstFixedSurface', + 'scaleFactorOfEarthMajorAxis', + 'scaleFactorOfEarthMinorAxis', 'scaleFactorOfFirstFixedSurface', + 'scaleFactorOfRadiusOfSphericalEarth', + 'scaleFactorOfSecondFixedSurface', 'scaleValuesBy', + 'scaledValueOfEarthMajorAxis', + 'scaledValueOfEarthMinorAxis', 'scaledValueOfFirstFixedSurface', - 'typeOfSecondFixedSurface', 'unitsOfSecondFixedSurface', - 'nameOfSecondFixedSurface', - 'scaleFactorOfSecondFixedSurface', - 'scaledValueOfSecondFixedSurface', 'pressureUnits', - 'typeOfLevel', 'level', 'bottomLevel', 'topLevel', - 'tempPressureUnits', 'EPS information', - 'typeOfEnsembleForecast', 'perturbationNumber', - 'numberOfForecastsInEnsemble', 'x', 'paramIdECMF', - 'paramId', 'shortNameECMF', 'shortName', 'unitsECMF', - 'units', 'nameECMF', 'name', 'cfNameECMF', 'cfName', - 'cfVarNameECMF', 'cfVarName', 'modelName', 'ifsParam', - 'PVPresent', 'deletePV', 'md5Section4', 'lengthOfHeaders', - 'md5Headers', 'sectionNumber', - 'grib 2 Section 5 DATA REPRESENTATION SECTION', - 'section5Length', 'numberOfSection', 'numberOfValues', - 'dataRepresentationTemplateNumber', 'packingType', - 'referenceValue', 'referenceValueError', - 'binaryScaleFactor', 'decimalScaleFactor', - 'optimizeScaleFactor', 'bitsPerValue', - 'typeOfOriginalFieldValues', 'md5Section5', - 'sectionNumber', 'grib 2 Section 6 BIT-MAP SECTION', - 'section6Length', 'numberOfSection', 'bitMapIndicator', - 'bitmapPresent', 'md5Section6', 'sectionNumber', - 'grib 2 Section 7 data', 'section7Length', - 'numberOfSection', 'codedValues', 'values', - 'packingError', 'unpackedError', 'maximum', 'minimum', - 'average', 'numberOfMissing', 'standardDeviation', - 'skewness', 'kurtosis', 'isConstant', - 'changeDecimalPrecision', 'decimalPrecision', - 'setBitsPerValue', 'getNumberOfValues', 'scaleValuesBy', - 'offsetValuesBy', 'productType', 'md5Section7', - 'section8Length', '7777'] + 'scaledValueOfRadiusOfSphericalEarth', + 'scaledValueOfSecondFixedSurface', 'scanningMode', + 'scanningMode5', 'scanningMode6', 'scanningMode7', + 'scanningMode8', 'second', 'section0Length', + 'section1Length', 'section2Length', 'section2Padding', + 'section3Length', 'section3Padding', 'section4Length', + 'section5Length', 'section6Length', 'section7Length', + 'section8Length', 'sectionNumber', + 'selectStepTemplateInstant', 'selectStepTemplateInterval', + 'setBitsPerValue', 'setCalendarId', 'shapeOfTheEarth', + 'shortName', 'shortNameECMF', 'significanceOfReferenceTime', + 'skewness', 'sourceOfGridDefinition', 'standardDeviation', + 'startStep', 'stepRange', 'stepType', 'stepTypeInternal', + 'stepUnits', 'stream', 'subCentre', + 'subdivisionsOfBasicAngle', 'tablesVersion', + 'tablesVersionLatest', 'tempPressureUnits', 'topLevel', + 'totalLength', 'type', 'typeOfEnsembleForecast', + 'typeOfFirstFixedSurface', 'typeOfGeneratingProcess', + 'typeOfLevel', 'typeOfOriginalFieldValues', + 'typeOfProcessedData', 'typeOfSecondFixedSurface', 'units', + 'unitsECMF', 'unitsOfFirstFixedSurface', + 'unitsOfSecondFixedSurface', 'unpackedError', + 'uvRelativeToGrid', 'validityDate', 'validityTime', + 'values', 'x', 'year'] +KNOWN_BUFR_KEYS = ['3HourPressureChange', '7777', 'BUFRstr', + 'airTemperatureAt2M', 'blockNumber', 'bufrHeaderCentre', + 'bufrHeaderSubCentre', 'bufrTemplate', + 'bufrdcExpandedDescriptors', 'centre', + 'characteristicOfPressureTendency', 'cloudAmount', + 'cloudCoverTotal', 'cloudType', 'compressedData', + 'corr1Data', 'corr2Data', 'corr3Data', 'corr4Data', + 'correction1', 'correction1Part', 'correction2', + 'correction2Part', 'correction3', 'correction3Part', + 'correction4', 'correction4Part', 'createNewData', + 'dataCategory', 'dataPresentIndicator', 'dataSubCategory', + 'day', 'defaultSequence', 'dewpointTemperatureAt2M', 'ed', + 'edition', 'expandedAbbreviations', 'expandedCodes', + 'expandedCrex_scales', 'expandedCrex_units', + 'expandedCrex_widths', 'expandedNames', + 'expandedOriginalCodes', 'expandedOriginalReferences', + 'expandedOriginalScales', 'expandedOriginalWidths', + 'expandedTypes', 'expandedUnits', 'generatingApplication', + 'globalDomain', 'heightOfBaseOfCloud', 'heightOfStation', + 'horizontalVisibility', 'hour', 'isSatellite', + 'isSatelliteType', 'latitude', 'lengthDescriptors', + 'localDay', 'localHour', 'localLatitude', 'localLongitude', + 'localMinute', 'localMonth', 'localSecond', + 'localSectionPresent', 'localTablesVersionNumber', + 'localYear', 'longitude', 'masterTableNumber', + 'masterTablesVersionNumber', 'md5Data', 'md5Structure', + 'messageLength', 'minute', 'month', 'nonCoordinatePressure', + 'numberOfSubsets', 'numberOfUnexpandedDescriptors', + 'observedData', 'operator', 'pastWeather1', 'pastWeather2', + 'presentWeather', 'pressureReducedToMeanSeaLevel', + 'qualityControl', 'rdbSubtype', 'rdbType', 'rdbtime', + 'rdbtimeDay', 'rdbtimeHour', 'rdbtimeMinute', + 'rdbtimeSecond', 'rectime', 'rectimeDay', 'rectimeHour', + 'rectimeMinute', 'rectimeSecond', 'relativeHumidity', + 'reservedSection2', 'reservedSection3', 'section1Length', + 'section1Padding', 'section2Length', 'section2Padding', + 'section3Flags', 'section3Length', 'section3Padding', + 'section4Length', 'section4Padding', 'section5Length', + 'sequences', 'spare', 'spare1', 'stationNumber', + 'stationType', 'subsetNumber', 'tableNumber', + 'templatesLocalDir', 'templatesMasterDir', 'totalLength', + 'totalPrecipitationPast6Hours', 'totalSnowDepth', + 'typicalCentury', 'typicalDate', 'typicalDay', + 'typicalHour', 'typicalMinute', 'typicalMonth', + 'typicalSecond', 'typicalTime', 'typicalYear', + 'typicalYearOfCentury', 'unexpandedDescriptors', + 'updateSequenceNumber', + 'verticalSignificanceSurfaceObservations', + 'windDirectionAt10M', 'windSpeedAt10M', 'year'] class TestGribFile(unittest.TestCase): + """Test GribFile functionality.""" def test_memory_management(self): @@ -179,6 +226,7 @@ class TestGribFile(unittest.TestCase): class TestGribMessage(unittest.TestCase): + """Test GribMessage functionality.""" def test_metadata(self): @@ -224,6 +272,7 @@ class TestGribMessage(unittest.TestCase): class TestGribIndex(unittest.TestCase): + """Test GribIndex functionality.""" def test_memory_management(self): @@ -255,5 +304,95 @@ class TestGribIndex(unittest.TestCase): idx.select(SELECTION_DICTIONARY) self.assertEqual(len(idx.open_messages), 1) + +class TestBufrFile(unittest.TestCase): + + """Test BufrFile functionality.""" + + def test_memory_management(self): + """Messages in BufrFile can be opened and closed properly.""" + with BufrFile(TESTBUFR) as bufr: + self.assertEqual(len(bufr), 3) + for i in range(len(bufr)): + msg = BufrMessage(bufr) + self.assertEqual(msg["bufrHeaderCentre"], 98) + self.assertEqual(len(bufr.open_messages), 3) + self.assertEquals(len(bufr.open_messages), 0) + + def test_message_counting_works(self): + """The BufrFile is aware of its messages.""" + with BufrFile(TESTBUFR) as bufr: + msg_count = len(bufr) + self.assertEqual(msg_count, 3) + + def test_iterator_protocol(self): + """The BufrFile allows pythonic iteration over all messages.""" + latitudes = [] + with BufrFile(TESTBUFR) as bufr: + for msg in bufr: + latitudes.append(msg["localLatitude"]) + self.assertSequenceEqual(latitudes, [70.93, 77, 78.92]) + + def test_read_past_last_message(self): + """Trying to open message on exhausted BUFR file raises IOError.""" + with BufrFile(TESTBUFR) as bufr: + for _ in range(len(bufr)): + BufrMessage(bufr) + self.assertRaises(IOError, lambda: BufrMessage(bufr)) + + def test_read_invalid_file(self): + """Trying to open message on nonexistent GRIB file raises IOError.""" + with NamedTemporaryFile(mode='r') as f: + with BufrFile(f.name) as bufr: + self.assertRaises(IOError, lambda: BufrMessage(bufr)) + + +class TestBufrMessage(unittest.TestCase): + + """Test BufrMessage functionality""" + + def test_metadata(self): + """Metadata is read correctly from BufrMessage.""" + with BufrFile(TESTBUFR) as bufr: + msg = BufrMessage(bufr) + for key in KNOWN_BUFR_KEYS: + assert key in msg.keys() + self.assertEqual(msg.size(), 220) + self.assertEqual(len(msg.keys()), len(msg)) + + def test_content(self): + """Data values are read correctly from BufrMessage.""" + with BufrFile(TESTBUFR) as bufr: + msg = BufrMessage(bufr) + self.assertEqual(msg["airTemperatureAt2M"], 274.5) + + # TODO: Test behavior with missing messages (SUP-1874) + + # This fails due to SUP-1875 + def test_value_setting(self): + """Keys can be set properly.""" + with BufrFile(TESTBUFR) as bufr: + msg = BufrMessage(bufr) + key, val = "localLongitude", 5 + msg[key] = val + self.assertEqual(msg[key], val) + + def test_serialize(self): + """Message can be serialized to file.""" + with BufrFile(TESTBUFR) as bufr: + msg = BufrMessage(bufr) + with open(TEST_OUTPUT, "w") as test: + msg.write(test) + os.unlink(TEST_OUTPUT) + + def test_clone(self): + """Messages can be used to produce clone Messages.""" + with BufrFile(TESTBUFR) as bufr: + msg = BufrMessage(bufr) + msg2 = BufrMessage(clone=msg) + self.assertSequenceEqual(msg.keys(), msg2.keys()) + + + if __name__ == "__main__": unittest.main() diff --git a/python/eccodes/__init__.py b/python/eccodes/__init__.py index 4a0f5fd11..42006b9f0 100644 --- a/python/eccodes/__init__.py +++ b/python/eccodes/__init__.py @@ -1,8 +1,11 @@ from __future__ import absolute_import +import sys from .eccodes import * from .eccodes import __version__ -from .high_level.gribfile import GribFile -from .high_level.gribmessage import GribMessage -from .high_level.gribindex import GribIndex +if sys.version_info >= (2, 6): + from .high_level.gribfile import GribFile + from .high_level.gribmessage import GribMessage + from .high_level.gribindex import GribIndex + from .high_level.bufr import BufrFile, BufrMessage diff --git a/python/eccodes/high_level/bufr.py b/python/eccodes/high_level/bufr.py new file mode 100644 index 000000000..e0d8db1a2 --- /dev/null +++ b/python/eccodes/high_level/bufr.py @@ -0,0 +1,134 @@ +""" +Classes for handling BUFR with a high level interface. + +``BufrFiles`` can be treated mostly as regular files and used as context +managers, as can ``BufrMessages``. Each of these classes destructs itself and +any child instances appropriately. + +Author: Daniel Lee, DWD, 2016 +""" + +from .. import eccodes +from .codesmessage import CodesMessage +from .codesfile import CodesFile + + +# TODO: Docstring is mostly redundant, perhaps move into base class? +class BufrMessage(CodesMessage): + + """ + A BUFR message. + + Each ``BufrMessage`` is stored as a key/value pair in a dictionary-like + structure. It can be used in a context manager or by itself. When the + ``BufrFile`` it belongs to is closed, it closes any open ``BufrMessage``s + that belong to it. If a ``BufrMessage`` is closed before its ``BufrFile`` + is closed, it informs the ``BufrFile`` of its closure. + + Scalar and vector values are set appropriately through the same method. + + Usage:: + + >>> with BufrFile(filename) as bufr: + ... # Access shortNames of all messages + ... for msg in bufr: + ... print(msg["shortName"]) + ... # Report number of keys in message + ... len(msg) + ... # Report message size in bytes + ... msg.size + ... # Report keys in message + ... msg.keys() + ... # Check if value is missing + ... msg.missing("scaleFactorOfSecondFixedSurface") + ... # Set scalar value + ... msg["scaleFactorOfSecondFixedSurface"] = 5 + ... # Check key's value + ... msg["scaleFactorOfSecondFixedSurface"] + ... # Set value to missing + ... msg.set_missing("scaleFactorOfSecondFixedSurface") + ... # Missing values raise exception when read with dict notation + ... msg["scaleFactorOfSecondFixedSurface"] + ... # Array values are set transparently + ... msg["values"] = [1, 2, 3] + ... # Messages can be written to file + ... with open(testfile, "w") as test: + ... msg.write(test) + ... # Messages can be cloned from other messages + ... msg2 = BufrMessage(clone=msg) + ... # If desired, messages can be closed manually or used in with + ... msg.close() + """ + + product_kind = eccodes.CODES_PRODUCT_BUFR + + # Arguments included explicitly to support introspection + # TODO: Can we get this to work with an index? + def __init__(self, codes_file=None, clone=None, sample=None, + headers_only=False): + """ + Open a message and inform the GRIB file that it's been incremented. + + The message is taken from ``codes_file``, cloned from ``clone`` or + ``sample``, or taken from ``index``, in that order of precedence. + """ + super(self.__class__, self).__init__(codes_file, clone, sample, + headers_only) + self._unpacked = False + + def get(self, key, ktype=None): + """Return requested value, unpacking data values if necessary.""" + # TODO: Only do this if accessing arrays that need unpacking + if not self._unpacked: + self.unpacked = True + return super(self.__class__, self).get(key, ktype) + + def missing(self, key): + """ + Report if key is missing. + + Overloaded due to confusing behavior in ``codes_is_missing`` (SUP-1874). + """ + return not bool(eccodes.codes_is_defined(self.codes_id, key)) + + def keys(self, namespace=None): + self.unpacked = True + return super(self.__class__, self).keys(namespace) + + @property + def unpacked(self): + return self._unpacked + + @unpacked.setter + def unpacked(self, val): + eccodes.codes_set(self.codes_id, "unpack", val) + self._unpacked = val + + def __setitem__(self, key, value): + """Set item and pack BUFR.""" + if not self._unpacked: + self.unpacked = True + super(self.__class__, self).__setitem__(key, value) + eccodes.codes_set(self.codes_id, "pack", True) + + +class BufrFile(CodesFile): + + """ + A BUFR file handle meant for use in a context manager. + + Individual messages can be accessed using the ``next`` method. Of course, + it is also possible to iterate over each message in the file:: + + >>> with BufrFile(filename) as bufr: + ... # Print number of messages in file + ... len(bufr) + ... # Open all messages in file + ... for msg in bufr: + ... print(msg["shortName"]) + ... len(bufr.open_messages) + >>> # When the file is closed, any open messages are closed + >>> len(bufr.open_messages) + """ + + MessageClass = BufrMessage diff --git a/python/eccodes/high_level/codesfile.py b/python/eccodes/high_level/codesfile.py index 6561e7325..1ff64e169 100644 --- a/python/eccodes/high_level/codesfile.py +++ b/python/eccodes/high_level/codesfile.py @@ -15,6 +15,9 @@ class CodesFile(file): read by ecCodes should implement. """ + #: Type of messages belonging to this file + MessageClass = None + def __init__(self, filename, mode="r"): """Open file and receive codes file handle.""" #: File handle for working with actual file on disc @@ -47,4 +50,7 @@ class CodesFile(file): return self def next(self): - raise NotImplementedError + try: + return self.MessageClass(self) + except IOError: + raise StopIteration() diff --git a/python/eccodes/high_level/codesmessage.py b/python/eccodes/high_level/codesmessage.py index 97be28ed2..c8891e706 100644 --- a/python/eccodes/high_level/codesmessage.py +++ b/python/eccodes/high_level/codesmessage.py @@ -17,8 +17,11 @@ class CodesMessage(object): messages read by ecCodes should implement. """ + #: ecCodes enum-like PRODUCT constant + product_kind = None + def __init__(self, codes_file=None, clone=None, sample=None, - other_args_found=False): + headers_only=False, other_args_found=False): """ Open a message and inform the host file that it's been incremented. @@ -30,8 +33,8 @@ class CodesMessage(object): :param clone: A valid ``CodesMessage`` :param sample: A valid sample path to create ``CodesMessage`` from """ - if not other_args_found and codes_file is None and clone is None and \ - sample is None: + if (not other_args_found and codes_file is None and clone is None and + sample is None): raise RuntimeError("CodesMessage initialization parameters not " "present.") #: Unique ID, for ecCodes interface @@ -39,7 +42,8 @@ class CodesMessage(object): #: File containing message self.codes_file = None if codes_file is not None: - self.codes_id = self.new_from_file(codes_file.file_handle) + self.codes_id = eccodes.codes_new_from_file( + codes_file.file_handle, self.product_kind, headers_only) if self.codes_id is None: raise IOError("CodesFile %s is exhausted" % codes_file.name) self.codes_file = codes_file @@ -57,16 +61,6 @@ class CodesMessage(object): outfile = self.codes_file.file_handle eccodes.codes_write(self.codes_id, outfile) - @staticmethod - def new_from_sample(samplename): - """For generic handling of message types.""" - raise NotImplementedError - - @staticmethod - def new_from_file(fileobj, headers_only=False): - """For generic handling of message types.""" - raise NotImplementedError - def __setitem__(self, key, value): """ Set value associated with key. diff --git a/python/eccodes/high_level/gribfile.py b/python/eccodes/high_level/gribfile.py index 7970eb58b..d8e64284e 100644 --- a/python/eccodes/high_level/gribfile.py +++ b/python/eccodes/high_level/gribfile.py @@ -28,8 +28,4 @@ class GribFile(CodesFile): >>> len(grib.open_messages) """ - def next(self): - try: - return GribMessage(self) - except IOError: - raise StopIteration() + MessageClass = GribMessage diff --git a/python/eccodes/high_level/gribmessage.py b/python/eccodes/high_level/gribmessage.py index 397326d0f..23399990f 100644 --- a/python/eccodes/high_level/gribmessage.py +++ b/python/eccodes/high_level/gribmessage.py @@ -60,10 +60,11 @@ class GribMessage(CodesMessage): ... msg.close() """ + product_kind = eccodes.CODES_PRODUCT_GRIB + # Arguments included explicitly to support introspection - # TODO: Include headers_only option def __init__(self, codes_file=None, clone=None, sample=None, - gribindex=None): + headers_only=False, gribindex=None): """ Open a message and inform the GRIB file that it's been incremented. @@ -74,7 +75,7 @@ class GribMessage(CodesMessage): if gribindex is None: grib_args_present = False super(self.__class__, self).__init__(codes_file, clone, sample, - grib_args_present) + headers_only, grib_args_present) #: GribIndex referencing message self.grib_index = None if gribindex is not None: @@ -92,16 +93,6 @@ class GribMessage(CodesMessage): if self.grib_index: self.grib_index.open_messages.remove(self) - # I'd like these to be simple function mappings but this worked better in - # the interest of time - @staticmethod - def new_from_sample(samplename): - return eccodes.codes_grib_new_from_samples(samplename) - - @staticmethod - def new_from_file(fileobj, headers_only=False): - return eccodes.codes_grib_new_from_file(fileobj, headers_only) - @property def gid(self): """Provided for backwards compatibility."""