mirror of https://github.com/ecmwf/eccodes.git
Merge pull request #9 in ECCODES/eccodes from ~ERGET/eccodes:feature/SUP-1862-high-level-python-bufr to feature/ECC-157-PythonicInterface
* commit '7a7ace2d5e0d9d68cf4b6be16fc2917fd8fcb3d1': Restore deleted .gitignore Inherit docstrings Add high-level BUFR interface. Make test robust to new keys Implement CodesMessage abstract class Extract general GribFile functions into CodesFile class for reuse with BUFR
This commit is contained in:
Shahram Najm
commit
7d5d786c90
|
|
@ -315,4 +315,6 @@ data/bufr/*test
|
|||
|
||||
*.sublime-workspace
|
||||
*.old
|
||||
.idea
|
||||
|
||||
build/
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
#!/bin/env python
|
||||
|
||||
"""
|
||||
Unit tests for ``PythonicGrib``.
|
||||
Unit tests for high level Python interface.
|
||||
|
||||
Author: Daniel Lee, DWD, 2014
|
||||
Author: Daniel Lee, DWD, 2016
|
||||
"""
|
||||
|
||||
import os
|
||||
|
|
@ -14,10 +14,11 @@ from eccodes import GribFile
|
|||
from eccodes import GribIndex
|
||||
from eccodes import GribMessage
|
||||
from eccodes.high_level.gribmessage import IndexNotSelectedError
|
||||
|
||||
from eccodes import BufrFile, BufrMessage
|
||||
|
||||
TESTGRIB = "../../data/high_level_api.grib2"
|
||||
TEST_OUTPUT = "test-output.grib"
|
||||
TESTBUFR = "../../data/bufr/syno_multi.bufr"
|
||||
TEST_OUTPUT = "test-output.codes"
|
||||
TEST_INDEX = "test.index"
|
||||
TEST_KEYS = ("dataDate", "stepRange")
|
||||
TEST_VALUES = 20110225, 0
|
||||
|
|
@ -26,9 +27,164 @@ for i1 in range(len(TEST_KEYS)):
|
|||
SELECTION_DICTIONARY[TEST_KEYS[i1]] = TEST_VALUES[i1]
|
||||
TEST_INDEX_OUTPUT = TESTGRIB
|
||||
TEST_STEPRANGE = ('0', '12', '18', '24', '6')
|
||||
# These keys should be available even if new keys are defined
|
||||
KNOWN_GRIB_KEYS = ['7777', 'EPS information', 'GRIBEditionNumber', 'N', 'NV',
|
||||
'Ni', 'Nj', 'PLPresent', 'PVPresent',
|
||||
'Parameter information', 'addEmptySection2',
|
||||
'addExtraLocalSection', 'alternativeRowScanning',
|
||||
'angleDivisor', 'angleMultiplier', 'angularPrecision',
|
||||
'average', 'backgroundProcess',
|
||||
'basicAngleOfTheInitialProductionDomain',
|
||||
'binaryScaleFactor', 'bitMapIndicator', 'bitmapPresent',
|
||||
'bitsPerValue', 'bottomLevel', 'centre',
|
||||
'centreDescription', 'cfName', 'cfNameECMF', 'cfVarName',
|
||||
'cfVarNameECMF', 'changeDecimalPrecision', 'class',
|
||||
'climateDateFrom', 'climateDateTo', 'codedValues',
|
||||
'dataDate', 'dataRepresentationTemplateNumber', 'dataTime',
|
||||
'day', 'decimalPrecision', 'decimalScaleFactor',
|
||||
'deleteCalendarId', 'deleteExtraLocalSection', 'deletePV',
|
||||
'discipline', 'distinctLatitudes', 'distinctLongitudes',
|
||||
'editionNumber', 'endStep', 'eps',
|
||||
'experimentVersionNumber', 'extraLocalSectionPresent',
|
||||
'forecastTime', 'g2grid', 'gaussianGridName',
|
||||
'genVertHeightCoords', 'generatingProcessIdentifier',
|
||||
'getNumberOfValues', 'global', 'globalDomain',
|
||||
'grib 2 Section 5 DATA REPRESENTATION SECTION',
|
||||
'grib 2 Section 6 BIT-MAP SECTION', 'grib 2 Section 7 data',
|
||||
'grib2LocalSectionNumber', 'grib2LocalSectionPresent',
|
||||
'grib2divider', 'gridDefinitionDescription',
|
||||
'gridDefinitionTemplateNumber',
|
||||
'gridDescriptionSectionPresent', 'gridType', 'hour',
|
||||
'hoursAfterDataCutoff', 'iDirectionIncrement',
|
||||
'iDirectionIncrementGiven', 'iDirectionIncrementInDegrees',
|
||||
'iScansNegatively', 'iScansPositively', 'identifier',
|
||||
'ieeeFloats', 'ifsParam', 'ijDirectionIncrementGiven',
|
||||
'indicatorOfUnitOfTimeRange',
|
||||
'interpretationOfNumberOfPoints', 'isConstant',
|
||||
'isHindcast', 'isOctahedral', 'is_uerra',
|
||||
'jDirectionIncrementGiven', 'jPointsAreConsecutive',
|
||||
'jScansPositively', 'julianDay', 'kurtosis', 'latLonValues',
|
||||
'latitudeOfFirstGridPoint',
|
||||
'latitudeOfFirstGridPointInDegrees',
|
||||
'latitudeOfLastGridPoint',
|
||||
'latitudeOfLastGridPointInDegrees', 'latitudes',
|
||||
'legBaseDate', 'legBaseTime', 'legNumber',
|
||||
'lengthOfHeaders', 'level', 'localDefinitionNumber',
|
||||
'localDir', 'localTablesVersion',
|
||||
'longitudeOfFirstGridPoint',
|
||||
'longitudeOfFirstGridPointInDegrees',
|
||||
'longitudeOfLastGridPoint',
|
||||
'longitudeOfLastGridPointInDegrees', 'longitudes',
|
||||
'mAngleMultiplier', 'mBasicAngle', 'marsClass',
|
||||
'marsStream', 'marsType', 'masterDir', 'maximum',
|
||||
'md5Headers', 'md5Section1', 'md5Section3', 'md5Section4',
|
||||
'md5Section5', 'md5Section6', 'md5Section7', 'minimum',
|
||||
'minute', 'minutesAfterDataCutoff', 'missingValue',
|
||||
'modelName', 'month', 'name', 'nameECMF',
|
||||
'nameOfFirstFixedSurface', 'nameOfSecondFixedSurface',
|
||||
'neitherPresent', 'numberOfDataPoints',
|
||||
'numberOfForecastsInEnsemble', 'numberOfMissing',
|
||||
'numberOfOctectsForNumberOfPoints', 'numberOfSection',
|
||||
'numberOfValues', 'oceanAtmosphereCoupling',
|
||||
'offsetValuesBy', 'optimizeScaleFactor', 'packingError',
|
||||
'packingType', 'paramId', 'paramIdECMF',
|
||||
'parameterCategory', 'parameterName', 'parameterNumber',
|
||||
'parameterUnits', 'perturbationNumber', 'pressureUnits',
|
||||
'productDefinitionTemplateNumber',
|
||||
'productDefinitionTemplateNumberInternal', 'productType',
|
||||
'productionStatusOfProcessedData', 'radius',
|
||||
'referenceDate', 'referenceValue', 'referenceValueError',
|
||||
'resolutionAndComponentFlags',
|
||||
'resolutionAndComponentFlags1',
|
||||
'resolutionAndComponentFlags2',
|
||||
'resolutionAndComponentFlags6',
|
||||
'resolutionAndComponentFlags7',
|
||||
'resolutionAndComponentFlags8',
|
||||
'scaleFactorOfEarthMajorAxis',
|
||||
'scaleFactorOfEarthMinorAxis',
|
||||
'scaleFactorOfFirstFixedSurface',
|
||||
'scaleFactorOfRadiusOfSphericalEarth',
|
||||
'scaleFactorOfSecondFixedSurface', 'scaleValuesBy',
|
||||
'scaledValueOfEarthMajorAxis',
|
||||
'scaledValueOfEarthMinorAxis',
|
||||
'scaledValueOfFirstFixedSurface',
|
||||
'scaledValueOfRadiusOfSphericalEarth',
|
||||
'scaledValueOfSecondFixedSurface', 'scanningMode',
|
||||
'scanningMode5', 'scanningMode6', 'scanningMode7',
|
||||
'scanningMode8', 'second', 'section0Length',
|
||||
'section1Length', 'section2Length', 'section2Padding',
|
||||
'section3Length', 'section3Padding', 'section4Length',
|
||||
'section5Length', 'section6Length', 'section7Length',
|
||||
'section8Length', 'sectionNumber',
|
||||
'selectStepTemplateInstant', 'selectStepTemplateInterval',
|
||||
'setBitsPerValue', 'setCalendarId', 'shapeOfTheEarth',
|
||||
'shortName', 'shortNameECMF', 'significanceOfReferenceTime',
|
||||
'skewness', 'sourceOfGridDefinition', 'standardDeviation',
|
||||
'startStep', 'stepRange', 'stepType', 'stepTypeInternal',
|
||||
'stepUnits', 'stream', 'subCentre',
|
||||
'subdivisionsOfBasicAngle', 'tablesVersion',
|
||||
'tablesVersionLatest', 'tempPressureUnits', 'topLevel',
|
||||
'totalLength', 'type', 'typeOfEnsembleForecast',
|
||||
'typeOfFirstFixedSurface', 'typeOfGeneratingProcess',
|
||||
'typeOfLevel', 'typeOfOriginalFieldValues',
|
||||
'typeOfProcessedData', 'typeOfSecondFixedSurface', 'units',
|
||||
'unitsECMF', 'unitsOfFirstFixedSurface',
|
||||
'unitsOfSecondFixedSurface', 'unpackedError',
|
||||
'uvRelativeToGrid', 'validityDate', 'validityTime',
|
||||
'values', 'x', 'year']
|
||||
KNOWN_BUFR_KEYS = ['3HourPressureChange', '7777', 'BUFRstr',
|
||||
'airTemperatureAt2M', 'blockNumber', 'bufrHeaderCentre',
|
||||
'bufrHeaderSubCentre', 'bufrTemplate',
|
||||
'bufrdcExpandedDescriptors', 'centre',
|
||||
'characteristicOfPressureTendency', 'cloudAmount',
|
||||
'cloudCoverTotal', 'cloudType', 'compressedData',
|
||||
'corr1Data', 'corr2Data', 'corr3Data', 'corr4Data',
|
||||
'correction1', 'correction1Part', 'correction2',
|
||||
'correction2Part', 'correction3', 'correction3Part',
|
||||
'correction4', 'correction4Part', 'createNewData',
|
||||
'dataCategory', 'dataPresentIndicator', 'dataSubCategory',
|
||||
'day', 'defaultSequence', 'dewpointTemperatureAt2M', 'ed',
|
||||
'edition', 'expandedAbbreviations', 'expandedCodes',
|
||||
'expandedCrex_scales', 'expandedCrex_units',
|
||||
'expandedCrex_widths', 'expandedNames',
|
||||
'expandedOriginalCodes', 'expandedOriginalReferences',
|
||||
'expandedOriginalScales', 'expandedOriginalWidths',
|
||||
'expandedTypes', 'expandedUnits', 'generatingApplication',
|
||||
'globalDomain', 'heightOfBaseOfCloud', 'heightOfStation',
|
||||
'horizontalVisibility', 'hour', 'isSatellite',
|
||||
'isSatelliteType', 'latitude', 'lengthDescriptors',
|
||||
'localDay', 'localHour', 'localLatitude', 'localLongitude',
|
||||
'localMinute', 'localMonth', 'localSecond',
|
||||
'localSectionPresent', 'localTablesVersionNumber',
|
||||
'localYear', 'longitude', 'masterTableNumber',
|
||||
'masterTablesVersionNumber', 'md5Data', 'md5Structure',
|
||||
'messageLength', 'minute', 'month', 'nonCoordinatePressure',
|
||||
'numberOfSubsets', 'numberOfUnexpandedDescriptors',
|
||||
'observedData', 'operator', 'pastWeather1', 'pastWeather2',
|
||||
'presentWeather', 'pressureReducedToMeanSeaLevel',
|
||||
'qualityControl', 'rdbSubtype', 'rdbType', 'rdbtime',
|
||||
'rdbtimeDay', 'rdbtimeHour', 'rdbtimeMinute',
|
||||
'rdbtimeSecond', 'rectime', 'rectimeDay', 'rectimeHour',
|
||||
'rectimeMinute', 'rectimeSecond', 'relativeHumidity',
|
||||
'reservedSection2', 'reservedSection3', 'section1Length',
|
||||
'section1Padding', 'section2Length', 'section2Padding',
|
||||
'section3Flags', 'section3Length', 'section3Padding',
|
||||
'section4Length', 'section4Padding', 'section5Length',
|
||||
'sequences', 'spare', 'spare1', 'stationNumber',
|
||||
'stationType', 'subsetNumber', 'tableNumber',
|
||||
'templatesLocalDir', 'templatesMasterDir', 'totalLength',
|
||||
'totalPrecipitationPast6Hours', 'totalSnowDepth',
|
||||
'typicalCentury', 'typicalDate', 'typicalDay',
|
||||
'typicalHour', 'typicalMinute', 'typicalMonth',
|
||||
'typicalSecond', 'typicalTime', 'typicalYear',
|
||||
'typicalYearOfCentury', 'unexpandedDescriptors',
|
||||
'updateSequenceNumber',
|
||||
'verticalSignificanceSurfaceObservations',
|
||||
'windDirectionAt10M', 'windSpeedAt10M', 'year']
|
||||
|
||||
|
||||
class TestGribFile(unittest.TestCase):
|
||||
|
||||
"""Test GribFile functionality."""
|
||||
|
||||
def test_memory_management(self):
|
||||
|
|
@ -41,14 +197,11 @@ class TestGribFile(unittest.TestCase):
|
|||
self.assertEqual(len(grib.open_messages), 5)
|
||||
self.assertEqual(len(grib.open_messages), 0)
|
||||
|
||||
def test_iteration_works(self):
|
||||
"""The GribFile allows proper iteration over all messages."""
|
||||
step_ranges = []
|
||||
def test_message_counting_works(self):
|
||||
"""The GribFile is aware of its messages."""
|
||||
with GribFile(TESTGRIB) as grib:
|
||||
for _ in range(len(grib)):
|
||||
msg = GribMessage(grib)
|
||||
step_ranges.append(msg["stepRange"])
|
||||
self.assertSequenceEqual(step_ranges, ["0", "6", "12", "18", "24"])
|
||||
msg_count = len(grib)
|
||||
self.assertEqual(msg_count, 5)
|
||||
|
||||
def test_iterator_protocol(self):
|
||||
"""The GribFile allows pythonic iteration over all messages."""
|
||||
|
|
@ -73,16 +226,17 @@ class TestGribFile(unittest.TestCase):
|
|||
|
||||
|
||||
class TestGribMessage(unittest.TestCase):
|
||||
|
||||
"""Test GribMessage functionality."""
|
||||
|
||||
def test_metadata(self):
|
||||
"""Metadata is read correctly from GribMessage."""
|
||||
with GribFile(TESTGRIB) as grib:
|
||||
msg = GribMessage(grib)
|
||||
key_count = 253
|
||||
self.assertEqual(len(msg), key_count)
|
||||
for key in KNOWN_GRIB_KEYS:
|
||||
assert key in msg.keys()
|
||||
self.assertEqual(msg.size(), 160219)
|
||||
self.assertEqual(len(msg.keys()), key_count)
|
||||
self.assertEqual(len(msg.keys()), len(msg))
|
||||
|
||||
def test_missing_message_behaviour(self):
|
||||
"""Missing messages are detected properly."""
|
||||
|
|
@ -118,6 +272,7 @@ class TestGribMessage(unittest.TestCase):
|
|||
|
||||
|
||||
class TestGribIndex(unittest.TestCase):
|
||||
|
||||
"""Test GribIndex functionality."""
|
||||
|
||||
def test_memory_management(self):
|
||||
|
|
@ -149,5 +304,95 @@ class TestGribIndex(unittest.TestCase):
|
|||
idx.select(SELECTION_DICTIONARY)
|
||||
self.assertEqual(len(idx.open_messages), 1)
|
||||
|
||||
|
||||
class TestBufrFile(unittest.TestCase):
|
||||
|
||||
"""Test BufrFile functionality."""
|
||||
|
||||
def test_memory_management(self):
|
||||
"""Messages in BufrFile can be opened and closed properly."""
|
||||
with BufrFile(TESTBUFR) as bufr:
|
||||
self.assertEqual(len(bufr), 3)
|
||||
for i in range(len(bufr)):
|
||||
msg = BufrMessage(bufr)
|
||||
self.assertEqual(msg["bufrHeaderCentre"], 98)
|
||||
self.assertEqual(len(bufr.open_messages), 3)
|
||||
self.assertEquals(len(bufr.open_messages), 0)
|
||||
|
||||
def test_message_counting_works(self):
|
||||
"""The BufrFile is aware of its messages."""
|
||||
with BufrFile(TESTBUFR) as bufr:
|
||||
msg_count = len(bufr)
|
||||
self.assertEqual(msg_count, 3)
|
||||
|
||||
def test_iterator_protocol(self):
|
||||
"""The BufrFile allows pythonic iteration over all messages."""
|
||||
latitudes = []
|
||||
with BufrFile(TESTBUFR) as bufr:
|
||||
for msg in bufr:
|
||||
latitudes.append(msg["localLatitude"])
|
||||
self.assertSequenceEqual(latitudes, [70.93, 77, 78.92])
|
||||
|
||||
def test_read_past_last_message(self):
|
||||
"""Trying to open message on exhausted BUFR file raises IOError."""
|
||||
with BufrFile(TESTBUFR) as bufr:
|
||||
for _ in range(len(bufr)):
|
||||
BufrMessage(bufr)
|
||||
self.assertRaises(IOError, lambda: BufrMessage(bufr))
|
||||
|
||||
def test_read_invalid_file(self):
|
||||
"""Trying to open message on nonexistent GRIB file raises IOError."""
|
||||
with NamedTemporaryFile(mode='r') as f:
|
||||
with BufrFile(f.name) as bufr:
|
||||
self.assertRaises(IOError, lambda: BufrMessage(bufr))
|
||||
|
||||
|
||||
class TestBufrMessage(unittest.TestCase):
|
||||
|
||||
"""Test BufrMessage functionality"""
|
||||
|
||||
def test_metadata(self):
|
||||
"""Metadata is read correctly from BufrMessage."""
|
||||
with BufrFile(TESTBUFR) as bufr:
|
||||
msg = BufrMessage(bufr)
|
||||
for key in KNOWN_BUFR_KEYS:
|
||||
assert key in msg.keys()
|
||||
self.assertEqual(msg.size(), 220)
|
||||
self.assertEqual(len(msg.keys()), len(msg))
|
||||
|
||||
def test_content(self):
|
||||
"""Data values are read correctly from BufrMessage."""
|
||||
with BufrFile(TESTBUFR) as bufr:
|
||||
msg = BufrMessage(bufr)
|
||||
self.assertEqual(msg["airTemperatureAt2M"], 274.5)
|
||||
|
||||
# TODO: Test behavior with missing messages (SUP-1874)
|
||||
|
||||
# This fails due to SUP-1875
|
||||
def test_value_setting(self):
|
||||
"""Keys can be set properly."""
|
||||
with BufrFile(TESTBUFR) as bufr:
|
||||
msg = BufrMessage(bufr)
|
||||
key, val = "localLongitude", 5
|
||||
msg[key] = val
|
||||
self.assertEqual(msg[key], val)
|
||||
|
||||
def test_serialize(self):
|
||||
"""Message can be serialized to file."""
|
||||
with BufrFile(TESTBUFR) as bufr:
|
||||
msg = BufrMessage(bufr)
|
||||
with open(TEST_OUTPUT, "w") as test:
|
||||
msg.write(test)
|
||||
os.unlink(TEST_OUTPUT)
|
||||
|
||||
def test_clone(self):
|
||||
"""Messages can be used to produce clone Messages."""
|
||||
with BufrFile(TESTBUFR) as bufr:
|
||||
msg = BufrMessage(bufr)
|
||||
msg2 = BufrMessage(clone=msg)
|
||||
self.assertSequenceEqual(msg.keys(), msg2.keys())
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -1,8 +1,11 @@
|
|||
from __future__ import absolute_import
|
||||
import sys
|
||||
|
||||
from .eccodes import *
|
||||
from .eccodes import __version__
|
||||
|
||||
from .high_level.gribfile import GribFile
|
||||
from .high_level.gribmessage import GribMessage
|
||||
from .high_level.gribindex import GribIndex
|
||||
if sys.version_info >= (2, 6):
|
||||
from .high_level.gribfile import GribFile
|
||||
from .high_level.gribmessage import GribMessage
|
||||
from .high_level.gribindex import GribIndex
|
||||
from .high_level.bufr import BufrFile, BufrMessage
|
||||
|
|
|
|||
|
|
@ -0,0 +1,79 @@
|
|||
"""
|
||||
Classes for handling BUFR with a high level interface.
|
||||
|
||||
``BufrFiles`` can be treated mostly as regular files and used as context
|
||||
managers, as can ``BufrMessages``. Each of these classes destructs itself and
|
||||
any child instances appropriately.
|
||||
|
||||
Author: Daniel Lee, DWD, 2016
|
||||
"""
|
||||
|
||||
from .. import eccodes
|
||||
from .codesmessage import CodesMessage
|
||||
from .codesfile import CodesFile
|
||||
|
||||
|
||||
class BufrMessage(CodesMessage):
|
||||
|
||||
__doc__ = "\n".join(CodesMessage.__doc__.splitlines()[4:]).format(
|
||||
prod_type="BUFR", classname="BufrMessage", parent="BufrFile",
|
||||
alias="bufr")
|
||||
|
||||
product_kind = eccodes.CODES_PRODUCT_BUFR
|
||||
|
||||
# Arguments included explicitly to support introspection
|
||||
# TODO: Can we get this to work with an index?
|
||||
def __init__(self, codes_file=None, clone=None, sample=None,
|
||||
headers_only=False):
|
||||
"""
|
||||
Open a message and inform the GRIB file that it's been incremented.
|
||||
|
||||
The message is taken from ``codes_file``, cloned from ``clone`` or
|
||||
``sample``, or taken from ``index``, in that order of precedence.
|
||||
"""
|
||||
super(self.__class__, self).__init__(codes_file, clone, sample,
|
||||
headers_only)
|
||||
self._unpacked = False
|
||||
|
||||
def get(self, key, ktype=None):
|
||||
"""Return requested value, unpacking data values if necessary."""
|
||||
# TODO: Only do this if accessing arrays that need unpacking
|
||||
if not self._unpacked:
|
||||
self.unpacked = True
|
||||
return super(self.__class__, self).get(key, ktype)
|
||||
|
||||
def missing(self, key):
|
||||
"""
|
||||
Report if key is missing.
|
||||
|
||||
Overloaded due to confusing behavior in ``codes_is_missing`` (SUP-1874).
|
||||
"""
|
||||
return not bool(eccodes.codes_is_defined(self.codes_id, key))
|
||||
|
||||
def keys(self, namespace=None):
|
||||
self.unpacked = True
|
||||
return super(self.__class__, self).keys(namespace)
|
||||
|
||||
@property
|
||||
def unpacked(self):
|
||||
return self._unpacked
|
||||
|
||||
@unpacked.setter
|
||||
def unpacked(self, val):
|
||||
eccodes.codes_set(self.codes_id, "unpack", val)
|
||||
self._unpacked = val
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
"""Set item and pack BUFR."""
|
||||
if not self._unpacked:
|
||||
self.unpacked = True
|
||||
super(self.__class__, self).__setitem__(key, value)
|
||||
eccodes.codes_set(self.codes_id, "pack", True)
|
||||
|
||||
|
||||
class BufrFile(CodesFile):
|
||||
|
||||
__doc__ = "\n".join(CodesFile.__doc__.splitlines()[4:]).format(
|
||||
prod_type="BUFR", classname="BufrFile", alias="bufr")
|
||||
|
||||
MessageClass = BufrMessage
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
"""
|
||||
``CodesFile`` class that implements a file that is readable by ecCodes and
|
||||
closes itself and its messages when it is no longer needed.
|
||||
|
||||
Author: Daniel Lee, DWD, 2016
|
||||
"""
|
||||
|
||||
from .. import eccodes
|
||||
|
||||
|
||||
class CodesFile(file):
|
||||
|
||||
"""
|
||||
An abstract class to specify and/or implement common behavior that files
|
||||
read by ecCodes should implement.
|
||||
|
||||
A {prod_type} file handle meant for use in a context manager.
|
||||
|
||||
Individual messages can be accessed using the ``next`` method. Of course,
|
||||
it is also possible to iterate over each message in the file::
|
||||
|
||||
>>> with {classname}(filename) as {alias}:
|
||||
... # Print number of messages in file
|
||||
... len({alias})
|
||||
... # Open all messages in file
|
||||
... for msg in {alias}:
|
||||
... print(msg[key_name])
|
||||
... len({alias}.open_messages)
|
||||
>>> # When the file is closed, any open messages are closed
|
||||
>>> len({alias}.open_messages)
|
||||
"""
|
||||
|
||||
#: Type of messages belonging to this file
|
||||
MessageClass = None
|
||||
|
||||
def __init__(self, filename, mode="r"):
|
||||
"""Open file and receive codes file handle."""
|
||||
#: File handle for working with actual file on disc
|
||||
#: The class holds the file it works with because ecCodes'
|
||||
# typechecking does not allow using inherited classes.
|
||||
self.file_handle = open(filename, mode)
|
||||
#: Number of message in file currently being read
|
||||
self.message = 0
|
||||
#: Open messages
|
||||
self.open_messages = []
|
||||
|
||||
def __exit__(self, type, value, traceback):
|
||||
"""Close all open messages, release GRIB file handle and close file."""
|
||||
while self.open_messages:
|
||||
self.open_messages.pop().close()
|
||||
self.file_handle.close()
|
||||
|
||||
def __len__(self):
|
||||
"""Return total number of messages in file."""
|
||||
return eccodes.codes_count_in_file(self.file_handle)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def close(self):
|
||||
"""Possibility to manually close file."""
|
||||
self.__exit__(None, None, None)
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def next(self):
|
||||
try:
|
||||
return self.MessageClass(self)
|
||||
except IOError:
|
||||
raise StopIteration()
|
||||
|
|
@ -0,0 +1,193 @@
|
|||
"""
|
||||
``CodesMessage`` class that implements a message readable by ecCodes that
|
||||
allows access to the message's key-value pairs in a dictionary-like manner
|
||||
and closes the message when it is no longer needed, coordinating this with
|
||||
its host file.
|
||||
|
||||
Author: Daniel Lee, DWD, 2016
|
||||
"""
|
||||
|
||||
from .. import eccodes
|
||||
|
||||
|
||||
class CodesMessage(object):
|
||||
|
||||
"""
|
||||
An abstract class to specify and/or implement common behavior that
|
||||
messages read by ecCodes should implement.
|
||||
|
||||
A {prod_type} message.
|
||||
|
||||
Each ``{classname}`` is stored as a key/value pair in a dictionary-like
|
||||
structure. It can be used in a context manager or by itself. When the
|
||||
``{parent}`` it belongs to is closed, the ``{parent}`` closes any open
|
||||
``{classname}``s that belong to it. If a ``{classname}`` is closed before
|
||||
its ``{parent}`` is closed, it informs the ``{parent}`` of its closure.
|
||||
|
||||
Scalar and vector values are set appropriately through the same method.
|
||||
|
||||
``{classname}``s can be instantiated from a ``{parent}``, cloned from
|
||||
other ``{classname}``s or taken from samples. Iterating over the members
|
||||
of a ``{parent}`` extracts the ``{classname}``s it contains until the
|
||||
``{parent}`` is exhausted.
|
||||
|
||||
Usage::
|
||||
|
||||
>>> with {parent}(filename) as {alias}:
|
||||
... # Access a key from each message
|
||||
... for msg in {alias}:
|
||||
... print(msg[key_name])
|
||||
... # Report number of keys in message
|
||||
... len(msg)
|
||||
... # Report message size in bytes
|
||||
... msg.size
|
||||
... # Report keys in message
|
||||
... msg.keys()
|
||||
... # Check if value is missing
|
||||
... msg.missing(key_name)
|
||||
... # Set scalar value
|
||||
... msg[scalar_key] = 5
|
||||
... # Check key's value
|
||||
... msg[scalar_key]
|
||||
... # Set value to missing
|
||||
... msg.set_missing(key_name)
|
||||
... # Missing values raise exception when read with dict notation
|
||||
... msg[key_name]
|
||||
... # Array values are set transparently
|
||||
... msg[array_key] = [1, 2, 3]
|
||||
... # Messages can be written to file
|
||||
... with open(testfile, "w") as test:
|
||||
... msg.write(test)
|
||||
... # Messages can be cloned from other messages
|
||||
... msg2 = {classname}(clone=msg)
|
||||
... # If desired, messages can be closed manually or used in with
|
||||
... msg.close()
|
||||
"""
|
||||
|
||||
#: ecCodes enum-like PRODUCT constant
|
||||
product_kind = None
|
||||
|
||||
def __init__(self, codes_file=None, clone=None, sample=None,
|
||||
headers_only=False, other_args_found=False):
|
||||
"""
|
||||
Open a message and inform the host file that it's been incremented.
|
||||
|
||||
If ``codes_file`` is not supplied, the message is cloned from
|
||||
``CodesMessage`` ``clone``. If neither is supplied,
|
||||
the ``CodesMessage`` is cloned from ``sample``.
|
||||
|
||||
:param codes_file: A file readable for ecCodes
|
||||
:param clone: A valid ``CodesMessage``
|
||||
:param sample: A valid sample path to create ``CodesMessage`` from
|
||||
"""
|
||||
if (not other_args_found and codes_file is None and clone is None and
|
||||
sample is None):
|
||||
raise RuntimeError("CodesMessage initialization parameters not "
|
||||
"present.")
|
||||
#: Unique ID, for ecCodes interface
|
||||
self.codes_id = None
|
||||
#: File containing message
|
||||
self.codes_file = None
|
||||
if codes_file is not None:
|
||||
self.codes_id = eccodes.codes_new_from_file(
|
||||
codes_file.file_handle, self.product_kind, headers_only)
|
||||
if self.codes_id is None:
|
||||
raise IOError("CodesFile %s is exhausted" % codes_file.name)
|
||||
self.codes_file = codes_file
|
||||
self.codes_file.message += 1
|
||||
self.codes_file.open_messages.append(self)
|
||||
elif clone is not None:
|
||||
self.codes_id = eccodes.codes_clone(clone.codes_id)
|
||||
elif sample is not None:
|
||||
self.codes_id = self.new_from_sample(sample)
|
||||
|
||||
def write(self, outfile=None):
|
||||
"""Write message to file."""
|
||||
if not outfile:
|
||||
# This is a hack because the API does not accept inheritance
|
||||
outfile = self.codes_file.file_handle
|
||||
eccodes.codes_write(self.codes_id, outfile)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
"""
|
||||
Set value associated with key.
|
||||
|
||||
Iterables and scalars are handled intelligently.
|
||||
"""
|
||||
# Passed value is iterable and not string
|
||||
if hasattr(value, "__iter__"):
|
||||
eccodes.codes_set_array(self.codes_id, key, value)
|
||||
else:
|
||||
eccodes.codes_set(self.codes_id, key, value)
|
||||
|
||||
def keys(self, namespace=None):
|
||||
"""Get available keys in message."""
|
||||
iterator = eccodes.codes_keys_iterator_new(self.codes_id,
|
||||
namespace=namespace)
|
||||
keys = []
|
||||
while eccodes.codes_keys_iterator_next(iterator):
|
||||
key = eccodes.codes_keys_iterator_get_name(iterator)
|
||||
keys.append(key)
|
||||
eccodes.codes_keys_iterator_delete(iterator)
|
||||
return keys
|
||||
|
||||
def size(self):
|
||||
"""Return size of message in bytes."""
|
||||
return eccodes.codes_get_message_size(self.codes_id)
|
||||
|
||||
def dump(self):
|
||||
"""Dump message's binary content."""
|
||||
return eccodes.codes_get_message(self.codes_id)
|
||||
|
||||
def get(self, key, ktype=None):
|
||||
"""Get value of a given key as its native or specified type."""
|
||||
if self.missing(key):
|
||||
raise KeyError("Key is missing from message.")
|
||||
if eccodes.codes_get_size(self.codes_id, key) > 1:
|
||||
ret = eccodes.codes_get_array(self.codes_id, key, ktype)
|
||||
else:
|
||||
ret = eccodes.codes_get(self.codes_id, key, ktype)
|
||||
return ret
|
||||
|
||||
def missing(self, key):
|
||||
"""Report if key is missing."""
|
||||
return bool(eccodes.codes_is_missing(self.codes_id, key))
|
||||
|
||||
def set_missing(self, key):
|
||||
"""Set a key to missing."""
|
||||
eccodes.codes_set_missing(self.codes_id, key)
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
"""Release message handle and inform host file of release."""
|
||||
eccodes.codes_release(self.codes_id)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def close(self):
|
||||
"""Possibility to manually close message."""
|
||||
self.__exit__(None, None, None)
|
||||
|
||||
def __contains__(self, key):
|
||||
"""Check whether a key is present in message."""
|
||||
return key in self.keys()
|
||||
|
||||
def __len__(self):
|
||||
"""Return key count."""
|
||||
return len(self.keys())
|
||||
|
||||
def __getitem__(self, key):
|
||||
"""Return value associated with key as its native type."""
|
||||
return self.get(key)
|
||||
|
||||
def __iter__(self):
|
||||
return iter(self.keys())
|
||||
|
||||
# Not yet implemented
|
||||
# def itervalues(self):
|
||||
# return self.values()
|
||||
|
||||
def items(self):
|
||||
"""Return list of tuples of all key/value pairs."""
|
||||
return [(key, self[key]) for key in self.keys()]
|
||||
|
||||
|
|
@ -5,61 +5,13 @@ messages when it is no longer needed.
|
|||
Author: Daniel Lee, DWD, 2014
|
||||
"""
|
||||
|
||||
from .. import eccodes
|
||||
from .codesfile import CodesFile
|
||||
from .gribmessage import GribMessage
|
||||
|
||||
|
||||
class GribFile(file):
|
||||
"""
|
||||
A GRIB file handle meant for use in a context manager.
|
||||
class GribFile(CodesFile):
|
||||
|
||||
Individual messages can be accessed using the ``next`` method. Of course,
|
||||
it is also possible to iterate over each message in the file::
|
||||
__doc__ = "\n".join(CodesFile.__doc__.splitlines()[4:]).format(
|
||||
prod_type="GRIB", classname="GribFile", alias="grib")
|
||||
|
||||
>>> with GribFile(filename) as grib:
|
||||
... # Print number of messages in file
|
||||
... len(grib)
|
||||
... # Open all messages in file
|
||||
... for msg in grib:
|
||||
... print(msg["shortName"])
|
||||
... len(grib.open_messages)
|
||||
>>> # When the file is closed, any open messages are closed
|
||||
>>> len(grib.open_messages)
|
||||
"""
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exception_type, exception_value, traceback):
|
||||
"""Close all open messages, release GRIB file handle and close file."""
|
||||
while self.open_messages:
|
||||
self.open_messages.pop().close()
|
||||
self.file_handle.close()
|
||||
|
||||
def close(self):
|
||||
"""Possibility to manually close file."""
|
||||
self.__exit__(None, None, None)
|
||||
|
||||
def __len__(self):
|
||||
"""Return total messages in GRIB file."""
|
||||
return eccodes.codes_count_in_file(self.file_handle)
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def next(self):
|
||||
try:
|
||||
return GribMessage(self)
|
||||
except IOError:
|
||||
raise StopIteration()
|
||||
|
||||
def __init__(self, filename, mode="r"):
|
||||
"""Open file and receive GRIB file handle."""
|
||||
#: File handle for working with actual file on disc
|
||||
#: The class holds the file it works with because the GRIB API's
|
||||
#: typechecking does not allow using inherited classes.
|
||||
self.file_handle = open(filename, mode)
|
||||
#: Number of message in GRIB file currently being read
|
||||
self.message = 0
|
||||
#: Open messages
|
||||
self.open_messages = []
|
||||
MessageClass = GribMessage
|
||||
|
|
|
|||
|
|
@ -6,8 +6,7 @@ message when it is no longer needed, coordinating this with its host file.
|
|||
Author: Daniel Lee, DWD, 2014
|
||||
"""
|
||||
|
||||
import collections
|
||||
|
||||
from .codesmessage import CodesMessage
|
||||
from .. import eccodes
|
||||
|
||||
|
||||
|
|
@ -15,185 +14,59 @@ class IndexNotSelectedError(Exception):
|
|||
"""GRIB index was requested before selecting key/value pairs."""
|
||||
|
||||
|
||||
class GribMessage(object):
|
||||
"""
|
||||
A GRIB message.
|
||||
class GribMessage(CodesMessage):
|
||||
|
||||
Each ``GribMessage`` is stored as a key/value pair in a dictionary-like
|
||||
structure. It can be used in a context manager or by itself. When the
|
||||
``GribFile`` it belongs to is closed, it closes any open ``GribMessage``s
|
||||
that belong to it. If a ``GribMessage`` is closed before its ``GribFile``
|
||||
is closed, it informs the ``GribFile`` of its closure.
|
||||
__doc__ = "\n".join(CodesMessage.__doc__.splitlines()[4:]).format(
|
||||
prod_type="GRIB", classname="GribMessage", parent="GribFile",
|
||||
alias="grib")
|
||||
|
||||
Scalar and vector values are set appropriately through the same method.
|
||||
product_kind = eccodes.CODES_PRODUCT_GRIB
|
||||
|
||||
Usage::
|
||||
|
||||
>>> with GribFile(filename) as grib:
|
||||
... # Access shortNames of all messages
|
||||
... for msg in grib:
|
||||
... print(msg["shortName"])
|
||||
... # Report number of keys in message
|
||||
... len(msg)
|
||||
... # Report message size in bytes
|
||||
... msg.size
|
||||
... # Report keys in message
|
||||
... msg.keys()
|
||||
... # Check if value is missing
|
||||
... msg.missing("scaleFactorOfSecondFixedSurface")
|
||||
... # Set scalar value
|
||||
... msg["scaleFactorOfSecondFixedSurface"] = 5
|
||||
... # Check key's value
|
||||
... msg["scaleFactorOfSecondFixedSurface"]
|
||||
... # Set value to missing
|
||||
... msg.set_missing("scaleFactorOfSecondFixedSurface")
|
||||
... # Missing values raise exception when read with dict notation
|
||||
... msg["scaleFactorOfSecondFixedSurface"]
|
||||
... # Array values are set transparently
|
||||
... msg["values"] = [1, 2, 3]
|
||||
... # Messages can be written to file
|
||||
... with open(testfile, "w") as test:
|
||||
... msg.write(test)
|
||||
... # Messages can be cloned from other messages
|
||||
... msg2 = GribMessage(clone=msg)
|
||||
... # If desired, messages can be closed manually or used in with
|
||||
... msg.close()
|
||||
"""
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exception_type, exception_value, traceback):
|
||||
"""Release GRIB message handle and inform file of release."""
|
||||
# This assert should never trigger
|
||||
# assert self.gid is not None
|
||||
eccodes.codes_release(self.gid)
|
||||
if self.grib_index:
|
||||
self.grib_index.open_messages.remove(self)
|
||||
|
||||
def close(self):
|
||||
"""Possibility to manually close message."""
|
||||
self.__exit__(None, None, None)
|
||||
|
||||
def __contains__(self, key):
|
||||
"""Check whether a key is present in message."""
|
||||
return key in self.keys()
|
||||
|
||||
def __len__(self):
|
||||
"""Return key count."""
|
||||
return len(self.keys())
|
||||
|
||||
def __getitem__(self, key):
|
||||
"""Return value associated with key as its native type."""
|
||||
return self.get(key)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
"""
|
||||
Set value associated with key.
|
||||
|
||||
If the object is iterable,
|
||||
"""
|
||||
# Alternative implemented (TODO: evaluate)
|
||||
# if eccodes.codes_get_size(self.gid, key) > 1:
|
||||
# eccodes.codes_set_array(self.gid, key, value)
|
||||
# else:
|
||||
# eccodes.codes_set(self.gid, key, value)
|
||||
|
||||
# Passed value is iterable and not string
|
||||
if (isinstance(value, collections.Iterable) and not
|
||||
isinstance(value, basestring)):
|
||||
eccodes.codes_set_array(self.gid, key, value)
|
||||
else:
|
||||
eccodes.codes_set(self.gid, key, value)
|
||||
|
||||
def __iter__(self):
|
||||
return iter(self.keys())
|
||||
|
||||
# Not yet implemented
|
||||
# def itervalues(self):
|
||||
# return self.values()
|
||||
|
||||
def items(self):
|
||||
"""Return list of tuples of all key/value pairs."""
|
||||
return [(key, self[key]) for key in self.keys()]
|
||||
|
||||
def keys(self, namespace=None):
|
||||
"""Get available keys in message."""
|
||||
iterator = eccodes.codes_keys_iterator_new(self.gid, namespace=namespace)
|
||||
keys = []
|
||||
while eccodes.codes_keys_iterator_next(iterator):
|
||||
key = eccodes.codes_keys_iterator_get_name(iterator)
|
||||
keys.append(key)
|
||||
eccodes.codes_keys_iterator_delete(iterator)
|
||||
return keys
|
||||
|
||||
def __init__(self, grib_file=None, clone=None, sample=None, gribindex=None):
|
||||
# Arguments included explicitly to support introspection
|
||||
def __init__(self, codes_file=None, clone=None, sample=None,
|
||||
headers_only=False, gribindex=None):
|
||||
"""
|
||||
Open a message and inform the GRIB file that it's been incremented.
|
||||
|
||||
If ``grib_file`` is not supplied, the message is cloned from
|
||||
``GribMessage`` ``clone``. If neither is supplied, the ``GribMessage``
|
||||
is cloned from ``sample``. If ``index`` is supplied as a GribIndex, the
|
||||
message is taken from the index.
|
||||
The message is taken from ``codes_file``, cloned from ``clone`` or
|
||||
``sample``, or taken from ``index``, in that order of precedence.
|
||||
"""
|
||||
#: Unique GRIB ID, for GRIB API interface
|
||||
self.gid = None
|
||||
#: File containing message
|
||||
self.grib_file = None
|
||||
grib_args_present = True
|
||||
if gribindex is None:
|
||||
grib_args_present = False
|
||||
super(self.__class__, self).__init__(codes_file, clone, sample,
|
||||
headers_only, grib_args_present)
|
||||
#: GribIndex referencing message
|
||||
self.grib_index = None
|
||||
if grib_file is not None:
|
||||
self.gid = eccodes.codes_grib_new_from_file(grib_file.file_handle)
|
||||
if self.gid is None:
|
||||
raise IOError("Grib file %s is exhausted." % grib_file.name)
|
||||
self.grib_file = grib_file
|
||||
self.grib_file.message += 1
|
||||
self.grib_file.open_messages.append(self)
|
||||
elif clone is not None:
|
||||
self.gid = eccodes.codes_clone(clone.gid)
|
||||
elif sample is not None:
|
||||
self.gid = eccodes.codes_grib_new_from_samples(sample)
|
||||
elif gribindex is not None:
|
||||
self.gid = eccodes.codes_new_from_index(gribindex.iid)
|
||||
if not self.gid:
|
||||
if gribindex is not None:
|
||||
self.codes_id = eccodes.codes_new_from_index(gribindex.iid)
|
||||
if not self.codes_id:
|
||||
raise IndexNotSelectedError("All keys must have selected "
|
||||
"values before receiving message "
|
||||
"from index.")
|
||||
self.grib_index = gribindex
|
||||
gribindex.open_messages.append(self)
|
||||
else:
|
||||
raise RuntimeError("Either grib_file, clone, sample or gribindex "
|
||||
"must be provided.")
|
||||
|
||||
def size(self):
|
||||
"""Return size of message in bytes."""
|
||||
return eccodes.codes_get_message_size(self.gid)
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
"""Release GRIB message handle and inform file of release."""
|
||||
super(self.__class__, self).__exit__(exc_type, exc_val, exc_tb)
|
||||
if self.grib_index:
|
||||
self.grib_index.open_messages.remove(self)
|
||||
|
||||
def dump(self):
|
||||
"""Dump message's binary content."""
|
||||
return eccodes.codes_get_message(self.gid)
|
||||
@property
|
||||
def gid(self):
|
||||
"""Provided for backwards compatibility."""
|
||||
return self.codes_id
|
||||
|
||||
def get(self, key, ktype=None):
|
||||
"""Get value of a given key as its native or specified type."""
|
||||
if self.missing(key):
|
||||
raise KeyError("Value of key %s is MISSING." % key)
|
||||
if eccodes.codes_get_size(self.gid, key) > 1:
|
||||
ret = eccodes.codes_get_array(self.gid, key, ktype)
|
||||
else:
|
||||
ret = eccodes.codes_get(self.gid, key, ktype)
|
||||
return ret
|
||||
@property
|
||||
def grib_file(self):
|
||||
"""Provided for backwards compatibility."""
|
||||
return self.codes_file
|
||||
|
||||
def missing(self, key):
|
||||
"""Report if key is missing."""
|
||||
return bool(eccodes.codes_is_missing(self.gid, key))
|
||||
@gid.setter
|
||||
def gid(self, val):
|
||||
self.codes_id = val
|
||||
|
||||
def set_missing(self, key):
|
||||
"""Set a key to missing."""
|
||||
eccodes.codes_set_missing(self.gid, key)
|
||||
|
||||
def write(self, outfile=None):
|
||||
"""Write message to file."""
|
||||
if not outfile:
|
||||
# This is a hack because the API does not accept inheritance
|
||||
outfile = self.grib_file.file_handle
|
||||
eccodes.codes_write(self.gid, outfile)
|
||||
@grib_file.setter
|
||||
def grib_file(self, val):
|
||||
self.codes_file = val
|
||||
|
|
|
|||
Loading…
Reference in New Issue