diff --git a/.gitignore b/.gitignore index f3e32e270..29223c823 100644 --- a/.gitignore +++ b/.gitignore @@ -315,4 +315,6 @@ data/bufr/*test *.sublime-workspace *.old +.idea +build/ diff --git a/examples/python/high_level_api.py b/examples/python/high_level_api.py index cac39569b..4fd6fe7e4 100644 --- a/examples/python/high_level_api.py +++ b/examples/python/high_level_api.py @@ -1,9 +1,9 @@ #!/bin/env python """ -Unit tests for ``PythonicGrib``. +Unit tests for high level Python interface. -Author: Daniel Lee, DWD, 2014 +Author: Daniel Lee, DWD, 2016 """ import os @@ -14,10 +14,11 @@ from eccodes import GribFile from eccodes import GribIndex from eccodes import GribMessage from eccodes.high_level.gribmessage import IndexNotSelectedError - +from eccodes import BufrFile, BufrMessage TESTGRIB = "../../data/high_level_api.grib2" -TEST_OUTPUT = "test-output.grib" +TESTBUFR = "../../data/bufr/syno_multi.bufr" +TEST_OUTPUT = "test-output.codes" TEST_INDEX = "test.index" TEST_KEYS = ("dataDate", "stepRange") TEST_VALUES = 20110225, 0 @@ -26,9 +27,164 @@ for i1 in range(len(TEST_KEYS)): SELECTION_DICTIONARY[TEST_KEYS[i1]] = TEST_VALUES[i1] TEST_INDEX_OUTPUT = TESTGRIB TEST_STEPRANGE = ('0', '12', '18', '24', '6') +# These keys should be available even if new keys are defined +KNOWN_GRIB_KEYS = ['7777', 'EPS information', 'GRIBEditionNumber', 'N', 'NV', + 'Ni', 'Nj', 'PLPresent', 'PVPresent', + 'Parameter information', 'addEmptySection2', + 'addExtraLocalSection', 'alternativeRowScanning', + 'angleDivisor', 'angleMultiplier', 'angularPrecision', + 'average', 'backgroundProcess', + 'basicAngleOfTheInitialProductionDomain', + 'binaryScaleFactor', 'bitMapIndicator', 'bitmapPresent', + 'bitsPerValue', 'bottomLevel', 'centre', + 'centreDescription', 'cfName', 'cfNameECMF', 'cfVarName', + 'cfVarNameECMF', 'changeDecimalPrecision', 'class', + 'climateDateFrom', 'climateDateTo', 'codedValues', + 'dataDate', 'dataRepresentationTemplateNumber', 'dataTime', + 'day', 'decimalPrecision', 'decimalScaleFactor', + 'deleteCalendarId', 'deleteExtraLocalSection', 'deletePV', + 'discipline', 'distinctLatitudes', 'distinctLongitudes', + 'editionNumber', 'endStep', 'eps', + 'experimentVersionNumber', 'extraLocalSectionPresent', + 'forecastTime', 'g2grid', 'gaussianGridName', + 'genVertHeightCoords', 'generatingProcessIdentifier', + 'getNumberOfValues', 'global', 'globalDomain', + 'grib 2 Section 5 DATA REPRESENTATION SECTION', + 'grib 2 Section 6 BIT-MAP SECTION', 'grib 2 Section 7 data', + 'grib2LocalSectionNumber', 'grib2LocalSectionPresent', + 'grib2divider', 'gridDefinitionDescription', + 'gridDefinitionTemplateNumber', + 'gridDescriptionSectionPresent', 'gridType', 'hour', + 'hoursAfterDataCutoff', 'iDirectionIncrement', + 'iDirectionIncrementGiven', 'iDirectionIncrementInDegrees', + 'iScansNegatively', 'iScansPositively', 'identifier', + 'ieeeFloats', 'ifsParam', 'ijDirectionIncrementGiven', + 'indicatorOfUnitOfTimeRange', + 'interpretationOfNumberOfPoints', 'isConstant', + 'isHindcast', 'isOctahedral', 'is_uerra', + 'jDirectionIncrementGiven', 'jPointsAreConsecutive', + 'jScansPositively', 'julianDay', 'kurtosis', 'latLonValues', + 'latitudeOfFirstGridPoint', + 'latitudeOfFirstGridPointInDegrees', + 'latitudeOfLastGridPoint', + 'latitudeOfLastGridPointInDegrees', 'latitudes', + 'legBaseDate', 'legBaseTime', 'legNumber', + 'lengthOfHeaders', 'level', 'localDefinitionNumber', + 'localDir', 'localTablesVersion', + 'longitudeOfFirstGridPoint', + 'longitudeOfFirstGridPointInDegrees', + 'longitudeOfLastGridPoint', + 'longitudeOfLastGridPointInDegrees', 'longitudes', + 'mAngleMultiplier', 'mBasicAngle', 'marsClass', + 'marsStream', 'marsType', 'masterDir', 'maximum', + 'md5Headers', 'md5Section1', 'md5Section3', 'md5Section4', + 'md5Section5', 'md5Section6', 'md5Section7', 'minimum', + 'minute', 'minutesAfterDataCutoff', 'missingValue', + 'modelName', 'month', 'name', 'nameECMF', + 'nameOfFirstFixedSurface', 'nameOfSecondFixedSurface', + 'neitherPresent', 'numberOfDataPoints', + 'numberOfForecastsInEnsemble', 'numberOfMissing', + 'numberOfOctectsForNumberOfPoints', 'numberOfSection', + 'numberOfValues', 'oceanAtmosphereCoupling', + 'offsetValuesBy', 'optimizeScaleFactor', 'packingError', + 'packingType', 'paramId', 'paramIdECMF', + 'parameterCategory', 'parameterName', 'parameterNumber', + 'parameterUnits', 'perturbationNumber', 'pressureUnits', + 'productDefinitionTemplateNumber', + 'productDefinitionTemplateNumberInternal', 'productType', + 'productionStatusOfProcessedData', 'radius', + 'referenceDate', 'referenceValue', 'referenceValueError', + 'resolutionAndComponentFlags', + 'resolutionAndComponentFlags1', + 'resolutionAndComponentFlags2', + 'resolutionAndComponentFlags6', + 'resolutionAndComponentFlags7', + 'resolutionAndComponentFlags8', + 'scaleFactorOfEarthMajorAxis', + 'scaleFactorOfEarthMinorAxis', + 'scaleFactorOfFirstFixedSurface', + 'scaleFactorOfRadiusOfSphericalEarth', + 'scaleFactorOfSecondFixedSurface', 'scaleValuesBy', + 'scaledValueOfEarthMajorAxis', + 'scaledValueOfEarthMinorAxis', + 'scaledValueOfFirstFixedSurface', + 'scaledValueOfRadiusOfSphericalEarth', + 'scaledValueOfSecondFixedSurface', 'scanningMode', + 'scanningMode5', 'scanningMode6', 'scanningMode7', + 'scanningMode8', 'second', 'section0Length', + 'section1Length', 'section2Length', 'section2Padding', + 'section3Length', 'section3Padding', 'section4Length', + 'section5Length', 'section6Length', 'section7Length', + 'section8Length', 'sectionNumber', + 'selectStepTemplateInstant', 'selectStepTemplateInterval', + 'setBitsPerValue', 'setCalendarId', 'shapeOfTheEarth', + 'shortName', 'shortNameECMF', 'significanceOfReferenceTime', + 'skewness', 'sourceOfGridDefinition', 'standardDeviation', + 'startStep', 'stepRange', 'stepType', 'stepTypeInternal', + 'stepUnits', 'stream', 'subCentre', + 'subdivisionsOfBasicAngle', 'tablesVersion', + 'tablesVersionLatest', 'tempPressureUnits', 'topLevel', + 'totalLength', 'type', 'typeOfEnsembleForecast', + 'typeOfFirstFixedSurface', 'typeOfGeneratingProcess', + 'typeOfLevel', 'typeOfOriginalFieldValues', + 'typeOfProcessedData', 'typeOfSecondFixedSurface', 'units', + 'unitsECMF', 'unitsOfFirstFixedSurface', + 'unitsOfSecondFixedSurface', 'unpackedError', + 'uvRelativeToGrid', 'validityDate', 'validityTime', + 'values', 'x', 'year'] +KNOWN_BUFR_KEYS = ['3HourPressureChange', '7777', 'BUFRstr', + 'airTemperatureAt2M', 'blockNumber', 'bufrHeaderCentre', + 'bufrHeaderSubCentre', 'bufrTemplate', + 'bufrdcExpandedDescriptors', 'centre', + 'characteristicOfPressureTendency', 'cloudAmount', + 'cloudCoverTotal', 'cloudType', 'compressedData', + 'corr1Data', 'corr2Data', 'corr3Data', 'corr4Data', + 'correction1', 'correction1Part', 'correction2', + 'correction2Part', 'correction3', 'correction3Part', + 'correction4', 'correction4Part', 'createNewData', + 'dataCategory', 'dataPresentIndicator', 'dataSubCategory', + 'day', 'defaultSequence', 'dewpointTemperatureAt2M', 'ed', + 'edition', 'expandedAbbreviations', 'expandedCodes', + 'expandedCrex_scales', 'expandedCrex_units', + 'expandedCrex_widths', 'expandedNames', + 'expandedOriginalCodes', 'expandedOriginalReferences', + 'expandedOriginalScales', 'expandedOriginalWidths', + 'expandedTypes', 'expandedUnits', 'generatingApplication', + 'globalDomain', 'heightOfBaseOfCloud', 'heightOfStation', + 'horizontalVisibility', 'hour', 'isSatellite', + 'isSatelliteType', 'latitude', 'lengthDescriptors', + 'localDay', 'localHour', 'localLatitude', 'localLongitude', + 'localMinute', 'localMonth', 'localSecond', + 'localSectionPresent', 'localTablesVersionNumber', + 'localYear', 'longitude', 'masterTableNumber', + 'masterTablesVersionNumber', 'md5Data', 'md5Structure', + 'messageLength', 'minute', 'month', 'nonCoordinatePressure', + 'numberOfSubsets', 'numberOfUnexpandedDescriptors', + 'observedData', 'operator', 'pastWeather1', 'pastWeather2', + 'presentWeather', 'pressureReducedToMeanSeaLevel', + 'qualityControl', 'rdbSubtype', 'rdbType', 'rdbtime', + 'rdbtimeDay', 'rdbtimeHour', 'rdbtimeMinute', + 'rdbtimeSecond', 'rectime', 'rectimeDay', 'rectimeHour', + 'rectimeMinute', 'rectimeSecond', 'relativeHumidity', + 'reservedSection2', 'reservedSection3', 'section1Length', + 'section1Padding', 'section2Length', 'section2Padding', + 'section3Flags', 'section3Length', 'section3Padding', + 'section4Length', 'section4Padding', 'section5Length', + 'sequences', 'spare', 'spare1', 'stationNumber', + 'stationType', 'subsetNumber', 'tableNumber', + 'templatesLocalDir', 'templatesMasterDir', 'totalLength', + 'totalPrecipitationPast6Hours', 'totalSnowDepth', + 'typicalCentury', 'typicalDate', 'typicalDay', + 'typicalHour', 'typicalMinute', 'typicalMonth', + 'typicalSecond', 'typicalTime', 'typicalYear', + 'typicalYearOfCentury', 'unexpandedDescriptors', + 'updateSequenceNumber', + 'verticalSignificanceSurfaceObservations', + 'windDirectionAt10M', 'windSpeedAt10M', 'year'] class TestGribFile(unittest.TestCase): + """Test GribFile functionality.""" def test_memory_management(self): @@ -41,14 +197,11 @@ class TestGribFile(unittest.TestCase): self.assertEqual(len(grib.open_messages), 5) self.assertEqual(len(grib.open_messages), 0) - def test_iteration_works(self): - """The GribFile allows proper iteration over all messages.""" - step_ranges = [] + def test_message_counting_works(self): + """The GribFile is aware of its messages.""" with GribFile(TESTGRIB) as grib: - for _ in range(len(grib)): - msg = GribMessage(grib) - step_ranges.append(msg["stepRange"]) - self.assertSequenceEqual(step_ranges, ["0", "6", "12", "18", "24"]) + msg_count = len(grib) + self.assertEqual(msg_count, 5) def test_iterator_protocol(self): """The GribFile allows pythonic iteration over all messages.""" @@ -73,16 +226,17 @@ class TestGribFile(unittest.TestCase): class TestGribMessage(unittest.TestCase): + """Test GribMessage functionality.""" def test_metadata(self): """Metadata is read correctly from GribMessage.""" with GribFile(TESTGRIB) as grib: msg = GribMessage(grib) - key_count = 253 - self.assertEqual(len(msg), key_count) + for key in KNOWN_GRIB_KEYS: + assert key in msg.keys() self.assertEqual(msg.size(), 160219) - self.assertEqual(len(msg.keys()), key_count) + self.assertEqual(len(msg.keys()), len(msg)) def test_missing_message_behaviour(self): """Missing messages are detected properly.""" @@ -118,6 +272,7 @@ class TestGribMessage(unittest.TestCase): class TestGribIndex(unittest.TestCase): + """Test GribIndex functionality.""" def test_memory_management(self): @@ -149,5 +304,95 @@ class TestGribIndex(unittest.TestCase): idx.select(SELECTION_DICTIONARY) self.assertEqual(len(idx.open_messages), 1) + +class TestBufrFile(unittest.TestCase): + + """Test BufrFile functionality.""" + + def test_memory_management(self): + """Messages in BufrFile can be opened and closed properly.""" + with BufrFile(TESTBUFR) as bufr: + self.assertEqual(len(bufr), 3) + for i in range(len(bufr)): + msg = BufrMessage(bufr) + self.assertEqual(msg["bufrHeaderCentre"], 98) + self.assertEqual(len(bufr.open_messages), 3) + self.assertEquals(len(bufr.open_messages), 0) + + def test_message_counting_works(self): + """The BufrFile is aware of its messages.""" + with BufrFile(TESTBUFR) as bufr: + msg_count = len(bufr) + self.assertEqual(msg_count, 3) + + def test_iterator_protocol(self): + """The BufrFile allows pythonic iteration over all messages.""" + latitudes = [] + with BufrFile(TESTBUFR) as bufr: + for msg in bufr: + latitudes.append(msg["localLatitude"]) + self.assertSequenceEqual(latitudes, [70.93, 77, 78.92]) + + def test_read_past_last_message(self): + """Trying to open message on exhausted BUFR file raises IOError.""" + with BufrFile(TESTBUFR) as bufr: + for _ in range(len(bufr)): + BufrMessage(bufr) + self.assertRaises(IOError, lambda: BufrMessage(bufr)) + + def test_read_invalid_file(self): + """Trying to open message on nonexistent GRIB file raises IOError.""" + with NamedTemporaryFile(mode='r') as f: + with BufrFile(f.name) as bufr: + self.assertRaises(IOError, lambda: BufrMessage(bufr)) + + +class TestBufrMessage(unittest.TestCase): + + """Test BufrMessage functionality""" + + def test_metadata(self): + """Metadata is read correctly from BufrMessage.""" + with BufrFile(TESTBUFR) as bufr: + msg = BufrMessage(bufr) + for key in KNOWN_BUFR_KEYS: + assert key in msg.keys() + self.assertEqual(msg.size(), 220) + self.assertEqual(len(msg.keys()), len(msg)) + + def test_content(self): + """Data values are read correctly from BufrMessage.""" + with BufrFile(TESTBUFR) as bufr: + msg = BufrMessage(bufr) + self.assertEqual(msg["airTemperatureAt2M"], 274.5) + + # TODO: Test behavior with missing messages (SUP-1874) + + # This fails due to SUP-1875 + def test_value_setting(self): + """Keys can be set properly.""" + with BufrFile(TESTBUFR) as bufr: + msg = BufrMessage(bufr) + key, val = "localLongitude", 5 + msg[key] = val + self.assertEqual(msg[key], val) + + def test_serialize(self): + """Message can be serialized to file.""" + with BufrFile(TESTBUFR) as bufr: + msg = BufrMessage(bufr) + with open(TEST_OUTPUT, "w") as test: + msg.write(test) + os.unlink(TEST_OUTPUT) + + def test_clone(self): + """Messages can be used to produce clone Messages.""" + with BufrFile(TESTBUFR) as bufr: + msg = BufrMessage(bufr) + msg2 = BufrMessage(clone=msg) + self.assertSequenceEqual(msg.keys(), msg2.keys()) + + + if __name__ == "__main__": unittest.main() diff --git a/python/eccodes/__init__.py b/python/eccodes/__init__.py index 4a0f5fd11..42006b9f0 100644 --- a/python/eccodes/__init__.py +++ b/python/eccodes/__init__.py @@ -1,8 +1,11 @@ from __future__ import absolute_import +import sys from .eccodes import * from .eccodes import __version__ -from .high_level.gribfile import GribFile -from .high_level.gribmessage import GribMessage -from .high_level.gribindex import GribIndex +if sys.version_info >= (2, 6): + from .high_level.gribfile import GribFile + from .high_level.gribmessage import GribMessage + from .high_level.gribindex import GribIndex + from .high_level.bufr import BufrFile, BufrMessage diff --git a/python/eccodes/high_level/bufr.py b/python/eccodes/high_level/bufr.py new file mode 100644 index 000000000..5f0e76f61 --- /dev/null +++ b/python/eccodes/high_level/bufr.py @@ -0,0 +1,79 @@ +""" +Classes for handling BUFR with a high level interface. + +``BufrFiles`` can be treated mostly as regular files and used as context +managers, as can ``BufrMessages``. Each of these classes destructs itself and +any child instances appropriately. + +Author: Daniel Lee, DWD, 2016 +""" + +from .. import eccodes +from .codesmessage import CodesMessage +from .codesfile import CodesFile + + +class BufrMessage(CodesMessage): + + __doc__ = "\n".join(CodesMessage.__doc__.splitlines()[4:]).format( + prod_type="BUFR", classname="BufrMessage", parent="BufrFile", + alias="bufr") + + product_kind = eccodes.CODES_PRODUCT_BUFR + + # Arguments included explicitly to support introspection + # TODO: Can we get this to work with an index? + def __init__(self, codes_file=None, clone=None, sample=None, + headers_only=False): + """ + Open a message and inform the GRIB file that it's been incremented. + + The message is taken from ``codes_file``, cloned from ``clone`` or + ``sample``, or taken from ``index``, in that order of precedence. + """ + super(self.__class__, self).__init__(codes_file, clone, sample, + headers_only) + self._unpacked = False + + def get(self, key, ktype=None): + """Return requested value, unpacking data values if necessary.""" + # TODO: Only do this if accessing arrays that need unpacking + if not self._unpacked: + self.unpacked = True + return super(self.__class__, self).get(key, ktype) + + def missing(self, key): + """ + Report if key is missing. + + Overloaded due to confusing behavior in ``codes_is_missing`` (SUP-1874). + """ + return not bool(eccodes.codes_is_defined(self.codes_id, key)) + + def keys(self, namespace=None): + self.unpacked = True + return super(self.__class__, self).keys(namespace) + + @property + def unpacked(self): + return self._unpacked + + @unpacked.setter + def unpacked(self, val): + eccodes.codes_set(self.codes_id, "unpack", val) + self._unpacked = val + + def __setitem__(self, key, value): + """Set item and pack BUFR.""" + if not self._unpacked: + self.unpacked = True + super(self.__class__, self).__setitem__(key, value) + eccodes.codes_set(self.codes_id, "pack", True) + + +class BufrFile(CodesFile): + + __doc__ = "\n".join(CodesFile.__doc__.splitlines()[4:]).format( + prod_type="BUFR", classname="BufrFile", alias="bufr") + + MessageClass = BufrMessage diff --git a/python/eccodes/high_level/codesfile.py b/python/eccodes/high_level/codesfile.py new file mode 100644 index 000000000..deb27d9e1 --- /dev/null +++ b/python/eccodes/high_level/codesfile.py @@ -0,0 +1,71 @@ +""" +``CodesFile`` class that implements a file that is readable by ecCodes and +closes itself and its messages when it is no longer needed. + +Author: Daniel Lee, DWD, 2016 +""" + +from .. import eccodes + + +class CodesFile(file): + + """ + An abstract class to specify and/or implement common behavior that files + read by ecCodes should implement. + + A {prod_type} file handle meant for use in a context manager. + + Individual messages can be accessed using the ``next`` method. Of course, + it is also possible to iterate over each message in the file:: + + >>> with {classname}(filename) as {alias}: + ... # Print number of messages in file + ... len({alias}) + ... # Open all messages in file + ... for msg in {alias}: + ... print(msg[key_name]) + ... len({alias}.open_messages) + >>> # When the file is closed, any open messages are closed + >>> len({alias}.open_messages) + """ + + #: Type of messages belonging to this file + MessageClass = None + + def __init__(self, filename, mode="r"): + """Open file and receive codes file handle.""" + #: File handle for working with actual file on disc + #: The class holds the file it works with because ecCodes' + # typechecking does not allow using inherited classes. + self.file_handle = open(filename, mode) + #: Number of message in file currently being read + self.message = 0 + #: Open messages + self.open_messages = [] + + def __exit__(self, type, value, traceback): + """Close all open messages, release GRIB file handle and close file.""" + while self.open_messages: + self.open_messages.pop().close() + self.file_handle.close() + + def __len__(self): + """Return total number of messages in file.""" + return eccodes.codes_count_in_file(self.file_handle) + + def __enter__(self): + return self + + def close(self): + """Possibility to manually close file.""" + self.__exit__(None, None, None) + + def __iter__(self): + return self + + def next(self): + try: + return self.MessageClass(self) + except IOError: + raise StopIteration() diff --git a/python/eccodes/high_level/codesmessage.py b/python/eccodes/high_level/codesmessage.py new file mode 100644 index 000000000..a54df7446 --- /dev/null +++ b/python/eccodes/high_level/codesmessage.py @@ -0,0 +1,193 @@ +""" +``CodesMessage`` class that implements a message readable by ecCodes that +allows access to the message's key-value pairs in a dictionary-like manner +and closes the message when it is no longer needed, coordinating this with +its host file. + +Author: Daniel Lee, DWD, 2016 +""" + +from .. import eccodes + + +class CodesMessage(object): + + """ + An abstract class to specify and/or implement common behavior that + messages read by ecCodes should implement. + + A {prod_type} message. + + Each ``{classname}`` is stored as a key/value pair in a dictionary-like + structure. It can be used in a context manager or by itself. When the + ``{parent}`` it belongs to is closed, the ``{parent}`` closes any open + ``{classname}``s that belong to it. If a ``{classname}`` is closed before + its ``{parent}`` is closed, it informs the ``{parent}`` of its closure. + + Scalar and vector values are set appropriately through the same method. + + ``{classname}``s can be instantiated from a ``{parent}``, cloned from + other ``{classname}``s or taken from samples. Iterating over the members + of a ``{parent}`` extracts the ``{classname}``s it contains until the + ``{parent}`` is exhausted. + + Usage:: + + >>> with {parent}(filename) as {alias}: + ... # Access a key from each message + ... for msg in {alias}: + ... print(msg[key_name]) + ... # Report number of keys in message + ... len(msg) + ... # Report message size in bytes + ... msg.size + ... # Report keys in message + ... msg.keys() + ... # Check if value is missing + ... msg.missing(key_name) + ... # Set scalar value + ... msg[scalar_key] = 5 + ... # Check key's value + ... msg[scalar_key] + ... # Set value to missing + ... msg.set_missing(key_name) + ... # Missing values raise exception when read with dict notation + ... msg[key_name] + ... # Array values are set transparently + ... msg[array_key] = [1, 2, 3] + ... # Messages can be written to file + ... with open(testfile, "w") as test: + ... msg.write(test) + ... # Messages can be cloned from other messages + ... msg2 = {classname}(clone=msg) + ... # If desired, messages can be closed manually or used in with + ... msg.close() + """ + + #: ecCodes enum-like PRODUCT constant + product_kind = None + + def __init__(self, codes_file=None, clone=None, sample=None, + headers_only=False, other_args_found=False): + """ + Open a message and inform the host file that it's been incremented. + + If ``codes_file`` is not supplied, the message is cloned from + ``CodesMessage`` ``clone``. If neither is supplied, + the ``CodesMessage`` is cloned from ``sample``. + + :param codes_file: A file readable for ecCodes + :param clone: A valid ``CodesMessage`` + :param sample: A valid sample path to create ``CodesMessage`` from + """ + if (not other_args_found and codes_file is None and clone is None and + sample is None): + raise RuntimeError("CodesMessage initialization parameters not " + "present.") + #: Unique ID, for ecCodes interface + self.codes_id = None + #: File containing message + self.codes_file = None + if codes_file is not None: + self.codes_id = eccodes.codes_new_from_file( + codes_file.file_handle, self.product_kind, headers_only) + if self.codes_id is None: + raise IOError("CodesFile %s is exhausted" % codes_file.name) + self.codes_file = codes_file + self.codes_file.message += 1 + self.codes_file.open_messages.append(self) + elif clone is not None: + self.codes_id = eccodes.codes_clone(clone.codes_id) + elif sample is not None: + self.codes_id = self.new_from_sample(sample) + + def write(self, outfile=None): + """Write message to file.""" + if not outfile: + # This is a hack because the API does not accept inheritance + outfile = self.codes_file.file_handle + eccodes.codes_write(self.codes_id, outfile) + + def __setitem__(self, key, value): + """ + Set value associated with key. + + Iterables and scalars are handled intelligently. + """ + # Passed value is iterable and not string + if hasattr(value, "__iter__"): + eccodes.codes_set_array(self.codes_id, key, value) + else: + eccodes.codes_set(self.codes_id, key, value) + + def keys(self, namespace=None): + """Get available keys in message.""" + iterator = eccodes.codes_keys_iterator_new(self.codes_id, + namespace=namespace) + keys = [] + while eccodes.codes_keys_iterator_next(iterator): + key = eccodes.codes_keys_iterator_get_name(iterator) + keys.append(key) + eccodes.codes_keys_iterator_delete(iterator) + return keys + + def size(self): + """Return size of message in bytes.""" + return eccodes.codes_get_message_size(self.codes_id) + + def dump(self): + """Dump message's binary content.""" + return eccodes.codes_get_message(self.codes_id) + + def get(self, key, ktype=None): + """Get value of a given key as its native or specified type.""" + if self.missing(key): + raise KeyError("Key is missing from message.") + if eccodes.codes_get_size(self.codes_id, key) > 1: + ret = eccodes.codes_get_array(self.codes_id, key, ktype) + else: + ret = eccodes.codes_get(self.codes_id, key, ktype) + return ret + + def missing(self, key): + """Report if key is missing.""" + return bool(eccodes.codes_is_missing(self.codes_id, key)) + + def set_missing(self, key): + """Set a key to missing.""" + eccodes.codes_set_missing(self.codes_id, key) + + def __exit__(self, exc_type, exc_val, exc_tb): + """Release message handle and inform host file of release.""" + eccodes.codes_release(self.codes_id) + + def __enter__(self): + return self + + def close(self): + """Possibility to manually close message.""" + self.__exit__(None, None, None) + + def __contains__(self, key): + """Check whether a key is present in message.""" + return key in self.keys() + + def __len__(self): + """Return key count.""" + return len(self.keys()) + + def __getitem__(self, key): + """Return value associated with key as its native type.""" + return self.get(key) + + def __iter__(self): + return iter(self.keys()) + + # Not yet implemented + # def itervalues(self): + # return self.values() + + def items(self): + """Return list of tuples of all key/value pairs.""" + return [(key, self[key]) for key in self.keys()] + diff --git a/python/eccodes/high_level/gribfile.py b/python/eccodes/high_level/gribfile.py index a6d4ab153..6e074fe09 100644 --- a/python/eccodes/high_level/gribfile.py +++ b/python/eccodes/high_level/gribfile.py @@ -5,61 +5,13 @@ messages when it is no longer needed. Author: Daniel Lee, DWD, 2014 """ -from .. import eccodes +from .codesfile import CodesFile from .gribmessage import GribMessage -class GribFile(file): - """ - A GRIB file handle meant for use in a context manager. +class GribFile(CodesFile): - Individual messages can be accessed using the ``next`` method. Of course, - it is also possible to iterate over each message in the file:: + __doc__ = "\n".join(CodesFile.__doc__.splitlines()[4:]).format( + prod_type="GRIB", classname="GribFile", alias="grib") - >>> with GribFile(filename) as grib: - ... # Print number of messages in file - ... len(grib) - ... # Open all messages in file - ... for msg in grib: - ... print(msg["shortName"]) - ... len(grib.open_messages) - >>> # When the file is closed, any open messages are closed - >>> len(grib.open_messages) - """ - - def __enter__(self): - return self - - def __exit__(self, exception_type, exception_value, traceback): - """Close all open messages, release GRIB file handle and close file.""" - while self.open_messages: - self.open_messages.pop().close() - self.file_handle.close() - - def close(self): - """Possibility to manually close file.""" - self.__exit__(None, None, None) - - def __len__(self): - """Return total messages in GRIB file.""" - return eccodes.codes_count_in_file(self.file_handle) - - def __iter__(self): - return self - - def next(self): - try: - return GribMessage(self) - except IOError: - raise StopIteration() - - def __init__(self, filename, mode="r"): - """Open file and receive GRIB file handle.""" - #: File handle for working with actual file on disc - #: The class holds the file it works with because the GRIB API's - #: typechecking does not allow using inherited classes. - self.file_handle = open(filename, mode) - #: Number of message in GRIB file currently being read - self.message = 0 - #: Open messages - self.open_messages = [] + MessageClass = GribMessage diff --git a/python/eccodes/high_level/gribmessage.py b/python/eccodes/high_level/gribmessage.py index 5a0ab2278..00c3613b6 100644 --- a/python/eccodes/high_level/gribmessage.py +++ b/python/eccodes/high_level/gribmessage.py @@ -6,8 +6,7 @@ message when it is no longer needed, coordinating this with its host file. Author: Daniel Lee, DWD, 2014 """ -import collections - +from .codesmessage import CodesMessage from .. import eccodes @@ -15,185 +14,59 @@ class IndexNotSelectedError(Exception): """GRIB index was requested before selecting key/value pairs.""" -class GribMessage(object): - """ - A GRIB message. +class GribMessage(CodesMessage): - Each ``GribMessage`` is stored as a key/value pair in a dictionary-like - structure. It can be used in a context manager or by itself. When the - ``GribFile`` it belongs to is closed, it closes any open ``GribMessage``s - that belong to it. If a ``GribMessage`` is closed before its ``GribFile`` - is closed, it informs the ``GribFile`` of its closure. + __doc__ = "\n".join(CodesMessage.__doc__.splitlines()[4:]).format( + prod_type="GRIB", classname="GribMessage", parent="GribFile", + alias="grib") - Scalar and vector values are set appropriately through the same method. + product_kind = eccodes.CODES_PRODUCT_GRIB - Usage:: - - >>> with GribFile(filename) as grib: - ... # Access shortNames of all messages - ... for msg in grib: - ... print(msg["shortName"]) - ... # Report number of keys in message - ... len(msg) - ... # Report message size in bytes - ... msg.size - ... # Report keys in message - ... msg.keys() - ... # Check if value is missing - ... msg.missing("scaleFactorOfSecondFixedSurface") - ... # Set scalar value - ... msg["scaleFactorOfSecondFixedSurface"] = 5 - ... # Check key's value - ... msg["scaleFactorOfSecondFixedSurface"] - ... # Set value to missing - ... msg.set_missing("scaleFactorOfSecondFixedSurface") - ... # Missing values raise exception when read with dict notation - ... msg["scaleFactorOfSecondFixedSurface"] - ... # Array values are set transparently - ... msg["values"] = [1, 2, 3] - ... # Messages can be written to file - ... with open(testfile, "w") as test: - ... msg.write(test) - ... # Messages can be cloned from other messages - ... msg2 = GribMessage(clone=msg) - ... # If desired, messages can be closed manually or used in with - ... msg.close() - """ - - def __enter__(self): - return self - - def __exit__(self, exception_type, exception_value, traceback): - """Release GRIB message handle and inform file of release.""" - # This assert should never trigger - # assert self.gid is not None - eccodes.codes_release(self.gid) - if self.grib_index: - self.grib_index.open_messages.remove(self) - - def close(self): - """Possibility to manually close message.""" - self.__exit__(None, None, None) - - def __contains__(self, key): - """Check whether a key is present in message.""" - return key in self.keys() - - def __len__(self): - """Return key count.""" - return len(self.keys()) - - def __getitem__(self, key): - """Return value associated with key as its native type.""" - return self.get(key) - - def __setitem__(self, key, value): - """ - Set value associated with key. - - If the object is iterable, - """ - # Alternative implemented (TODO: evaluate) - # if eccodes.codes_get_size(self.gid, key) > 1: - # eccodes.codes_set_array(self.gid, key, value) - # else: - # eccodes.codes_set(self.gid, key, value) - - # Passed value is iterable and not string - if (isinstance(value, collections.Iterable) and not - isinstance(value, basestring)): - eccodes.codes_set_array(self.gid, key, value) - else: - eccodes.codes_set(self.gid, key, value) - - def __iter__(self): - return iter(self.keys()) - - # Not yet implemented - # def itervalues(self): - # return self.values() - - def items(self): - """Return list of tuples of all key/value pairs.""" - return [(key, self[key]) for key in self.keys()] - - def keys(self, namespace=None): - """Get available keys in message.""" - iterator = eccodes.codes_keys_iterator_new(self.gid, namespace=namespace) - keys = [] - while eccodes.codes_keys_iterator_next(iterator): - key = eccodes.codes_keys_iterator_get_name(iterator) - keys.append(key) - eccodes.codes_keys_iterator_delete(iterator) - return keys - - def __init__(self, grib_file=None, clone=None, sample=None, gribindex=None): + # Arguments included explicitly to support introspection + def __init__(self, codes_file=None, clone=None, sample=None, + headers_only=False, gribindex=None): """ Open a message and inform the GRIB file that it's been incremented. - If ``grib_file`` is not supplied, the message is cloned from - ``GribMessage`` ``clone``. If neither is supplied, the ``GribMessage`` - is cloned from ``sample``. If ``index`` is supplied as a GribIndex, the - message is taken from the index. + The message is taken from ``codes_file``, cloned from ``clone`` or + ``sample``, or taken from ``index``, in that order of precedence. """ - #: Unique GRIB ID, for GRIB API interface - self.gid = None - #: File containing message - self.grib_file = None + grib_args_present = True + if gribindex is None: + grib_args_present = False + super(self.__class__, self).__init__(codes_file, clone, sample, + headers_only, grib_args_present) #: GribIndex referencing message self.grib_index = None - if grib_file is not None: - self.gid = eccodes.codes_grib_new_from_file(grib_file.file_handle) - if self.gid is None: - raise IOError("Grib file %s is exhausted." % grib_file.name) - self.grib_file = grib_file - self.grib_file.message += 1 - self.grib_file.open_messages.append(self) - elif clone is not None: - self.gid = eccodes.codes_clone(clone.gid) - elif sample is not None: - self.gid = eccodes.codes_grib_new_from_samples(sample) - elif gribindex is not None: - self.gid = eccodes.codes_new_from_index(gribindex.iid) - if not self.gid: + if gribindex is not None: + self.codes_id = eccodes.codes_new_from_index(gribindex.iid) + if not self.codes_id: raise IndexNotSelectedError("All keys must have selected " "values before receiving message " "from index.") self.grib_index = gribindex gribindex.open_messages.append(self) - else: - raise RuntimeError("Either grib_file, clone, sample or gribindex " - "must be provided.") - def size(self): - """Return size of message in bytes.""" - return eccodes.codes_get_message_size(self.gid) + def __exit__(self, exc_type, exc_val, exc_tb): + """Release GRIB message handle and inform file of release.""" + super(self.__class__, self).__exit__(exc_type, exc_val, exc_tb) + if self.grib_index: + self.grib_index.open_messages.remove(self) - def dump(self): - """Dump message's binary content.""" - return eccodes.codes_get_message(self.gid) + @property + def gid(self): + """Provided for backwards compatibility.""" + return self.codes_id - def get(self, key, ktype=None): - """Get value of a given key as its native or specified type.""" - if self.missing(key): - raise KeyError("Value of key %s is MISSING." % key) - if eccodes.codes_get_size(self.gid, key) > 1: - ret = eccodes.codes_get_array(self.gid, key, ktype) - else: - ret = eccodes.codes_get(self.gid, key, ktype) - return ret + @property + def grib_file(self): + """Provided for backwards compatibility.""" + return self.codes_file - def missing(self, key): - """Report if key is missing.""" - return bool(eccodes.codes_is_missing(self.gid, key)) + @gid.setter + def gid(self, val): + self.codes_id = val - def set_missing(self, key): - """Set a key to missing.""" - eccodes.codes_set_missing(self.gid, key) - - def write(self, outfile=None): - """Write message to file.""" - if not outfile: - # This is a hack because the API does not accept inheritance - outfile = self.grib_file.file_handle - eccodes.codes_write(self.gid, outfile) + @grib_file.setter + def grib_file(self, val): + self.codes_file = val