ECC-1231: Python bindings: Remove the experimental high-level interface

This commit is contained in:
Shahram Najm 2021-04-11 17:07:02 +01:00
parent af98a69aa1
commit 418157db49
11 changed files with 3 additions and 565 deletions

View File

@ -102,11 +102,9 @@ else()
)
endif()
# The high level python test requires new features in the unittest
# Some tests require new features
# which are only there for python 2.7 onwards
if( HAVE_PYTHON2 AND PYTHON_VERSION_STRING VERSION_GREATER "2.7" )
#ecbuild_info("Python examples: Adding test for High-level Pythonic Interface")
list( APPEND tests_extra high_level_api )
list( APPEND tests_extra grib_set_keys ) # Uses OrderedDict
endif()

View File

@ -1,9 +1,7 @@
#!/bin/env python
"""
Unit tests for high level Python interface for GRIB.
Author: Daniel Lee, DWD, 2016
This is now deprecated. Use cfgrib instead
"""
import os

View File

@ -4,8 +4,3 @@ import sys
from .eccodes import *
from .eccodes import __version__
if sys.version_info >= (2, 6):
from .high_level.gribfile import GribFile
from .high_level.gribmessage import GribMessage
from .high_level.gribindex import GribIndex
from .high_level.bufr import BufrFile, BufrMessage

View File

@ -1,97 +0,0 @@
"""
Classes for handling BUFR with a high level interface.
``BufrFiles`` can be treated mostly as regular files and used as context
managers, as can ``BufrMessages``. Each of these classes destructs itself and
any child instances appropriately.
Author: Daniel Lee, DWD, 2016
"""
from .. import eccodes
from .codesmessage import CodesMessage
from .codesfile import CodesFile
class BufrMessage(CodesMessage):
__doc__ = "\n".join(CodesMessage.__doc__.splitlines()[4:]).format(
prod_type="BUFR", classname="BufrMessage", parent="BufrFile",
alias="bufr")
product_kind = eccodes.CODES_PRODUCT_BUFR
# Arguments included explicitly to support introspection
# TODO: Can we get this to work with an index?
def __init__(self, codes_file=None, clone=None, sample=None,
headers_only=False):
"""
Open a message and inform the GRIB file that it's been incremented.
The message is taken from ``codes_file``, cloned from ``clone`` or
``sample``, or taken from ``index``, in that order of precedence.
"""
super(self.__class__, self).__init__(codes_file, clone, sample,
headers_only)
#self._unpacked = False
#def get(self, key, ktype=None):
# """Return requested value, unpacking data values if necessary."""
# # TODO: Only do this if accessing arrays that need unpacking
# if not self._unpacked:
# self.unpacked = True
# return super(self.__class__, self).get(key, ktype)
#def missing(self, key):
# """
# Report if key is missing.#
#
# Overloaded due to confusing behaviour in ``codes_is_missing`` (SUP-1874).
# """
# return not bool(eccodes.codes_is_defined(self.codes_id, key))
def unpack(self):
"""Decode data section"""
eccodes.codes_set(self.codes_id, 'unpack', 1)
def pack(self):
"""Encode data section"""
eccodes.codes_set(self.codes_id, 'pack', 1)
def keys(self, namespace=None):
#self.unpack()
#return super(self.__class__, self).keys(namespace)
iterator = eccodes.codes_bufr_keys_iterator_new(self.codes_id)
keys = []
while eccodes.codes_bufr_keys_iterator_next(iterator):
key = eccodes.codes_bufr_keys_iterator_get_name(iterator)
keys.append(key)
eccodes.codes_bufr_keys_iterator_delete(iterator)
return keys
#@property
#def unpacked(self):
# return self._unpacked
#@unpacked.setter
#def unpacked(self, val):
# eccodes.codes_set(self.codes_id, "unpack", val)
# self._unpacked = val
#def __setitem__(self, key, value):
# """Set item and pack BUFR."""
# if not self._unpacked:
# self.unpacked = True
# super(self.__class__, self).__setitem__(key, value)
# eccodes.codes_set(self.codes_id, "pack", True)
def copy_data(self, destMsg):
"""Copy data values from this message to another message"""
return eccodes.codes_bufr_copy_data(self.codes_id, destMsg.codes_id)
class BufrFile(CodesFile):
__doc__ = "\n".join(CodesFile.__doc__.splitlines()[4:]).format(
prod_type="BUFR", classname="BufrFile", alias="bufr")
MessageClass = BufrMessage

View File

@ -1,71 +0,0 @@
"""
``CodesFile`` class that implements a file that is readable by ecCodes and
closes itself and its messages when it is no longer needed.
Author: Daniel Lee, DWD, 2016
"""
from .. import eccodes
class CodesFile(file):
"""
An abstract class to specify and/or implement common behaviour that files
read by ecCodes should implement.
A {prod_type} file handle meant for use in a context manager.
Individual messages can be accessed using the ``next`` method. Of course,
it is also possible to iterate over each message in the file::
>>> with {classname}(filename) as {alias}:
... # Print number of messages in file
... len({alias})
... # Open all messages in file
... for msg in {alias}:
... print(msg[key_name])
... len({alias}.open_messages)
>>> # When the file is closed, any open messages are closed
>>> len({alias}.open_messages)
"""
#: Type of messages belonging to this file
MessageClass = None
def __init__(self, filename, mode="r"):
"""Open file and receive codes file handle."""
#: File handle for working with actual file on disc
#: The class holds the file it works with because ecCodes'
# typechecking does not allow using inherited classes.
self.file_handle = open(filename, mode)
#: Number of message in file currently being read
self.message = 0
#: Open messages
self.open_messages = []
def __exit__(self, exception_type, exception_value, traceback):
"""Close all open messages, release file handle and close file."""
while self.open_messages:
self.open_messages.pop().close()
self.file_handle.close()
def __len__(self):
"""Return total number of messages in file."""
return eccodes.codes_count_in_file(self.file_handle)
def __enter__(self):
return self
def close(self):
"""Possibility to manually close file."""
self.__exit__(None, None, None)
def __iter__(self):
return self
def next(self):
try:
return self.MessageClass(self)
except IOError:
raise StopIteration()

View File

@ -1,186 +0,0 @@
"""
``CodesMessage`` class that implements a message readable by ecCodes that
allows access to the message's key-value pairs in a dictionary-like manner
and closes the message when it is no longer needed, coordinating this with
its host file.
Author: Daniel Lee, DWD, 2016
"""
from .. import eccodes
class CodesMessage(object):
"""
An abstract class to specify and/or implement common behaviour that
messages read by ecCodes should implement.
A {prod_type} message.
Each ``{classname}`` is stored as a key/value pair in a dictionary-like
structure. It can be used in a context manager or by itself. When the
``{parent}`` it belongs to is closed, the ``{parent}`` closes any open
``{classname}``s that belong to it. If a ``{classname}`` is closed before
its ``{parent}`` is closed, it informs the ``{parent}`` of its closure.
Scalar and vector values are set appropriately through the same method.
``{classname}``s can be instantiated from a ``{parent}``, cloned from
other ``{classname}``s or taken from samples. Iterating over the members
of a ``{parent}`` extracts the ``{classname}``s it contains until the
``{parent}`` is exhausted.
Usage::
>>> with {parent}(filename) as {alias}:
... # Access a key from each message
... for msg in {alias}:
... print(msg[key_name])
... # Report number of keys in message
... len(msg)
... # Report message size in bytes
... msg.size
... # Report keys in message
... msg.keys()
... # Set scalar value
... msg[scalar_key] = 5
... # Check key's value
... msg[scalar_key]
... msg[key_name]
... # Array values are set transparently
... msg[array_key] = [1, 2, 3]
... # Messages can be written to file
... with open(testfile, "w") as test:
... msg.write(test)
... # Messages can be cloned from other messages
... msg2 = {classname}(clone=msg)
... # If desired, messages can be closed manually or used in with
... msg.close()
"""
#: ecCodes enum-like PRODUCT constant
product_kind = None
def __init__(self, codes_file=None, clone=None, sample=None,
headers_only=False, other_args_found=False):
"""
Open a message and inform the host file that it's been incremented.
If ``codes_file`` is not supplied, the message is cloned from
``CodesMessage`` ``clone``. If neither is supplied,
the ``CodesMessage`` is cloned from ``sample``.
:param codes_file: A file readable for ecCodes
:param clone: A valid ``CodesMessage``
:param sample: A valid sample path to create ``CodesMessage`` from
"""
if not other_args_found and codes_file is None and clone is None and sample is None:
raise RuntimeError("CodesMessage initialization parameters not "
"present.")
#: Unique ID, for ecCodes interface
self.codes_id = None
#: File containing message
self.codes_file = None
if codes_file is not None:
self.codes_id = eccodes.codes_new_from_file(
codes_file.file_handle, self.product_kind, headers_only)
if self.codes_id is None:
raise IOError("CodesFile %s is exhausted" % codes_file.name)
self.codes_file = codes_file
self.codes_file.message += 1
self.codes_file.open_messages.append(self)
elif clone is not None:
self.codes_id = eccodes.codes_clone(clone.codes_id)
elif sample is not None:
self.codes_id = eccodes.codes_new_from_samples(
sample, self.product_kind)
def write(self, outfile=None):
"""Write message to file."""
if not outfile:
# This is a hack because the API does not accept inheritance
outfile = self.codes_file.file_handle
eccodes.codes_write(self.codes_id, outfile)
def __setitem__(self, key, value):
"""
Set value associated with key.
Iterables and scalars are handled intelligently.
"""
# Passed key is iterable. Value has to be iterable too
if hasattr(key, "__iter__"):
if type(key) != type(value):
raise TypeError('Key must have same type as value')
if len(key) != len(value):
raise ValueError('Key array must have same size as value array')
eccodes.codes_set_key_vals(self.codes_id,",".join([str(key[i])+"="+str(value[i]) for i in range(len(key))]))
elif hasattr(value, "__iter__"):
# Passed value is iterable and not string
eccodes.codes_set_array(self.codes_id, key, value)
else:
eccodes.codes_set(self.codes_id, key, value)
def keys(self, namespace=None):
"""Get available keys in message."""
iterator = eccodes.codes_keys_iterator_new(self.codes_id,
namespace=namespace)
keys = []
while eccodes.codes_keys_iterator_next(iterator):
key = eccodes.codes_keys_iterator_get_name(iterator)
keys.append(key)
eccodes.codes_keys_iterator_delete(iterator)
return keys
def size(self):
"""Return size of message in bytes."""
return eccodes.codes_get_message_size(self.codes_id)
def dump(self):
"""Dump message's binary content."""
return eccodes.codes_get_message(self.codes_id)
def get(self, key, ktype=None):
"""Get value of a given key as its native or specified type."""
# if self.missing(key):
# raise KeyError("Value of key %s is MISSING." % key)
if eccodes.codes_get_size(self.codes_id, key) > 1:
ret = eccodes.codes_get_array(self.codes_id, key, ktype)
else:
ret = eccodes.codes_get(self.codes_id, key, ktype)
return ret
def __exit__(self, exc_type, exc_val, exc_tb):
"""Release message handle and inform host file of release."""
eccodes.codes_release(self.codes_id)
def __enter__(self):
return self
def close(self):
"""Possibility to manually close message."""
self.__exit__(None, None, None)
def __contains__(self, key):
"""Check whether a key is present in message."""
return key in self.keys()
def __len__(self):
"""Return key count."""
return len(self.keys())
def __getitem__(self, key):
"""Return value associated with key as its native type."""
return self.get(key)
def __iter__(self):
return iter(self.keys())
# Not yet implemented
# def itervalues(self):
# return self.values()
def items(self):
"""Return list of tuples of all key/value pairs."""
return [(key, self[key]) for key in self.keys()]

View File

@ -1,17 +0,0 @@
"""
``GribFile`` class that implements a GRIB file that closes itself and its
messages when it is no longer needed.
Author: Daniel Lee, DWD, 2014
"""
from .codesfile import CodesFile
from .gribmessage import GribMessage
class GribFile(CodesFile):
__doc__ = "\n".join(CodesFile.__doc__.splitlines()[4:]).format(
prod_type="GRIB", classname="GribFile", alias="grib")
MessageClass = GribMessage

View File

@ -1,102 +0,0 @@
"""
``GribIndex`` class that implements a GRIB index that allows access to
ecCodes's index functionality.
Author: Daniel Lee, DWD, 2014
"""
from .. import eccodes
from .gribmessage import GribMessage
class GribIndex(object):
"""
A GRIB index meant for use in a context manager.
Usage::
>>> # Create index from file with keys
>>> with GribIndex(filename, keys) as idx:
... # Write index to file
... idx.write(index_file)
>>> # Read index from file
>>> with GribIndex(file_index=index_file) as idx:
... # Add new file to index
... idx.add(other_filename)
... # Report number of unique values for given key
... idx.size(key)
... # Report unique values indexed by key
... idx.values(key)
... # Request GribMessage matching key, value
... msg = idx.select({key: value})
"""
def __enter__(self):
return self
def __exit__(self, exception_type, exception_value, traceback):
"""Release GRIB message handle and inform file of release."""
while self.open_messages:
self.open_messages[0].close()
eccodes.codes_index_release(self.iid)
def close(self):
"""Possibility to manually close index."""
self.__exit__(None, None, None)
def __init__(self, filename=None, keys=None, file_index=None,
grib_index=None):
"""
Create new GRIB index over ``keys`` from ``filename``.
``filename`` should be a string of the desired file's filename.
``keys`` should be a sequence of keys to index. ``file_index`` should
be a string of the file that the index should be loaded from.
``grib_index`` should be another ``GribIndex``.
If ``filename`` and ``keys`` are provided, the ``GribIndex`` is
initialized over the given keys from the given file. If they are not
provided, the ``GribIndex`` is read from ``indexfile``. If
``grib_index`` is provided, it is cloned from the given ``GribIndex``.
"""
#: Grib index ID
self.iid = None
if filename and keys:
self.iid = eccodes.codes_index_new_from_file(filename, keys)
elif file_index:
self.iid = eccodes.codes_index_read(file_index)
elif grib_index:
self.iid = eccodes.codes_new_from_index(grib_index.iid)
else:
raise RuntimeError("No source was supplied "
"(possibilities: grib_file, clone, sample).")
#: Indexed keys. Only available if GRIB is initialized from file.
self.keys = keys
#: Open GRIB messages
self.open_messages = []
def size(self, key):
"""Return number of distinct values for index key."""
return eccodes.codes_index_get_size(self.iid, key)
def values(self, key, ktype=str):
"""Return distinct values of index key."""
return eccodes.codes_index_get(self.iid, key, ktype)
def add(self, filename):
"""Add ``filename`` to the ``GribIndex``."""
eccodes.codes_index_add_file(self.iid, filename)
def write(self, outfile):
"""Write index to filename at ``outfile``."""
eccodes.codes_index_write(self.iid, outfile)
def select(self, key_value_pairs):
"""
Return message associated with given key value pairs.
``key_value_pairs`` should be passed as a dictionary.
"""
for key in key_value_pairs:
eccodes.codes_index_select(self.iid, key, key_value_pairs[key])
return GribMessage(gribindex=self)

View File

@ -1,80 +0,0 @@
"""
``GribMessage`` class that implements a GRIB message that allows access to
the message's key-value pairs in a dictionary-like manner and closes the
message when it is no longer needed, coordinating this with its host file.
Author: Daniel Lee, DWD, 2014
"""
from .codesmessage import CodesMessage
from .. import eccodes
class IndexNotSelectedError(Exception):
"""GRIB index was requested before selecting key/value pairs."""
class GribMessage(CodesMessage):
__doc__ = "\n".join(CodesMessage.__doc__.splitlines()[4:]).format(
prod_type="GRIB", classname="GribMessage", parent="GribFile",
alias="grib")
product_kind = eccodes.CODES_PRODUCT_GRIB
# Arguments included explicitly to support introspection
def __init__(self, codes_file=None, clone=None, sample=None,
headers_only=False, gribindex=None):
"""
Open a message and inform the GRIB file that it's been incremented.
The message is taken from ``codes_file``, cloned from ``clone`` or
``sample``, or taken from ``index``, in that order of precedence.
"""
grib_args_present = True
if gribindex is None:
grib_args_present = False
super(self.__class__, self).__init__(codes_file, clone, sample,
headers_only, grib_args_present)
#: GribIndex referencing message
self.grib_index = None
if gribindex is not None:
self.codes_id = eccodes.codes_new_from_index(gribindex.iid)
if not self.codes_id:
raise IndexNotSelectedError("All keys must have selected "
"values before receiving message "
"from index.")
self.grib_index = gribindex
gribindex.open_messages.append(self)
def __exit__(self, exc_type, exc_val, exc_tb):
"""Release GRIB message handle and inform file of release."""
super(self.__class__, self).__exit__(exc_type, exc_val, exc_tb)
if self.grib_index:
self.grib_index.open_messages.remove(self)
def missing(self, key):
"""Report if the value of a key is MISSING."""
return bool(eccodes.codes_is_missing(self.codes_id, key))
def set_missing(self, key):
"""Set the value of key to MISSING."""
eccodes.codes_set_missing(self.codes_id, key)
@property
def gid(self):
"""Provided for backwards compatibility."""
return self.codes_id
@property
def grib_file(self):
"""Provided for backwards compatibility."""
return self.codes_file
@gid.setter
def gid(self, val):
self.codes_id = val
@grib_file.setter
def grib_file(self, val):
self.codes_file = val

View File

@ -74,4 +74,4 @@ setup(name='eccodes',
url='https://confluence.ecmwf.int/display/ECC/ecCodes+Home',
download_url='https://confluence.ecmwf.int/display/ECC/Releases',
ext_modules=[Extension('gribapi._gribapi_swig', **attdict)],
packages=['eccodes', 'eccodes.high_level', 'gribapi'])
packages=['eccodes', 'gribapi'])