'''
NcML reading implementation using pyjnius
@author: Mr. Srikanth Nagella
'''
# pylint: disable=E1103
# pylint: disable=no-name-in-module
#Loading of NCML jar file
import os
import string
import logging
import numpy as np
import jnius_config
from netCDF4 import netcdftime
ncmlpath, file_name = os.path.split(__file__)
ncmlpath = os.path.join(ncmlpath, "jars", "netcdfAll-4.6.jar")
jnius_config.set_classpath('.',ncmlpath)
try:
if os.environ['http_proxy'] is not None:
#split the proxy name and port
proxylist = string.split(os.environ['http_proxy'],':')
proxy_host = proxylist[0]
proxy_port = proxylist[1]
jnius_config.add_options('-Dhttp.proxyHost='+proxy_host,'-Dhttp.proxyPort='+proxy_port)
except:
print("Didn't find a proxy environment variable")
NetcdfDataset = None
NcMLReader = None
Section = None
try:
from jnius import autoclass
def init_jnius():
global NetcdfDataset
global NcMLReader
global Section
NetcdfDataset = autoclass('ucar.nc2.dataset.NetcdfDataset')
NcMLReader = autoclass('ucar.nc2.ncml.NcMLReader')
Section = autoclass('ucar.ma2.Section')
init_jnius()
except ImportError:
print('Warning: Please make sure pyjnius is installed and jvm.dll/libjvm.so/libjvm.dylib is in the path')
time_counter_const = "time_counter"
class Reader(object):
""" This class is the high level of object for the NCML reader, from here using grid type
will return the grid data
usage:
>>> reader = Reader("NCML Filename")
>>> reader['t']['votemper'][:,:,:,:]
"""
grid_type_list = ['t','u','v','i']
time_counter = time_counter_const
def __init__(self, uri, time_adjust):
self.uri = uri
self.time_adjust = time_adjust
try:
self.dataset = NetcdfDataset.openFile(self.uri, None)
except (IOError, RuntimeError):
self.logger.error('Cannot open the file '+self.uri)
self.grid = GridGroup(self.uri,self.dataset)
self._get_source_timedata(self.grid,self.time_adjust)
def __del__(self):
""" Destructor close the netcdf file """
if self.dataset is not None:
self.dataset.close()
def __getitem__(self,val):
""" Returns the grid. it doesn't matter what type of grid is requested all the
variables are in the same object. This is to keep it consistent with the Local directory
reader"""
if val in self.grid_type_list:
return self.grid
else:
return None
def _get_source_timedata(self, grid, t_adjust):
""" Get the source time data information. builds up sourcedata objects of a given grid """
timevar = grid[self.time_counter]
grid.time_counter = timevar[:]+t_adjust
grid.date_counter = []
for index in range(0,len(grid.time_counter)):
grid.date_counter.append(netcdftime.utime(grid.units,
grid.calendar).num2date(grid.time_counter[index]))
def close(self):
""" This is not yet implemented. TODO: keep the netcdf file open until its expicitly
closed """
pass
class GridGroup(object):
""" This class is to provide an indirection to the grid type. since ncml
file has aggregation of all the variables this is just a place holder"""
logger = logging.getLogger(__name__)
def __init__(self, filename, dataset):
""" This class is the source data that holds the dataset information """
self.file_name = filename
self.units = None
self.calendar = None
self.date_counter = None
self.seconds = None
self.time_counter = None
self.dataset = dataset
self.update_atrributes()
def __del__(self):
self.dataset = None
def __getitem__(self, val):
""" Returns the data requested """
try:
return Variable(self.dataset, val)
except KeyError:
self.logger.error('Cannot find the requested variable '+self.variable)
except (IOError, RuntimeError):
self.logger.error('Cannot open the file '+self.file_name)
return None
def get_meta_data(self, variable, source_dic):
""" Returns a dictionary with meta data information correspoinding to the variable """
#source_dic = {}
try:
dvar = self.dataset.findVariable(variable)
if dvar is None:
raise KeyError()
source_dic['sf'] = 1
source_dic['os'] = 0
mv_attr = dvar.findAttributeIgnoreCase('missing_value')
if mv_attr is not None:
source_dic['mv'] = mv_attr.getValues().copyToNDJavaArray()
sf_attr = dvar.findAttributeIgnoreCase('scale_factor')
if sf_attr is not None:
source_dic['sf'] = sf_attr.getValues().copyToNDJavaArray()
os_attr = dvar.findAttributeIgnoreCase('add_offset')
if os_attr is not None:
source_dic['os'] = os_attr.getValues().copyToNDJavaArray()
fv_attr = dvar.findAttributeIgnoreCase('_FillValue')
if fv_attr is not None:
source_dic['mv'] = fv_attr.getValue(0)
return source_dic
except KeyError:
self.logger.error('Cannot find the requested variable '+variable)
except (IOError, RuntimeError):
self.logger.error('Cannot open the file '+self.file_name)
return None
def update_atrributes(self):
""" Updates the units and calendar information for the grid """
try:
var = Variable(self.dataset, time_counter_const)
self.units = var.get_attribute_value("units")
self.calendar = var.get_attribute_value("calendar")
except KeyError:
self.logger.error('Cannot find the requested variable '+self.variable)
except (IOError, RuntimeError):
self.logger.error('Cannot open the file '+self.file_name)
return None
class Variable(object):
def __init__(self, dataset, variable):
self.logger = logging.getLogger(__name__)
self.dataset = dataset
self.variable = variable
def __str__(self):
return "PyNEMO NcML Object from file: %s and variable %s" % self.file_name, self.variable
def __len__(self):
""" Returns the length of the variable """
try:
dvar = self.dataset.findVariable(self.variable)
if dvar is None:
raise KeyError()
val = dvar.getDimension(0).getLength()
return val
except KeyError:
self.logger.error('Cannot find the requested variable '+self.variable)
return None
def __getitem__(self, val):
""" Returns the data requested """
if type(val) != tuple:
val = (val,)
try:
dvar = self.dataset.findVariable(self.variable)
if dvar is None:
raise KeyError()
dims = dvar.getShape()
# get the requested slice and extract that information from jnius
# Check if the request data is with in the dataset dimensions
start = [0]*len(dims)
stop = dims
stride = [1]*len(dims)
new_dims = tuple()
np_input = False
for idx in range(0,len(dims)):
try:
if val[idx].start is not None:
start[idx] = val[idx].start
if val[idx].step is not None:
stride[idx] = val[idx].step
if val[idx].stop is not None:
if val[idx].stop == -1:
val[idx] = stop[idx] - 1
elif val[idx].stop > stop[idx]:
val[idx].stop = stop[idx]
stop[idx] = (val[idx].stop - start[idx])//stride[idx]
if (val[idx].stop - start[idx])%stride[idx] != 0:
stop[idx] = stop[idx] + 1
new_dims = new_dims+(stop[idx],)
except IndexError:
pass
except AttributeError:
if isinstance(val[idx],int):
start[idx] = val[idx]
stop[idx] = 1
elif isinstance(val[idx],np.ndarray):
new_dims = new_dims+(val[idx].shape)
np_input = True
# Create a section object that represents the requested slice
start = [int(i) for i in start]
stop = [int(i) for i in stop]
stride = [int(i) for i in stride]
selected_section = Section(start,stop,stride)
data_array = dvar.read(selected_section)
retval = data_array.copyToNDJavaArray()
#TODO: copy it into numpy instead of Java array and then convert to numpy
# convert to numpy array
retval = np.asarray(retval)
self.logger.info(retval.shape)
if np_input: #if an array is passed as selection
ret_dim_list = ()
for idx in range(0,len(dims)):
if isinstance(val[idx], np.ndarray):
ret_dim_list2 = ret_dim_list+(val[idx],)
# can't do all the reductions at once due to Index Error: shape mismatch
retval = retval[ret_dim_list2]
ret_dim_list = ret_dim_list+(slice(None,None,None),)
self.logger.info(ret_dim_list)
self.logger.info(retval.shape)
self.logger.info(ret_dim_list)
# reshape to reflect the request
retval = np.reshape(retval, new_dims)
return retval
except KeyError:
self.logger.error('Cannot find the requested variable '+self.variable)
except (IOError, RuntimeError):
self.logger.error('Cannot open the file '+self.file_name)
return None
def _get_dimensions(self):
""" Returns the dimensions of the variables """
try:
dvar = self.dataset.findVariable(self.variable)
if dvar is None:
raise KeyError()
retval = tuple(dvar.getDimensionsString().split(' '))
return retval
except KeyError:
self.logger.error('Cannot find the requested variable '+self.variable)
return None
def get_attribute_value(self, attr_name):
""" Returns the attribute value of the variable """
try:
dvar = self.dataset.findVariable(self.variable)
if dvar is None:
raise KeyError()
attr = dvar.findAttributeIgnoreCase(attr_name)
if attr is not None:
retval = attr.getValue(0)
return retval
except KeyError:
self.logger.error('Cannot find the requested variable '+self.variable)
return None
class NcMLFile(object):
def __init__(self,filename):
self.dataset = None
try:
self.dataset = NetcdfDataset.openFile(filename, None)
except (IOError, RuntimeError):
self.logger.error('Cannot open the file '+filename)
def __getitem__(self,val):
return Variable(self.dataset,val)
def close(self):
if self.dataset is not None:
self.dataset.close()