'''
Created on 6 Aug 2015
@author: Shirley Crompton, UK Science and Technology Facilities Council
'''
import logging
import os
import xml.etree.ElementTree as ET
from PyQt4 import QtGui
from PyQt4 import QtCore
from PyQt4.QtCore import pyqtSlot
from . import nemo_ncml_tab_widget
from thredds_crawler.crawl import Crawl
class Ncml_generator(QtGui.QDialog):
'''
Gui editor to capture user input for the purpose of generating NCML representation of pynemo source datasets.
'''
def __init__(self, basefile):
'''
Initialises the UI components
'''
super(Ncml_generator, self).__init__() # no params yet, may be allow user to predefine an input ncml for edit????
#Logging for class
self.logger = logging.getLogger(__name__) #logger config'ed in pynemo_exe.py
if not basefile:
testpath, file_name = os.path.split(__file__)
self.baseFile = os.path.join(testpath,'base.ncml')
else:
self.baseFile = basefile
print('ncml baseFile : ', str(self.baseFile))
self.filename = None # store the output file pointer
self.initUI()
def initUI(self):
QtGui.QToolTip.setFont(QtGui.QFont('SansSerif', 11))
'''
vbox is the top container
'''
#the
vbox = QtGui.QVBoxLayout(self)
vbox.setSpacing(10)
vbox.setContentsMargins(10, 10, 5, 5)
'''
top panel for output file
'''
top_outfile_label = QtGui.QLabel(str('Output filename').encode('utf-8'))
self.top_outfile_name = QtGui.QLineEdit() #location is pre-defined
self.top_outfile_name.setToolTip(str('Define output file').encode('utf-8'))
self.top_outfile_name.returnPressed.connect(self.get_fname_input)
top_outfile_button = QtGui.QPushButton(str('Select file').encode('utf-8'))
top_outfile_button.clicked.connect(self.get_fname)
top_grpBox = QtGui.QGroupBox(str('Define output file').encode('utf-8'), None)
top_grid = QtGui.QGridLayout(top_grpBox)
top_grid.setVerticalSpacing(5)
top_grid.setHorizontalSpacing(10)
top_grid.addWidget(top_outfile_label, 1, 0)
top_grid.addWidget(self.top_outfile_name, 1, 1)
top_grid.addWidget(top_outfile_button, 1,2, QtCore.Qt.AlignRight)
'''
middle panel for tab folder
'''
self.tabWidget = QtGui.QTabWidget()
self.tracer_tab = nemo_ncml_tab_widget.Ncml_tab(str("Tracer").encode('utf-8'))
self.tracer_tab.setEnabled(False)
self.dynamic_tab = nemo_ncml_tab_widget.Ncml_tab(str("Dynamics").encode('utf-8'))
self.dynamic_tab.setEnabled(False)
self.ice_tab = nemo_ncml_tab_widget.Ncml_tab(str("Ice").encode('utf-8'))
self.ice_tab.setEnabled(False)
self.ecosys_tab = nemo_ncml_tab_widget.Ncml_tab(str("Ecosystem").encode('utf-8'))
self.ecosys_tab.setEnabled(False)
self.grid_tab = nemo_ncml_tab_widget.Ncml_tab(str("Grid").encode('utf-8'))
self.grid_tab.setEnabled(False)
self.tabWidget.addTab(self.tracer_tab, str("Tracer").encode('utf-8'))
self.tabWidget.addTab(self.dynamic_tab, str("Dynamics").encode('utf-8'))
self.tabWidget.addTab(self.ice_tab, str("Ice").encode('utf-8'))
self.tabWidget.addTab(self.ecosys_tab, str("Ecosystem").encode('utf-8')) # should be disabled
self.tabWidget.addTab(self.grid_tab, str("Grid").encode('utf-8')) # should be disabled
self.tabWidget.setMovable(False)
# if self.tabWidget.widget(self.tabWidget.currentIndex()).isEnabled() is True:
# self.connect(self.tabWidget, SIGNAL('currentChanged(int)'),self.enable_btn_update)
self.tabWidget.currentChanged.connect(lambda: self.enable_btn_update(enable_btn))
'''
button bar
'''
go_btn = QtGui.QPushButton(str('Generate').encode('utf-8'))
go_btn.setToolTip(str('Add all variable definitions before generating NcML file.').encode('utf-8'))
cancel_btn = QtGui.QPushButton(str('Cancel').encode('utf-8'))
enable_btn = QtGui.QPushButton(str('Enable Tab').encode('utf-8'))
#layout button bar
btn_hBox = QtGui.QHBoxLayout(None)
btn_hBox.setMargin(5)
btn_hBox.setSpacing(10)
btn_hBox.setAlignment(QtCore.Qt.AlignRight)
btn_hBox.addWidget(enable_btn)
btn_hBox.addWidget(cancel_btn)
btn_hBox.addWidget(go_btn)
go_btn.clicked.connect(self.generate)
cancel_btn.clicked.connect(self.close)
enable_btn.clicked.connect(lambda: self.enable_tab(enable_btn))
# enable_btn.clicked.connect(self.enable_tab)
'''
Assemble the top layout container
'''
vbox.addWidget(top_grpBox)
vbox.addWidget(self.tabWidget)
vbox.addLayout(btn_hBox)
#self.setLayout(grp_box)
self.setWindowIcon(QtGui.QIcon('/Users/jdha/anaconda/lib/python2.7/site-packages/pynemo-0.2-py2.7.egg/pynemo/gui/nemo_icon.png')) #doesn't work
self.setWindowTitle(str("PyNEMO NcML Generator").encode('utf-8'))
self.resize(650,300)
#has to change the default focus to stop the output file QTextedit to trigger the widget in focus when enter is pressed. Not sure why this happens???
self.tabWidget.setFocus()
#show the window
self.show()
'''
file picker call back for output file input field
'''
@pyqtSlot()
def get_fname(self):
# When you call getOpenFileName, a file picker dialog is created
# and if the user selects a file, it's path is returned, and if not
# (ie, the user cancels the operation) None is returned
fname = QtGui.QFileDialog.getSaveFileName(self, 'Select output file', '', selectedFilter='*.ncml')
if fname:
self.filename = fname #returns a QString
self.top_outfile_name.setText(str(fname))
#print 'the output file is set to : ' + self.filename
'''
output file text box call back handler
'''
@pyqtSlot()
def get_fname_input(self):
self.filename = self.top_outfile_name.text()
#print 'the output file is manually set to : ' + self.filename
'''
call back to handle the generate button pressed
'''
@pyqtSlot()
def enable_btn_update(self, enable_btn):
if self.tabWidget.widget(self.tabWidget.currentIndex()).isEnabled() is True:
enable_btn.setText(str('Disable Tab').encode('utf-8'))
else:
enable_btn.setText(str('Enable Tab').encode('utf-8'))
'''
call back to handle the generate button pressed
'''
@pyqtSlot()
def enable_tab(self,enable_btn):
# def enable_tab(self):
#validate output file
if self.tabWidget.widget(self.tabWidget.currentIndex()).isEnabled() is True:
self.tabWidget.widget(self.tabWidget.currentIndex()).setEnabled(False)
enable_btn.setText(str('Enable Tab').encode('utf-8'))
else:
self.tabWidget.widget(self.tabWidget.currentIndex()).setEnabled(True)
enable_btn.setText(str('Disable Tab').encode('utf-8'))
'''
call back to handle the generate button pressed
'''
@pyqtSlot()
def generate(self):
#validate output file
if self.filename is None or self.filename == "":
if self.top_outfile_name.text() is None or self.top_outfile_name.text() == "":
QtGui.QMessageBox.critical(self, str('Something is wrong').encode('utf-8'), str('No output file specified!').encode('utf-8'), QtGui.QMessageBox.Ok, QtGui.QMessageBox.Ok)
return
else:
self.filename = self.top_outfile_name.text()
if(os.path.exists(os.path.dirname(str(self.filename)))) == False:
#if os.path.dirname(os.path.dirname(os.path.exists(os.path.normpath(str(self.filename))))) == False:
QtGui.QMessageBox.critical(self, str('Something is wrong').encode('utf-8'), str('Invalid output directory! Cannot generate file!').encode('utf-8'), QtGui.QMessageBox.Ok, QtGui.QMessageBox.Ok)
#print 'invalid target directory! Cannot generate.'
return
#validate if all the variables are defined, use the mandatory src field as a proxy
# also need to check that the tab is active
tabsList = []
if self.tracer_tab.isEnabled() is True:
if self.tracer_tab.votemper.src != "" and \
self.tracer_tab.vosaline.src != "" :
tabsList.extend([self.tracer_tab.votemper, self.tracer_tab.vosaline])
else:
QtGui.QMessageBox.information(self, str('Something is wrong').encode('utf-8'), str('Not all the variables under the tracer tab have been defined!').encode('utf-8'), QtGui.QMessageBox.Ok, QtGui.QMessageBox.Ok)
if self.ice_tab.isEnabled() is True:
if self.ice_tab.ileadfra.src != "" and \
self.ice_tab.iicethic.src != "" and \
self.ice_tab.isnowthi.src != "" :
tabsList.extend([self.ice_tab.iicethic, self.ice_tab.ileadfra, self.ice_tab.isnowthi])
else:
QtGui.QMessageBox.information(self, str('Something is wrong').encode('utf-8'), str('Not all the variables under the ice tab have been defined!').encode('utf-8'), QtGui.QMessageBox.Ok, QtGui.QMessageBox.Ok)
if self.dynamic_tab.isEnabled() is True:
if self.dynamic_tab.vozocrtx.src != "" and \
self.dynamic_tab.vozocrtx.src != "" and \
self.dynamic_tab.sossheig.src != "" :
tabsList.extend([self.dynamic_tab.vozocrtx, self.dynamic_tab.vomecrty, self.dynamic_tab.sossheig])
else:
QtGui.QMessageBox.information(self, str('Something is wrong').encode('utf-8'), str('Not all the variables under the dynamics tab have been defined!').encode('utf-8'), QtGui.QMessageBox.Ok, QtGui.QMessageBox.Ok)
if self.grid_tab.isEnabled() is True:
if self.grid_tab.gdept.src != "" and \
self.grid_tab.gdepw.src != "" and \
self.grid_tab.mbathy.src != "" and \
self.grid_tab.e3t.src != "" and \
self.grid_tab.e3u.src != "" and \
self.grid_tab.e3v.src != "" :
tabsList.extend([self.grid_tab.gdept, self.grid_tab.gdepw, self.grid_tab.mbathy, self.grid_tab.e3t, self.grid_tab.e3u, self.grid_tab.e3v])
else:
QtGui.QMessageBox.information(self, str('Something is wrong').encode('utf-8'), str('Not all the variables under the grid tab have been defined!').encode('utf-8'), QtGui.QMessageBox.Ok, QtGui.QMessageBox.Ok)
try:
self.generateNcML(tabsList) #go ahead and do it
except:
raise
QtGui.QMessageBox.information(self, str('Success.').encode('utf-8'), str('NcML file generated.').encode('utf-8'), QtGui.QMessageBox.Ok, QtGui.QMessageBox.Ok)
'''
Function to generates the NcML text and write it to the user defined output file
'''
def generateNcML(self, tabsList):
#first open the default base file
ns = '{http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2}'
self.tree = self._parseNcml()
self.root = self.tree.getroot()
#create a netcdf element for each tab variable
for tab in tabsList:
netcdfE = ET.Element(ns+str('netcdf').encode('utf-8')) #src directory is converted to the correct format when added/
if str(tab.src).startswith("http:") or str(tab.src).startswith("https:"):
#Its url so use thredds crawler to get the urls
urls = self.url_trawler(tab.src,str(tab.regex))
aggE = ET.Element(ns+str('aggregation').encode('utf-8'), name=str(str(tab.name)).encode('utf-8'), type=str('joinExisting').encode('utf-8'), dimName=str('time_counter').encode('utf-8')) #tab.name already encoded
for nc_url in urls:
tcNetcdf = ET.Element(ns+str('netcdf').encode('utf-8'), location=str(str(nc_url)).encode('utf-8'))
aggE.append(tcNetcdf)
netcdfE.append(aggE)
else:
scanE = ET.Element(ns+str('scan').encode('utf-8'), location=str(str(tab.src)).encode('utf-8'), regExp=str(str(tab.regex)).encode('utf-8'))
if tab.subdirs == True:
scanE.set(str('subdirs').encode('utf-8'), str('true').encode('utf-8'))
aggE = ET.Element(ns+str('aggregation').encode('utf-8'), name=str(str(tab.name)).encode('utf-8'), type=str('joinExisting').encode('utf-8'), dimName=str('time_counter').encode('utf-8')) #tab.name already encoded
aggE.append(scanE)
netcdfE.append(aggE)
self.root[0].append(netcdfE) #add the new netcdf element to the top aggregation
#deal with variable name change TODO put this into a loop?
if tab.old_name is not None and tab.old_name != "":
vname = str('variable').encode('utf-8')
#v is None
if tab.name == str('temperature').encode('utf-8') and tab.old_name != str('votemper').encode('utf-8'):
v = ET.Element(ns+vname, name='votemper', orgName = str(tab.old_name))
self.root.append(v)
elif tab.name == str('salinity').encode('utf-8') and tab.old_name != str('vosaline').encode('utf-8'):
v = ET.Element(ns+vname, name='vosaline', orgName = str(tab.old_name))
self.root.append(v)
elif tab.name == str('ice_thickness').encode('utf-8') and tab.old_name != str('iicethic').encode('utf-8'):
v = ET.Element(ns+vname, name='iicethic', orgName = str(tab.old_name))
self.root.append(v)
elif tab.name == str('leads_fraction').encode('utf-8') and tab.old_name != str('ileadfra').encode('utf-8'):
v = ET.Element(ns+vname, name='ileadfra', orgName = str(tab.old_name))
self.root.append(v)
elif tab.name == str('snow_thickness').encode('utf-8') and tab.old_name != str('isnowthi').encode('utf-8'):
v = ET.Element(ns+vname, name='isnowthi', orgName = str(tab.old_name))
self.root.append(v)
elif tab.name == str('zonal_velocity').encode('utf-8') and tab.old_name != str('vozocrtx').encode('utf-8'):
v = ET.Element(ns+vname, name='vozocrtx', orgName = str(tab.old_name))
self.root.append(v)
elif tab.name == str('meridian_velocity').encode('utf-8') and tab.old_name != str('vomecrty').encode('utf-8'):
v = ET.Element(ns+vname, name='vomecrty', orgName = str(tab.old_name))
self.root.append(v)
elif tab.name == str('sea_surface_height').encode('utf-8') and tab.old_name != str('sossheig').encode('utf-8'):
v = ET.Element(ns+vname, name='sossheig', orgName = str(tab.old_name))
self.root.append(v)
elif tab.name == str('depth_at_t_points').encode('utf-8') and tab.old_name != str('gdept').encode('utf-8'):
v = ET.Element(ns+vname, name='gdept', orgName = str(tab.old_name))
self.root.append(v)
elif tab.name == str('depth_at_w_points').encode('utf-8') and tab.old_name != str('gdepw').encode('utf-8'):
v = ET.Element(ns+vname, name='gdepw', orgName = str(tab.old_name))
self.root.append(v)
elif tab.name == str('number_of_wet_levels').encode('utf-8') and tab.old_name != str('mbathy').encode('utf-8'):
v = ET.Element(ns+vname, name='mbathy', orgName = str(tab.old_name))
self.root.append(v)
elif tab.name == str('vertical_scale_factors_at_t_points').encode('utf-8') and tab.old_name != str('e3t').encode('utf-8'):
v = ET.Element(ns+vname, name='e3t', orgName = str(tab.old_name))
self.root.append(v)
elif tab.name == str('vertical_scale_factors_at_u_points').encode('utf-8') and tab.old_name != str('e3u').encode('utf-8'):
v = ET.Element(ns+vname, name='e3u', orgName = str(tab.old_name))
self.root.append(v)
elif tab.name == str('vertical_scale_factors_at_v_points').encode('utf-8') and tab.old_name != str('e3v').encode('utf-8'):
v = ET.Element(ns+vname, name='e3v', orgName = str(tab.old_name))
self.root.append(v)
#write ncml to file
try:
self.indent(self.root, 0) #24Aug15 format the xml for pretty printing
self.tree.write(self.filename, encoding='utf-8')
except IOError as xxx_todo_changeme:
(errno, strerror) = xxx_todo_changeme.args
self.logger.error("I/O error({0}): {1}".format(errno, strerror))
except:
self.logger.error('Error generating ncml file')
raise
'''
Function to retrieve the NcML file template
'''
def _parseNcml(self):
try:
parser = ET.XMLParser(encoding="utf-8")
tree = ET.parse(self.baseFile, parser=parser)
return tree
except ET.ParseError as v:
row, column = v.position
print("error on row", row, "column", column, ":", v)
'''
Function to format xml. Based on code provided by http://effbot.org/zone/element-lib
'''
def indent(self, elem, level=0):
i = "\n" + level*" "
if len(elem):
if not elem.text or not elem.text.strip():
elem.text = i + " "
if not elem.tail or not elem.tail.strip():
elem.tail = i
for elem in elem:
self.indent(elem, level+1)
if not elem.tail or not elem.tail.strip():
elem.tail = i
else:
if level and (not elem.tail or not elem.tail.strip()):
elem.tail = i
"""
This method trawls throught the url with a given expression and returns the
list of urls that match the expression
"""
def url_trawler(self, url, expr):
if url.endswith(".xml"):
c = Crawl(url, select=[expr])
elif url.endswith("/"): # we'll try and add catalog.xml as the user may have just provided a directory
c = Crawl(url+"catalog.xml", select=[expr])
else: # we'll try and add catalog.xml as the user may have just provided a directory
c = Crawl(url+"/catalog.xml", select=[expr])
urls = [s.get("url") for d in c.datasets for s in d.services if s.get("service").lower()=="opendap"]
return urls