From a1d80702ba49135dbdfcd87cd2450ba83d685010 Mon Sep 17 00:00:00 2001
From: thopri <thopri@noc.ac.uk>
Date: Tue, 28 Apr 2020 16:14:42 +0100
Subject: [PATCH] fixed CMEMS download bug and updated troubleshooting
---
docs/source/CMEMS_downloader_usage.rst | 5 ++
docs/source/troubleshooting.rst | 3 +
inputs/namelist_cmems.bdy | 27 +++++----
inputs/namelist_remote.bdy | 12 ++--
pynemo/nemo_bdy_dl_cmems.py | 1 -
pynemo/nemo_bdy_ncpop.py | 2 +-
pynemo/profile.py | 81 +++++++++++++-------------
7 files changed, 70 insertions(+), 61 deletions(-)
diff --git a/docs/source/CMEMS_downloader_usage.rst b/docs/source/CMEMS_downloader_usage.rst
index 83536f8..26ab132 100644
--- a/docs/source/CMEMS_downloader_usage.rst
+++ b/docs/source/CMEMS_downloader_usage.rst
@@ -1,6 +1,11 @@
CMEMS downloader usage
=======================
+**IMPORTANT** The CMEMS downloader has only been tested with the GLOBAL_ANALYSIS_FORECAST_PHY_001_024 model and specifcally
+the hourly SSH and U V product. This also has temperature stored within it, but not salinity. Other models and products should work but are
+currently likely to need some changes to the code to cope with different variable names within the data. This will be fixed
+in a later release of PyNEMO that is able to handle different variable and tracer names.
+
PyNEMO has a CMEMS downloading function incorporated within it, this will download a section of the CMEMS global model (more models to be added)
'GLOBAL_ANALYSIS_FORECAST_PHY_001_024-TDS' for the defined time period in the namelist file
diff --git a/docs/source/troubleshooting.rst b/docs/source/troubleshooting.rst
index 2347dfd..c9a98fe 100644
--- a/docs/source/troubleshooting.rst
+++ b/docs/source/troubleshooting.rst
@@ -15,3 +15,6 @@ which may help to diagnose issues. E.g. ValueErrors that are result of a THREDDS
3. Getting this error 'Warning: Please make sure pyjnius is installed and jvm.dll/libjvm.so/libjvm.dylib is in the path' ?
* This error is displayed when the application cannot find the java installation on the local machine. please install a java 7.x runtime from http://www.oracle.com/technetwork/java/javase/downloads/jre7-downloads-1880261.html and append the path to the library in the system path. eg. on windows SET PATH="C:\\Program Files (x86)\\Java\\jre1.7\\bin\\client" on Linux in shell export LD_LIBRARY_PATH=/opt/java/jdk1.7.0_45/jre/lib/amd64/server:$LD_LIBRARY_PATH in osx export DYLD_LIBRARY_PATH=/System/Library/Java/JavaVirtualMachines/jdk1.7.0_09.jdk/Contents/Home/jre/lib/server:$DYLD_LIBRARY_PATH
+
+4. Pyjinus error? Socket Timeout? JVM issues? This sometimes happend when requesting tracer boundaries such as temp and salinty along with
+tidal boundaries. Re running PyNEMO usally works.
diff --git a/inputs/namelist_cmems.bdy b/inputs/namelist_cmems.bdy
index 20119d5..d57ecce 100644
--- a/inputs/namelist_cmems.bdy
+++ b/inputs/namelist_cmems.bdy
@@ -52,10 +52,10 @@
! CMEMS Data Source Configuration
!------------------------------------------------------------------------------
ln_use_cmems = .true.
- ln_download_cmems = .false.
+ ln_download_cmems = .true.
sn_cmems_dir = '/Users/thopri/Projects/PyNEMO/inputs/' ! where to download CMEMS input files (static and variable)
- ln_download_static = .true.
- ln_subset_static = .true.
+ ln_download_static = .false.
+ ln_subset_static = .false.
nn_num_retry = 4 ! how many times to retry CMEMS download after non critical errors?
!------------------------------------------------------------------------------
! CMEMS MOTU Configuration (for Boundary Data)
@@ -94,7 +94,7 @@
! (if ln_mask_file=.TRUE.)
ln_dyn2d = .false. ! boundary conditions for
! barotropic fields
- ln_dyn3d = .false. ! boundary conditions for
+ ln_dyn3d = .false. ! boundary conditions for
! baroclinic velocities
ln_tra = .true. ! boundary conditions for T and S
ln_ice = .false. ! ice boundary condition
@@ -104,11 +104,13 @@
! unstructured open boundaries tidal parameters
!------------------------------------------------------------------------------
ln_tide = .false. ! =T : produce bdy tidal conditions
- sn_tide_model = 'FES' ! Name of tidal model (FES|TPXO)
+ sn_tide_model = 'fes' ! Name of tidal model (fes|tpxo)
clname(1) = 'M2' ! constituent name
- clname(2) = 'S2'
- clname(3) = 'K2'
- ln_trans = .true. ! interpolate transport rather than
+ clname(2) = 'S2'
+ clname(3) = 'O1'
+ clname(4) = 'K1'
+ clname(5) = 'N2'
+ ln_trans = .false. ! interpolate transport rather than
! velocities
!------------------------------------------------------------------------------
! Time information
@@ -119,9 +121,12 @@
nn_month_end = 03 ! month end (default = 12 is years>1)
sn_dst_calendar = 'gregorian' ! output calendar format
nn_base_year = 1960 ! base year for time counter
- sn_tide_grid = './src_data/tide/grid_tpxo7.2.nc'
- sn_tide_h = './src_data/tide/h_tpxo7.2.nc'
- sn_tide_u = './src_data/tide/u_tpxo7.2.nc'
+ ! TPXO file locations
+ sn_tide_grid = '/Users/thopri/Projects/PyNEMO/DATA/TPXO/grid_tpxo7.2.nc'
+ sn_tide_h = '/Users/thopri/Projects/PyNEMO/DATA/TPXO/h_tpxo7.2.nc'
+ sn_tide_u = '/Users/thopri/Projects/PyNEMO/DATA/TPXO/u_tpxo7.2.nc'
+ ! location of FES data
+ sn_tide_fes = '/Users/thopri/Projects/PyNEMO/DATA/FES/'
!------------------------------------------------------------------------------
! Additional parameters
diff --git a/inputs/namelist_remote.bdy b/inputs/namelist_remote.bdy
index 45bc2dc..eb8825b 100644
--- a/inputs/namelist_remote.bdy
+++ b/inputs/namelist_remote.bdy
@@ -58,17 +58,17 @@
! (if ln_mask_file=.TRUE.)
ln_dyn2d = .false. ! boundary conditions for
! barotropic fields
- ln_dyn3d = .true. ! boundary conditions for
+ ln_dyn3d = .false. ! boundary conditions for
! baroclinic velocities
ln_tra = .true. ! boundary conditions for T and S
- ln_ice = .false. ! ice boundary condition
+ ln_ice = .false. ! ice boundary condition
nn_rimwidth = 9 ! width of the relaxation zone
!------------------------------------------------------------------------------
! unstructured open boundaries tidal parameters
!------------------------------------------------------------------------------
ln_tide = .true. ! =T : produce bdy tidal conditions
- sn_tide_model = 'fes' ! Name of tidal model (fes|tpxo)
+ sn_tide_model = 'tpxo' ! Name of tidal model (fes|tpxo)
clname(1) = 'M2' ! constituent name
clname(2) = 'S2'
clname(3) = 'O1'
@@ -86,9 +86,9 @@
sn_dst_calendar = 'gregorian' ! output calendar format
nn_base_year = 1960 ! base year for time counter
! TPXO file locations
- sn_tide_grid = '/Users/thopri/Projects/PyNEMO/DATA/TPXO/grid_tpxo9.nc'
- sn_tide_h = '/Users/thopri/Projects/PyNEMO/DATA/TPXO/h_tpxo9.v2a.nc'
- sn_tide_u = '/Users/thopri/Projects/PyNEMO/DATA/TPXO/u_tpxo9.v2a.nc'
+ sn_tide_grid = '/Users/thopri/Projects/PyNEMO/DATA/TPXO/grid_tpxo7.2.nc'
+ sn_tide_h = '/Users/thopri/Projects/PyNEMO/DATA/TPXO/h_tpxo7.2.nc'
+ sn_tide_u = '/Users/thopri/Projects/PyNEMO/DATA/TPXO/u_tpxo7.2.nc'
! location of FES data
sn_tide_fes = '/Users/thopri/Projects/PyNEMO/DATA/FES/'
diff --git a/pynemo/nemo_bdy_dl_cmems.py b/pynemo/nemo_bdy_dl_cmems.py
index 616517c..e429aa8 100644
--- a/pynemo/nemo_bdy_dl_cmems.py
+++ b/pynemo/nemo_bdy_dl_cmems.py
@@ -19,7 +19,6 @@ from pynemo.utils import cmems_errors as errors
logger = logging.getLogger(__name__)
# TODO: Fix double spacing issue on CMEMS download log entries.
-# TODO: Add some sort of file check so CMEMS files that are already successfully downloaded aren't redownloaded
'''
This function checks to see if the MOTU client is installed on the PyNEMO python environment. If it is not installed
error code 1 is returned . If it is installed the version number of the installed client is returned as a string
diff --git a/pynemo/nemo_bdy_ncpop.py b/pynemo/nemo_bdy_ncpop.py
index 298945f..0548e54 100644
--- a/pynemo/nemo_bdy_ncpop.py
+++ b/pynemo/nemo_bdy_ncpop.py
@@ -19,7 +19,7 @@ def write_data_to_file(filename, variable_name, data):
count = data.shape
three_dim_variables = ['votemper', 'vosaline', 'N1p', 'N3n', 'N5s','thetao','so','uo','vo','vobtcrtx','vozocrtx','vobtcrty','vomecrty']
- two_dim_variables = ['sossheig', 'vobtcrtx', 'vobtcrty', 'iicethic', 'ileadfra', 'isnowthi','zos']
+ two_dim_variables = ['sossheig', 'iicethic', 'ileadfra', 'isnowthi','zos']
if variable_name in three_dim_variables:
if len(count) == 3:
diff --git a/pynemo/profile.py b/pynemo/profile.py
index d9c3846..0f126a8 100644
--- a/pynemo/profile.py
+++ b/pynemo/profile.py
@@ -202,9 +202,8 @@ def download_cmems(setup_filepath=0):
if err_chk == 2:
dl_cmems.clean_up(settings)
sys.exit(dl)
- if dl == 1:
- # if the request is too large try monthly intervals
- for re in range(settings['num_retry']):
+ if dl == 1:
+ # if the request is too large try monthly intervals
logger.warning('CMEMS request too large, try monthly downloads...(this may take awhile)')
mnth_dl = dl_cmems.MWD_request_cmems(settings, date_min, date_max, 'M')
if mnth_dl == 0:
@@ -227,52 +226,50 @@ def download_cmems(setup_filepath=0):
sys.exit(mnth_dl)
if mnth_dl == 1:
# if the request is too large try weekly intervals
- for re in range(settings['num_retry']):
- logger.warning('CMEMS request still too large, trying weekly downloads...(this will take longer...)')
- wk_dl = dl_cmems.MWD_request_cmems(settings, date_min, date_max, 'W')
- if wk_dl == 0:
- logger.info('CMEMS weekly request successful')
+ logger.warning('CMEMS request still too large, trying weekly downloads...(this will take longer...)')
+ wk_dl = dl_cmems.MWD_request_cmems(settings, date_min, date_max, 'W')
+ if wk_dl == 0:
+ logger.info('CMEMS weekly request successful')
+ break
+ if type(wk_dl) == str:
+ err_chk = dl_cmems.err_parse(wk_dl,'MOTU')
+ if err_chk == 0:
+ logger.info('retrying CMEMS download....retry number ' + str(re + 1) + ' of ' + str(settings['num_retry']))
+ if re == (settings['num_retry'] - 1):
+ logger.critical('reached retry limit defined in BDY file, exiting now')
+ logger.critical(wk_dl)
+ dl_cmems.clean_up(settings)
+ sys.exit(wk_dl)
+ if err_chk == 1:
+ dl_cmems.clean_up(settings)
+ sys.exit(wk_dl)
+ if err_chk == 2:
+ dl_cmems.clean_up(settings)
+ sys.exit(wk_dl)
+ if wk_dl == 1:
+ # if the request is too large try daily intervals.
+ logger.warning('CMESM request STILL too large, trying daily downloads....(even longer.....)')
+ dy_dl = dl_cmems.MWD_request_cmems(settings, date_min, date_max, 'D')
+ if dy_dl == 0:
+ logger.info('CMEMS daily request successful')
break
- if type(wk_dl) == str:
- err_chk = dl_cmems.err_parse(wk_dl,'MOTU')
+ # if the request is still too large then smaller domain is required.
+ if dy_dl == str:
+ # perform error check for retry
+ err_chk = dl_cmems.err_parse(dy_dl,'MOTU')
if err_chk == 0:
logger.info('retrying CMEMS download....retry number ' + str(re + 1) + ' of ' + str(settings['num_retry']))
if re == (settings['num_retry'] - 1):
logger.critical('reached retry limit defined in BDY file, exiting now')
- logger.critical(wk_dl)
+ logger.critical(dy_dl)
dl_cmems.clean_up(settings)
- sys.exit(wk_dl)
+ sys.exit(dy_dl)
if err_chk == 1:
dl_cmems.clean_up(settings)
- sys.exit(wk_dl)
+ sys.exit(dy_dl)
if err_chk == 2:
dl_cmems.clean_up(settings)
- sys.exit(wk_dl)
- if wk_dl == 1:
- # if the request is too large try daily intervals.
- for re in range(settings['num_retry']):
- logger.warning('CMESM request STILL too large, trying daily downloads....(even longer.....)')
- dy_dl = dl_cmems.MWD_request_cmems(settings, date_min, date_max, 'D')
- if dy_dl == 0:
- logger.info('CMEMS daily request successful')
- break
- # if the request is still too large then smaller domain is required.
- if dy_dl == str:
- # perform error check for retry
- err_chk = dl_cmems.err_parse(dy_dl,'MOTU')
- if err_chk == 0:
- logger.info('retrying CMEMS download....retry number ' + str(re + 1) + ' of ' + str(settings['num_retry']))
- if re == (settings['num_retry'] - 1):
- logger.critical('reached retry limit defined in BDY file, exiting now')
- logger.critical(dy_dl)
- dl_cmems.clean_up(settings)
- sys.exit(dy_dl)
- if err_chk == 1:
- dl_cmems.clean_up(settings)
- sys.exit(dy_dl)
- if err_chk == 2:
- dl_cmems.clean_up(settings)
- sys.exit(dy_dl)
+ sys.exit(dy_dl)
# end of messy if statements to split requests into months, weeks and days as needed.
dl_cmems.clean_up(settings)
logger.info('============================================')
@@ -513,7 +510,7 @@ def process_bdy(setup_filepath=0, mask_gui=False):
logger.info('using CMEMS variable names......')
if ln_tra:
var_in['t'].extend(['thetao'])
- var_in['t'].extend(['so'])
+ #var_in['t'].extend(['so'])
if ln_dyn2d or ln_dyn3d:
var_in['u'].extend(['uo'])
@@ -538,7 +535,7 @@ def process_bdy(setup_filepath=0, mask_gui=False):
var_in['t'].extend(['sossheig'])
if ln_ice:
- var_in['t'].extend(['ice1', 'ice2', 'ice3'])
+ var_in['t'].extend(['iicethic', 'ileadfra', 'isnowthi'])
if 'use_cmems' not in settings:
logger.info('using existing PyNEMO variable names.....')
@@ -553,7 +550,7 @@ def process_bdy(setup_filepath=0, mask_gui=False):
var_in['t'].extend(['sossheig'])
if ln_ice:
- var_in['t'].extend(['ice1', 'ice2', 'ice3'])
+ var_in['t'].extend(['iicethic', 'ileadfra', 'isnowthi'])
# As variables are associated with grd there must be a filename attached
# to each variable
--
GitLab