Skip to content

GitLab

Commit a1d80702 authored by thopri's avatar thopri
Browse files

fixed CMEMS download bug and updated troubleshooting

parent b5b10a4d
Hide whitespace changes
Inline Side-by-side
Showing with 70 additions and 61 deletions
docs/source/CMEMS_downloader_usage.rst
View file @ a1d80702
CMEMS downloader usage
=======================
**IMPORTANT** The CMEMS downloader has only been tested with the GLOBAL_ANALYSIS_FORECAST_PHY_001_024 model and specifcally
the hourly SSH and U V product. This also has temperature stored within it, but not salinity. Other models and products should work but are
currently likely to need some changes to the code to cope with different variable names within the data. This will be fixed
in a later release of PyNEMO that is able to handle different variable and tracer names.
PyNEMO has a CMEMS downloading function incorporated within it, this will download a section of the CMEMS global model (more models to be added)
'GLOBAL_ANALYSIS_FORECAST_PHY_001_024-TDS' for the defined time period in the namelist file
......
docs/source/troubleshooting.rst
View file @ a1d80702
......@@ -15,3 +15,6 @@ which may help to diagnose issues. E.g. ValueErrors that are result of a THREDDS
3. Getting this error 'Warning: Please make sure pyjnius is installed and jvm.dll/libjvm.so/libjvm.dylib is in the path' ?
* This error is displayed when the application cannot find the java installation on the local machine. please install a java 7.x runtime from http://www.oracle.com/technetwork/java/javase/downloads/jre7-downloads-1880261.html and append the path to the library in the system path. eg. on windows SET PATH="C:\\Program Files (x86)\\Java\\jre1.7\\bin\\client" on Linux in shell export LD_LIBRARY_PATH=/opt/java/jdk1.7.0_45/jre/lib/amd64/server:$LD_LIBRARY_PATH in osx export DYLD_LIBRARY_PATH=/System/Library/Java/JavaVirtualMachines/jdk1.7.0_09.jdk/Contents/Home/jre/lib/server:$DYLD_LIBRARY_PATH
4. Pyjinus error? Socket Timeout? JVM issues? This sometimes happend when requesting tracer boundaries such as temp and salinty along with
tidal boundaries. Re running PyNEMO usally works.
inputs/namelist_cmems.bdy
View file @ a1d80702
......@@ -52,10 +52,10 @@
! CMEMS Data Source Configuration
!------------------------------------------------------------------------------
ln_use_cmems = .true.
ln_download_cmems = .false.
ln_download_cmems = .true.
sn_cmems_dir = '/Users/thopri/Projects/PyNEMO/inputs/' ! where to download CMEMS input files (static and variable)
ln_download_static = .true.
ln_subset_static = .true.
ln_download_static = .false.
ln_subset_static = .false.
nn_num_retry = 4 ! how many times to retry CMEMS download after non critical errors?
!------------------------------------------------------------------------------
! CMEMS MOTU Configuration (for Boundary Data)
......@@ -94,7 +94,7 @@
! (if ln_mask_file=.TRUE.)
ln_dyn2d = .false. ! boundary conditions for
! barotropic fields
ln_dyn3d = .false. ! boundary conditions for
ln_dyn3d = .false. ! boundary conditions for
! baroclinic velocities
ln_tra = .true. ! boundary conditions for T and S
ln_ice = .false. ! ice boundary condition
......@@ -104,11 +104,13 @@
! unstructured open boundaries tidal parameters
!------------------------------------------------------------------------------
ln_tide = .false. ! =T : produce bdy tidal conditions
sn_tide_model = 'FES' ! Name of tidal model (FES|TPXO)
sn_tide_model = 'fes' ! Name of tidal model (fes|tpxo)
clname(1) = 'M2' ! constituent name
clname(2) = 'S2'
clname(3) = 'K2'
ln_trans = .true. ! interpolate transport rather than
clname(2) = 'S2'
clname(3) = 'O1'
clname(4) = 'K1'
clname(5) = 'N2'
ln_trans = .false. ! interpolate transport rather than
! velocities
!------------------------------------------------------------------------------
! Time information
......@@ -119,9 +121,12 @@
nn_month_end = 03 ! month end (default = 12 is years>1)
sn_dst_calendar = 'gregorian' ! output calendar format
nn_base_year = 1960 ! base year for time counter
sn_tide_grid = './src_data/tide/grid_tpxo7.2.nc'
sn_tide_h = './src_data/tide/h_tpxo7.2.nc'
sn_tide_u = './src_data/tide/u_tpxo7.2.nc'
! TPXO file locations
sn_tide_grid = '/Users/thopri/Projects/PyNEMO/DATA/TPXO/grid_tpxo7.2.nc'
sn_tide_h = '/Users/thopri/Projects/PyNEMO/DATA/TPXO/h_tpxo7.2.nc'
sn_tide_u = '/Users/thopri/Projects/PyNEMO/DATA/TPXO/u_tpxo7.2.nc'
! location of FES data
sn_tide_fes = '/Users/thopri/Projects/PyNEMO/DATA/FES/'
!------------------------------------------------------------------------------
! Additional parameters
......
inputs/namelist_remote.bdy
View file @ a1d80702
......@@ -58,17 +58,17 @@
! (if ln_mask_file=.TRUE.)
ln_dyn2d = .false. ! boundary conditions for
! barotropic fields
ln_dyn3d = .true. ! boundary conditions for
ln_dyn3d = .false. ! boundary conditions for
! baroclinic velocities
ln_tra = .true. ! boundary conditions for T and S
ln_ice = .false. ! ice boundary condition
ln_ice = .false. ! ice boundary condition
nn_rimwidth = 9 ! width of the relaxation zone
!------------------------------------------------------------------------------
! unstructured open boundaries tidal parameters
!------------------------------------------------------------------------------
ln_tide = .true. ! =T : produce bdy tidal conditions
sn_tide_model = 'fes' ! Name of tidal model (fes|tpxo)
sn_tide_model = 'tpxo' ! Name of tidal model (fes|tpxo)
clname(1) = 'M2' ! constituent name
clname(2) = 'S2'
clname(3) = 'O1'
......@@ -86,9 +86,9 @@
sn_dst_calendar = 'gregorian' ! output calendar format
nn_base_year = 1960 ! base year for time counter
! TPXO file locations
sn_tide_grid = '/Users/thopri/Projects/PyNEMO/DATA/TPXO/grid_tpxo9.nc'
sn_tide_h = '/Users/thopri/Projects/PyNEMO/DATA/TPXO/h_tpxo9.v2a.nc'
sn_tide_u = '/Users/thopri/Projects/PyNEMO/DATA/TPXO/u_tpxo9.v2a.nc'
sn_tide_grid = '/Users/thopri/Projects/PyNEMO/DATA/TPXO/grid_tpxo7.2.nc'
sn_tide_h = '/Users/thopri/Projects/PyNEMO/DATA/TPXO/h_tpxo7.2.nc'
sn_tide_u = '/Users/thopri/Projects/PyNEMO/DATA/TPXO/u_tpxo7.2.nc'
! location of FES data
sn_tide_fes = '/Users/thopri/Projects/PyNEMO/DATA/FES/'
......
pynemo/nemo_bdy_dl_cmems.py
View file @ a1d80702
......@@ -19,7 +19,6 @@ from pynemo.utils import cmems_errors as errors
logger = logging.getLogger(__name__)
# TODO: Fix double spacing issue on CMEMS download log entries.
# TODO: Add some sort of file check so CMEMS files that are already successfully downloaded aren't redownloaded
'''
This function checks to see if the MOTU client is installed on the PyNEMO python environment. If it is not installed
error code 1 is returned . If it is installed the version number of the installed client is returned as a string
......
pynemo/nemo_bdy_ncpop.py
View file @ a1d80702
......@@ -19,7 +19,7 @@ def write_data_to_file(filename, variable_name, data):
count = data.shape
three_dim_variables = ['votemper', 'vosaline', 'N1p', 'N3n', 'N5s','thetao','so','uo','vo','vobtcrtx','vozocrtx','vobtcrty','vomecrty']
two_dim_variables = ['sossheig', 'vobtcrtx', 'vobtcrty', 'iicethic', 'ileadfra', 'isnowthi','zos']
two_dim_variables = ['sossheig', 'iicethic', 'ileadfra', 'isnowthi','zos']
if variable_name in three_dim_variables:
if len(count) == 3:
......
pynemo/profile.py
View file @ a1d80702
......@@ -202,9 +202,8 @@ def download_cmems(setup_filepath=0):
if err_chk == 2:
dl_cmems.clean_up(settings)
sys.exit(dl)
if dl == 1:
# if the request is too large try monthly intervals
for re in range(settings['num_retry']):
if dl == 1:
# if the request is too large try monthly intervals
logger.warning('CMEMS request too large, try monthly downloads...(this may take awhile)')
mnth_dl = dl_cmems.MWD_request_cmems(settings, date_min, date_max, 'M')
if mnth_dl == 0:
......@@ -227,52 +226,50 @@ def download_cmems(setup_filepath=0):
sys.exit(mnth_dl)
if mnth_dl == 1:
# if the request is too large try weekly intervals
for re in range(settings['num_retry']):
logger.warning('CMEMS request still too large, trying weekly downloads...(this will take longer...)')
wk_dl = dl_cmems.MWD_request_cmems(settings, date_min, date_max, 'W')
if wk_dl == 0:
logger.info('CMEMS weekly request successful')
logger.warning('CMEMS request still too large, trying weekly downloads...(this will take longer...)')
wk_dl = dl_cmems.MWD_request_cmems(settings, date_min, date_max, 'W')
if wk_dl == 0:
logger.info('CMEMS weekly request successful')
break
if type(wk_dl) == str:
err_chk = dl_cmems.err_parse(wk_dl,'MOTU')
if err_chk == 0:
logger.info('retrying CMEMS download....retry number ' + str(re + 1) + ' of ' + str(settings['num_retry']))
if re == (settings['num_retry'] - 1):
logger.critical('reached retry limit defined in BDY file, exiting now')
logger.critical(wk_dl)
dl_cmems.clean_up(settings)
sys.exit(wk_dl)
if err_chk == 1:
dl_cmems.clean_up(settings)
sys.exit(wk_dl)
if err_chk == 2:
dl_cmems.clean_up(settings)
sys.exit(wk_dl)
if wk_dl == 1:
# if the request is too large try daily intervals.
logger.warning('CMESM request STILL too large, trying daily downloads....(even longer.....)')
dy_dl = dl_cmems.MWD_request_cmems(settings, date_min, date_max, 'D')
if dy_dl == 0:
logger.info('CMEMS daily request successful')
break
if type(wk_dl) == str:
err_chk = dl_cmems.err_parse(wk_dl,'MOTU')
# if the request is still too large then smaller domain is required.
if dy_dl == str:
# perform error check for retry
err_chk = dl_cmems.err_parse(dy_dl,'MOTU')
if err_chk == 0:
logger.info('retrying CMEMS download....retry number ' + str(re + 1) + ' of ' + str(settings['num_retry']))
if re == (settings['num_retry'] - 1):
logger.critical('reached retry limit defined in BDY file, exiting now')
logger.critical(wk_dl)
logger.critical(dy_dl)
dl_cmems.clean_up(settings)
sys.exit(wk_dl)
sys.exit(dy_dl)
if err_chk == 1:
dl_cmems.clean_up(settings)
sys.exit(wk_dl)
sys.exit(dy_dl)
if err_chk == 2:
dl_cmems.clean_up(settings)
sys.exit(wk_dl)
if wk_dl == 1:
# if the request is too large try daily intervals.
for re in range(settings['num_retry']):
logger.warning('CMESM request STILL too large, trying daily downloads....(even longer.....)')
dy_dl = dl_cmems.MWD_request_cmems(settings, date_min, date_max, 'D')
if dy_dl == 0:
logger.info('CMEMS daily request successful')
break
# if the request is still too large then smaller domain is required.
if dy_dl == str:
# perform error check for retry
err_chk = dl_cmems.err_parse(dy_dl,'MOTU')
if err_chk == 0:
logger.info('retrying CMEMS download....retry number ' + str(re + 1) + ' of ' + str(settings['num_retry']))
if re == (settings['num_retry'] - 1):
logger.critical('reached retry limit defined in BDY file, exiting now')
logger.critical(dy_dl)
dl_cmems.clean_up(settings)
sys.exit(dy_dl)
if err_chk == 1:
dl_cmems.clean_up(settings)
sys.exit(dy_dl)
if err_chk == 2:
dl_cmems.clean_up(settings)
sys.exit(dy_dl)
sys.exit(dy_dl)
# end of messy if statements to split requests into months, weeks and days as needed.
dl_cmems.clean_up(settings)
logger.info('============================================')
......@@ -513,7 +510,7 @@ def process_bdy(setup_filepath=0, mask_gui=False):
logger.info('using CMEMS variable names......')
if ln_tra:
var_in['t'].extend(['thetao'])
var_in['t'].extend(['so'])
#var_in['t'].extend(['so'])
if ln_dyn2d or ln_dyn3d:
var_in['u'].extend(['uo'])
......@@ -538,7 +535,7 @@ def process_bdy(setup_filepath=0, mask_gui=False):
var_in['t'].extend(['sossheig'])
if ln_ice:
var_in['t'].extend(['ice1', 'ice2', 'ice3'])
var_in['t'].extend(['iicethic', 'ileadfra', 'isnowthi'])
if 'use_cmems' not in settings:
logger.info('using existing PyNEMO variable names.....')
......@@ -553,7 +550,7 @@ def process_bdy(setup_filepath=0, mask_gui=False):
var_in['t'].extend(['sossheig'])
if ln_ice:
var_in['t'].extend(['ice1', 'ice2', 'ice3'])
var_in['t'].extend(['iicethic', 'ileadfra', 'isnowthi'])
# As variables are associated with grd there must be a filename attached
# to each variable
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment