Skip to content

GitLab

Commit 52e87bc9 authored by thopri's avatar thopri
Browse files

updated CMEMS request handling

parent dabe1205
Hide whitespace changes
Inline Side-by-side
Showing with 27 additions and 28 deletions
inputs/namelist_cmems.bdy
View file @ 52e87bc9
......@@ -56,7 +56,7 @@
sn_cmems_dir = '/Users/thopri/Projects/PyNEMO/inputs/' ! where to download CMEMS input files (static and variable)
ln_download_static = .false.
ln_subset_static = .false.
nn_num_restart = 2
nn_num_retry = 2 ! how many times to retry CMEMS download after non critical errors?
!------------------------------------------------------------------------------
! CMEMS MOTU Configuration (for Boundary Data)
!------------------------------------------------------------------------------
......@@ -72,7 +72,6 @@
sn_ftp_server = 'nrt.cmems-du.eu'
sn_static_dir = '/Core/GLOBAL_ANALYSIS_FORECAST_PHY_001_024/global-analysis-forecast-phy-001-024-statics'
sn_static_filenames = 'GLO-MFC_001_024_coordinates.nc GLO-MFC_001_024_mask_bathy.nc GLO-MFC_001_024_mdt.nc'
sn_cmems_dir = '/Users/thopri/Projects/PyNEMO/inputs/' ! where to download CMEMS static files
sn_cdo_loc = '/opt/local/bin/cdo' ! location of cdo executable can be found by running "where cdo"
!------------------------------------------------------------------------------
! CMEMS Extent Configuration
......@@ -115,9 +114,9 @@
! Time information
!------------------------------------------------------------------------------
nn_year_000 = 2017 ! year start
nn_year_end = 2018 ! year end
nn_year_end = 2017 ! year end
nn_month_000 = 01 ! month start (default = 1 is years>1)
nn_month_end = 04 ! month end (default = 12 is years>1)
nn_month_end = 03 ! month end (default = 12 is years>1)
sn_dst_calendar = 'gregorian' ! output calendar format
nn_base_year = 1960 ! base year for time counter
sn_tide_grid = './src_data/tide/grid_tpxo7.2.nc'
......
pynemo/nemo_bdy_dl_cmems.py
View file @ 52e87bc9
......@@ -163,8 +163,8 @@ def MWD_request_cmems(args,date_min,date_max,F):
logger.error(dy_dl)
return dy_dl
if (not F == 'M' or not F == 'W' or not F == 'D'):
time_int_err = 'incorrect character used to define time download interval please use M, W or D'
if not any('M W D' in F):
time_int_err = 'incorrect string used to define time download interval please use M, W or D'
logger.error(time_int_err)
return time_int_err
......
pynemo/profile.py
View file @ 52e87bc9
......@@ -104,7 +104,7 @@ def download_cmems(setup_filepath=0):
Setup = setup.Setup(setup_filepath) # default settings file
settings = Setup.settings
if settings['download_static'] == True:
for re in range(settings['num_restart']):
for re in range(settings['num_retry']):
logger.info('CMEMS Static data requested: downloading static data now.... (this may take awhile)')
static = dl_cmems.get_static(settings)
if static == 0:
......@@ -113,9 +113,9 @@ def download_cmems(setup_filepath=0):
if type(static) == str:
err_chk = dl_cmems.err_parse(static,'FTP')
if err_chk == 0:
logger.info('retrying FTP download....restart number '+str(re+1)+' of '+str(settings['num_restart']) )
if re == (settings['num_restart']-1):
logger.critical('reached restart limit defined in BDY file, exiting now')
logger.info('retrying FTP download....retry number '+str(re+1)+' of '+str(settings['num_retry']) )
if re == (settings['num_retry']-1):
logger.critical('reached retry limit defined in BDY file, exiting now')
logger.critical(static)
dl_cmems.clean_up(settings)
sys.exit(static)
......@@ -177,7 +177,7 @@ def download_cmems(setup_filepath=0):
dl_cmems.clean_up(settings)
sys.exit(error_msg)
# download request for CMEMS data, try whole time interval first.
for re in range(settings['num_restart']):
for re in range(settings['num_retry']):
logger.info('starting CMES download now (this can take a while)...')
dl = dl_cmems.request_cmems(settings, date_min, date_max)
if dl == 0:
......@@ -187,15 +187,15 @@ def download_cmems(setup_filepath=0):
if type(dl) == str:
# check error message against logged errors
err_chk = dl_cmems.err_parse(dl,'MOTU')
# error is known and restarting is likely to work
# error is known and retry is likely to work
if err_chk == 0:
logger.info('retrying CMEMS download....restart number '+str(re+1)+' of '+str(settings['num_restart']) )
if re == (settings['num_restart']-1):
logger.critical('reached restart limit defined in BDY file, exiting now')
logger.info('retrying CMEMS download....retry number '+str(re+1)+' of '+str(settings['num_retry']) )
if re == (settings['num_retry']-1):
logger.critical('reached retry limit defined in BDY file, exiting now')
logger.critical(dl)
dl_cmems.clean_up(settings)
sys.exit(dl)
# error is known and restarting is likely to not work
# error is known and retry is likely to not work
if err_chk == 1:
dl_cmems.clean_up(settings)
sys.exit(dl)
......@@ -205,7 +205,7 @@ def download_cmems(setup_filepath=0):
sys.exit(dl)
if dl == 1:
# if the request is too large try monthly intervals
for re in range(settings['num_restart']):
for re in range(settings['num_retry']):
logger.warning('CMEMS request too large, try monthly downloads...(this may take awhile)')
mnth_dl = dl_cmems.MWD_request_cmems(settings, date_min, date_max, 'M')
if mnth_dl == 0:
......@@ -214,9 +214,9 @@ def download_cmems(setup_filepath=0):
if type(mnth_dl) == str:
err_chk = dl_cmems.err_parse(mnth_dl,'MOTU')
if err_chk == 0:
logger.info('retrying CMEMS download....restart number '+str(re+1)+' of '+str(settings['num_restart']) )
if re == (settings['num_restart']-1):
logger.critical('reached restart limit defined in BDY file, exiting now')
logger.info('retrying CMEMS download....retry number '+str(re+1)+' of '+str(settings['num_retry']) )
if re == (settings['num_retry']-1):
logger.critical('reached retry limit defined in BDY file, exiting now')
logger.critical(mnth_dl)
dl_cmems.clean_up(settings)
sys.exit(mnth_dl)
......@@ -228,7 +228,7 @@ def download_cmems(setup_filepath=0):
sys.exit(mnth_dl)
if mnth_dl == 1:
# if the request is too large try weekly intervals
for re in range(settings['num_restart']):
for re in range(settings['num_retry']):
logger.warning('CMEMS request still too large, trying weekly downloads...(this will take longer...)')
wk_dl = dl_cmems.MWD_request_cmems(settings, date_min, date_max, 'W')
if wk_dl == 0:
......@@ -237,9 +237,9 @@ def download_cmems(setup_filepath=0):
if type(wk_dl) == str:
err_chk = dl_cmems.err_parse(wk_dl,'MOTU')
if err_chk == 0:
logger.info('retrying CMEMS download....restart number ' + str(re + 1) + ' of ' + str(settings['num_restart']))
if re == (settings['num_restart'] - 1):
logger.critical('reached restart limit defined in BDY file, exiting now')
logger.info('retrying CMEMS download....retry number ' + str(re + 1) + ' of ' + str(settings['num_retry']))
if re == (settings['num_retry'] - 1):
logger.critical('reached retry limit defined in BDY file, exiting now')
logger.critical(wk_dl)
dl_cmems.clean_up(settings)
sys.exit(wk_dl)
......@@ -251,7 +251,7 @@ def download_cmems(setup_filepath=0):
sys.exit(wk_dl)
if wk_dl == 1:
# if the request is too large try daily intervals.
for re in range(settings['num_restart']):
for re in range(settings['num_retry']):
logger.warning('CMESM request STILL too large, trying daily downloads....(even longer.....)')
dy_dl = dl_cmems.MWD_request_cmems(settings, date_min, date_max, 'D')
if dy_dl == 0:
......@@ -262,9 +262,9 @@ def download_cmems(setup_filepath=0):
# perform error check for retry
err_chk = dl_cmems.err_parse(dy_dl,'MOTU')
if err_chk == 0:
logger.info('retrying CMEMS download....restart number ' + str(re + 1) + ' of ' + str(settings['num_restart']))
if re == (settings['num_restart'] - 1):
logger.critical('reached restart limit defined in BDY file, exiting now')
logger.info('retrying CMEMS download....retry number ' + str(re + 1) + ' of ' + str(settings['num_retry']))
if re == (settings['num_retry'] - 1):
logger.critical('reached retry limit defined in BDY file, exiting now')
logger.critical(dy_dl)
dl_cmems.clean_up(settings)
sys.exit(dy_dl)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment