diff --git a/inputs/namelist_cmems.bdy b/inputs/namelist_cmems.bdy
index 2883afb3daff2578d81827e84aaeea438b16c236..531ad6499faa7cf2681b47786df3aacc2056b77d 100644
--- a/inputs/namelist_cmems.bdy
+++ b/inputs/namelist_cmems.bdy
@@ -56,7 +56,7 @@
sn_cmems_dir = '/Users/thopri/Projects/PyNEMO/inputs/' ! where to download CMEMS input files (static and variable)
ln_download_static = .false.
ln_subset_static = .false.
- nn_num_retry = 4 ! how many times to retry CMEMS download after non critical errors?
+ nn_num_retry = 1 ! how many times to retry CMEMS download after non critical errors?
!------------------------------------------------------------------------------
! CMEMS MOTU Configuration (for Boundary Data)
!------------------------------------------------------------------------------
diff --git a/pynemo/nemo_bdy_dl_cmems.py b/pynemo/nemo_bdy_dl_cmems.py
index e1642a74eea491f0d2121444be915ec402dfa579..54fcbc5e50cdc1283e8541fd55bdc1604333e08a 100644
--- a/pynemo/nemo_bdy_dl_cmems.py
+++ b/pynemo/nemo_bdy_dl_cmems.py
@@ -11,6 +11,7 @@ import ftplib
import re
import pandas as pd
from datetime import datetime
+from pathlib import Path
import glob
import os
#local imports
@@ -18,6 +19,7 @@ from pynemo.utils import cmems_errors as errors
logger = logging.getLogger(__name__)
# TODO: Fix double spacing issue on CMEMS download log entries.
+# TODO: Add some sort of file check so CMEMS files that are already successfully downloaded aren't redownloaded
'''
This function checks to see if the MOTU client is installed on the PyNEMO python environment. If it is not installed
error code 1 is returned . If it is installed the version number of the installed client is returned as a string
@@ -51,6 +53,7 @@ def get_static(args):
logger.error('Unable to import CMEMS credentials, see Readme for instructions on adding to PyNEMO')
return 'Unable to import credential file, have you created one?'
try:
+ logger.info('connecting to FTP host......')
ftp = ftplib.FTP(host=args['ftp_server'], user=CMEMS_cred.user, passwd=CMEMS_cred.pwd)
except ftplib.error_temp:
return 'temporary error in FTP connection, please try running PyNEMO again........'
@@ -64,10 +67,13 @@ def get_static(args):
# TODO: add try excepts to handle issues with files being missing etc.
# TODO: Check there is enough space to download as well.....
# TODO: Handle timeouts etc as well......
+ logger.info('navigating to download directoy.......')
ftp.cwd(args['static_dir'])
+ logger.info('generating download filename list......')
filenames = args['static_filenames'].split(' ')
for f in filenames:
try:
+ logger.info('downloading '+f+' now......')
ftp.retrbinary("RETR " + f, open(args['cmems_dir']+f, 'wb').write)
except ftplib.error_temp:
return 'temporary error in FTP download, please try running PyNEMO again........'
@@ -243,48 +249,55 @@ def request_cmems(args, date_min, date_max):
filedata = filedata.replace('4Y4LMQLAKP10YFUE', ','.join(grids[key]))
filedata = filedata.replace('QFCN2P56ZQSA7YNK', locs[key])
filedata = filedata.replace('YSLTB459ZW0P84GE', args['dl_prefix']+'_'+str(date_min)+'_'+str(date_max)+'_'+str(key)+'.nc')
-
- with open(args['cmems_config'], 'w') as file:
- file.write(filedata)
-
- with Popen(['motuclient', '--size','--config-file', args['cmems_config']], stdout=PIPE, bufsize=1, universal_newlines=True) as p:
- for line in p.stdout:
- line = line.replace("[ INFO]", "")
- logger.info(line)
- if 'Error' in line:
- return 'Error found in CMEMS download report, please check downloaded data'
- if 'Done' in line:
- logger.info('download of request xml file for variable ' + ' '.join(grids[key]) + ' successful')
- if p.returncode != 0:
- return str(p.returncode)
-
- logger.info('checking size of request for variables '+' '.join(grids[key]))
- xml = locs[key]+args['dl_prefix']+'_'+str(date_min)+'_'+str(date_max)+'_'+str(key)+ '.xml'
- try:
- root = ET.parse(xml).getroot()
- except ET.ParseError:
- return 'Parse Error in XML file, This generally occurs when CMEMS service is down and returns an unexpected XML.'
-
- logger.info('size of request ' + root.attrib['size'])
-
- if 'OK' in root.attrib['msg']:
- logger.info('request valid, downloading now......')
-
- with Popen(['motuclient', '--config-file', args['cmems_config']], stdout=PIPE, bufsize=1, universal_newlines=True) as p:
- for line in p.stdout:
- line = line.replace("[ INFO]", "")
- logger.info(line)
- if 'Error' in line:
- return 'Error found in CMEMS download report, please check downloaded data'
- if 'Done' in line:
- logger.info('download of request xml file for variable ' + ' '.join(grids[key]) + ' successful')
- if p.returncode != 0:
- return str(p.returncode)
-
- elif 'too big' in root.attrib['msg']:
- return 1
- else:
- return 'unable to determine if size request is valid (too big or not)'
+
+ file_chk = Path(locs[key] + args['dl_prefix'] + '_' + str(date_min) + '_' + str(date_max) + '_' + str(key) + '.nc')
+
+ if file_chk.is_file() == True:
+ logger.warning('filename of download already exists, please check file is valid, skipping to next item......')
+
+ if file_chk.is_file() == False:
+
+ with open(args['cmems_config'], 'w') as file:
+ file.write(filedata)
+
+ with Popen(['motuclient', '--size','--config-file', args['cmems_config']], stdout=PIPE, bufsize=1, universal_newlines=True) as p:
+ for line in p.stdout:
+ line = line.replace("[ INFO]", "")
+ logger.info(line)
+ if 'Error' in line:
+ return 'Error found in CMEMS download report, please check downloaded data'
+ if 'Done' in line:
+ logger.info('download of request xml file for variable ' + ' '.join(grids[key]) + ' successful')
+ if p.returncode != 0:
+ return str(p.returncode)
+
+ logger.info('checking size of request for variables '+' '.join(grids[key]))
+ xml = locs[key]+args['dl_prefix']+'_'+str(date_min)+'_'+str(date_max)+'_'+str(key)+ '.xml'
+ try:
+ root = ET.parse(xml).getroot()
+ except ET.ParseError:
+ return 'Parse Error in XML file, This generally occurs when CMEMS service is down and returns an unexpected XML.'
+
+ logger.info('size of request ' + root.attrib['size']+'Kb')
+
+ if 'OK' in root.attrib['msg']:
+ logger.info('request valid, downloading now......')
+
+ with Popen(['motuclient', '--config-file', args['cmems_config']], stdout=PIPE, bufsize=1, universal_newlines=True) as p:
+ for line in p.stdout:
+ line = line.replace("[ INFO]", "")
+ logger.info(line)
+ if 'Error' in line:
+ return 'Error found in CMEMS download report, please check downloaded data'
+ if 'Done' in line:
+ logger.info('download of request data file for variable ' + ' '.join(grids[key]) + ' successful')
+ if p.returncode != 0:
+ return str(p.returncode)
+
+ elif 'too big' in root.attrib['msg']:
+ return 1
+ else:
+ return 'unable to determine if size request is valid (too big or not)'
return 0
diff --git a/pynemo/profile.py b/pynemo/profile.py
index 12f69ddc1b2c13fde1e2459bb95df2497985311d..a1722258129dbc4a8c8698fb31083b99a40c3634 100644
--- a/pynemo/profile.py
+++ b/pynemo/profile.py
@@ -74,10 +74,6 @@ class Grid(object):
logger = logging.getLogger(__name__)
logging.basicConfig(filename='nrct.log', level=logging.INFO)
-# define a Handler which writes INFO messages or higher to the sys.stderr
-console = logging.StreamHandler()
-console.setLevel(logging.INFO)
-
def download_cmems(setup_filepath=0):
'''
CMEMS download function.
@@ -92,6 +88,7 @@ def download_cmems(setup_filepath=0):
:param mask_gui:
:return:
'''
+ logger.info('============================================')
logger.info('Start CMEMS download Logging: ' + time.asctime())
logger.info('============================================')
@@ -122,6 +119,7 @@ def download_cmems(setup_filepath=0):
dl_cmems.clean_up(settings)
sys.exit(static)
dl_cmems.clean_up(settings)
+
# subset downloaded static grid files to match downloaded CMEMS data
if settings['subset_static'] == False:
logger.info('CMEMS subset static data not requested')
@@ -277,9 +275,9 @@ def download_cmems(setup_filepath=0):
sys.exit(dy_dl)
# end of messy if statements to split requests into months, weeks and days as needed.
dl_cmems.clean_up(settings)
-
+ logger.info('============================================')
logger.info('End CMEMS download: ' + time.asctime())
- logger.info('==========================================')
+ logger.info('============================================')
def process_bdy(setup_filepath=0, mask_gui=False):
@@ -297,7 +295,7 @@ def process_bdy(setup_filepath=0, mask_gui=False):
"""
# Start Logger
-
+ logger.info('============================================')
logger.info('Start NRCT Logging: '+time.asctime())
logger.info('============================================')