diff --git a/README.rst b/README.rst
index a86e8ab83cae470fb81b42f3571491f729e9a3c8..3e9c47459d0358c0b90ae90f45c5862e066df891 100644
--- a/README.rst
+++ b/README.rst
@@ -64,6 +64,20 @@ Other commands include -d which downloads the specified CMEMS data in the nameli
$ pynemo -d /PyNEMO/inputs/namelist_cmems.bdy
+To use the CMEMS download service an account needs to be created at http://marine.copernicus.eu/services-portfolio/access-to-products/
+Once created the user name and password need to be added to PyNEMO. To do this a file with the name CMEMS_cred.py in the utils folder
+needs to be created with two defined strings one called user and the other called pwd to define the user name and password.::
+
+ $ touch pynemo/utils/CMEMS_cred.py
+ $ vim pynemo/utils/CMEMS_cred.py
+ press i
+ user='username'
+ pwd='password'
+ press esc and then :q
+
+**IMPORTANT** This will create a py file in the right place with the parameters required to download CMEMS, the password is stored as plain text so please
+do not reuse any existing password!
+
**Additional NOTES**
The above path for Java Home was valid for a Macbook Pro 2015 with macOS Catalina and Java SDK 13.0.2
@@ -71,7 +85,6 @@ however for different java versions, operating systems etc this may be different
The conda environment yaml file has been tested with Miniconda 3.7 and found to install the enironment correctly.
-
Contribution guidelines
-----------------------
@@ -100,7 +113,9 @@ Unit Tests
To test operation of the PyNEMO module, running the PyTest script in the unit tests folder will perform a range of tests on different child grids,
e.g. checking the interpolation of the source data on to the child grid. To do this the following command is required::
- $ pytest -q pynemo/unit_test.py
+ $ pytest -q pynemo/pynemo_unit_test.py
+
+The results of the test will show if all tests pass or the errors that result from failed tests.
Currently **(26/03/2020)** there are 7 tests that cover checking the interpolation results of different child grids. The input data is generated as part of the
test and is removed afterwards. The number of tests will be increased in the future to cover more PyNEMO functionality.
diff --git a/inputs/namelist_cmems.bdy b/inputs/namelist_cmems.bdy
index 20508ebdd773f2431dce73001b13c5bbf7d8a5a4..2883afb3daff2578d81827e84aaeea438b16c236 100644
--- a/inputs/namelist_cmems.bdy
+++ b/inputs/namelist_cmems.bdy
@@ -52,11 +52,11 @@
! CMEMS Data Source Configuration
!------------------------------------------------------------------------------
ln_use_cmems = .true.
- ln_download_cmems = .false.
+ ln_download_cmems = .true.
sn_cmems_dir = '/Users/thopri/Projects/PyNEMO/inputs/' ! where to download CMEMS input files (static and variable)
ln_download_static = .false.
ln_subset_static = .false.
- nn_num_retry = 2 ! how many times to retry CMEMS download after non critical errors?
+ nn_num_retry = 4 ! how many times to retry CMEMS download after non critical errors?
!------------------------------------------------------------------------------
! CMEMS MOTU Configuration (for Boundary Data)
!------------------------------------------------------------------------------
diff --git a/pynemo/nemo_bdy_dl_cmems.py b/pynemo/nemo_bdy_dl_cmems.py
index 96be72a52029f710d3a61aec281ac65e5eafe78a..e1642a74eea491f0d2121444be915ec402dfa579 100644
--- a/pynemo/nemo_bdy_dl_cmems.py
+++ b/pynemo/nemo_bdy_dl_cmems.py
@@ -4,7 +4,7 @@ Set of functions to download CMEMS files using FTP (for static mask data) and MO
"""
# import modules
-from subprocess import Popen, PIPE
+from subprocess import Popen, PIPE, CalledProcessError
import xml.etree.ElementTree as ET
import logging
import ftplib
@@ -16,8 +16,8 @@ import os
#local imports
from pynemo.utils import cmems_errors as errors
-logger = logging.getLogger('CMEMS Downloader')
-
+logger = logging.getLogger(__name__)
+# TODO: Fix double spacing issue on CMEMS download log entries.
'''
This function checks to see if the MOTU client is installed on the PyNEMO python environment. If it is not installed
error code 1 is returned . If it is installed the version number of the installed client is returned as a string
@@ -48,8 +48,8 @@ def get_static(args):
try:
from pynemo.utils import CMEMS_cred
except ImportError:
- logger.error('Unable to import CMEMS creditials, see Readme for instructions on adding to PyNEMO')
- return 'Unable to import credintials file'
+ logger.error('Unable to import CMEMS credentials, see Readme for instructions on adding to PyNEMO')
+ return 'Unable to import credential file, have you created one?'
try:
ftp = ftplib.FTP(host=args['ftp_server'], user=CMEMS_cred.user, passwd=CMEMS_cred.pwd)
except ftplib.error_temp:
@@ -247,21 +247,18 @@ def request_cmems(args, date_min, date_max):
with open(args['cmems_config'], 'w') as file:
file.write(filedata)
- stdout,stderr = Popen(['motuclient', '--size','--config-file', args['cmems_config']], stdout=PIPE, stderr=PIPE, universal_newlines=True).communicate()
- stdout = stdout.strip()
- stderr = stderr.strip()
- logger.info('checking size of request for variables '+' '.join(grids[key]))
-
- if 'ERROR' in stdout:
- idx = stdout.find('ERROR')
- return stdout[idx-1:-1]
-
- if 'Done' in stdout:
- logger.info('download of request xml file for variable ' + ' '.join(grids[key]) + ' successful')
-
- if len(stderr) > 0:
- return stderr
+ with Popen(['motuclient', '--size','--config-file', args['cmems_config']], stdout=PIPE, bufsize=1, universal_newlines=True) as p:
+ for line in p.stdout:
+ line = line.replace("[ INFO]", "")
+ logger.info(line)
+ if 'Error' in line:
+ return 'Error found in CMEMS download report, please check downloaded data'
+ if 'Done' in line:
+ logger.info('download of request xml file for variable ' + ' '.join(grids[key]) + ' successful')
+ if p.returncode != 0:
+ return str(p.returncode)
+ logger.info('checking size of request for variables '+' '.join(grids[key]))
xml = locs[key]+args['dl_prefix']+'_'+str(date_min)+'_'+str(date_max)+'_'+str(key)+ '.xml'
try:
root = ET.parse(xml).getroot()
@@ -272,19 +269,17 @@ def request_cmems(args, date_min, date_max):
if 'OK' in root.attrib['msg']:
logger.info('request valid, downloading now......')
- stdout,stderr = Popen(['motuclient', '--config-file', args['cmems_config']], stdout=PIPE, stderr=PIPE, universal_newlines=True).communicate()
- stdout = stdout.strip()
- stderr = stderr.strip()
-
- if 'ERROR' in stdout:
- idx = stdout.find('ERROR')
- return stdout[idx:-1]
-
- if 'Done' in stdout:
- logger.info('downloading of variables '+' '.join(grids[key])+' successful')
- if len(stderr) > 0:
- return stderr
+ with Popen(['motuclient', '--config-file', args['cmems_config']], stdout=PIPE, bufsize=1, universal_newlines=True) as p:
+ for line in p.stdout:
+ line = line.replace("[ INFO]", "")
+ logger.info(line)
+ if 'Error' in line:
+ return 'Error found in CMEMS download report, please check downloaded data'
+ if 'Done' in line:
+ logger.info('download of request xml file for variable ' + ' '.join(grids[key]) + ' successful')
+ if p.returncode != 0:
+ return str(p.returncode)
elif 'too big' in root.attrib['msg']:
return 1
diff --git a/pynemo/nemo_bdy_extr_tm3.py b/pynemo/nemo_bdy_extr_tm3.py
index 9693c194379637918f9c5310ad433f118fa7d528..cbaa0e64d972eeabcdf99fe5f1aa57e5c4f1cbdf 100644
--- a/pynemo/nemo_bdy_extr_tm3.py
+++ b/pynemo/nemo_bdy_extr_tm3.py
@@ -847,7 +847,7 @@ class Extract:
try:
self.d_bdy[v][year]['data'] = intfn(np.arange(time_000, time_end, 86400))
except ValueError as e:
- logging.error('Value error in time_counter, does time horizon in data and bdy file match?')
+ self.logger.error('Value error in time_counter, does time horizon in data and bdy file match?')
raise ValueError('Value error in time_counter, does time horizon in data and bdy file match?') from e
else:
for v in self.var_nam:
@@ -857,7 +857,7 @@ class Extract:
try:
self.d_bdy[v].data[t::dstep, :, :] = intfn(np.arange(time_000,time_end, 86400))
except ValueError as e:
- logging.error('Value error in time_counter, does time horizon in data and bdy file match?')
+ self.logger.error('Value error in time_counter, does time horizon in data and bdy file match?')
raise ValueError('Value error in time_counter, does time horizon in data and bdy file match?') from e
self.time_counter = time_counter
diff --git a/pynemo/pynemo_exe.py b/pynemo/pynemo_exe.py
index 5c9aa966c3b925c106c6c87a38f77012e4b09c8f..1f8fb043d9e538582a9919469b7536fec1bf2f6d 100644
--- a/pynemo/pynemo_exe.py
+++ b/pynemo/pynemo_exe.py
@@ -18,7 +18,7 @@ def main():
setup_file = ''
mask_gui = False
try:
- opts, dummy_args = getopt.getopt(sys.argv[1:], "hs:dg", ["help", "setup=", "download_cmems=", "mask_gui"])
+ opts, dummy_args = getopt.getopt(sys.argv[1:], "hs:d:g", ["help", "setup=", "download_cmems=", "mask_gui"])
except getopt.GetoptError:
print("usage: pynemo -g -s -d <namelist.bdy> ")
sys.exit(2)
diff --git a/pynemo/unit_test.py b/pynemo/pynemo_unit_test.py
similarity index 100%
rename from pynemo/unit_test.py
rename to pynemo/pynemo_unit_test.py