From 6069981aa8d59c94ae9cacc85cbcc3ea2babdeaa Mon Sep 17 00:00:00 2001
From: thopri <thopri@noc.ac.uk>
Date: Fri, 27 Mar 2020 14:58:17 +0000
Subject: [PATCH] CMEMS download progress now prints to log file
---
README.rst | 19 ++++++-
inputs/namelist_cmems.bdy | 4 +-
pynemo/nemo_bdy_dl_cmems.py | 57 +++++++++-----------
pynemo/nemo_bdy_extr_tm3.py | 4 +-
pynemo/pynemo_exe.py | 2 +-
pynemo/{unit_test.py => pynemo_unit_test.py} | 0
6 files changed, 48 insertions(+), 38 deletions(-)
rename pynemo/{unit_test.py => pynemo_unit_test.py} (100%)
diff --git a/README.rst b/README.rst
index a86e8ab..3e9c474 100644
--- a/README.rst
+++ b/README.rst
@@ -64,6 +64,20 @@ Other commands include -d which downloads the specified CMEMS data in the nameli
$ pynemo -d /PyNEMO/inputs/namelist_cmems.bdy
+To use the CMEMS download service an account needs to be created at http://marine.copernicus.eu/services-portfolio/access-to-products/
+Once created the user name and password need to be added to PyNEMO. To do this a file with the name CMEMS_cred.py in the utils folder
+needs to be created with two defined strings one called user and the other called pwd to define the user name and password.::
+
+ $ touch pynemo/utils/CMEMS_cred.py
+ $ vim pynemo/utils/CMEMS_cred.py
+ press i
+ user='username'
+ pwd='password'
+ press esc and then :q
+
+**IMPORTANT** This will create a py file in the right place with the parameters required to download CMEMS, the password is stored as plain text so please
+do not reuse any existing password!
+
**Additional NOTES**
The above path for Java Home was valid for a Macbook Pro 2015 with macOS Catalina and Java SDK 13.0.2
@@ -71,7 +85,6 @@ however for different java versions, operating systems etc this may be different
The conda environment yaml file has been tested with Miniconda 3.7 and found to install the enironment correctly.
-
Contribution guidelines
-----------------------
@@ -100,7 +113,9 @@ Unit Tests
To test operation of the PyNEMO module, running the PyTest script in the unit tests folder will perform a range of tests on different child grids,
e.g. checking the interpolation of the source data on to the child grid. To do this the following command is required::
- $ pytest -q pynemo/unit_test.py
+ $ pytest -q pynemo/pynemo_unit_test.py
+
+The results of the test will show if all tests pass or the errors that result from failed tests.
Currently **(26/03/2020)** there are 7 tests that cover checking the interpolation results of different child grids. The input data is generated as part of the
test and is removed afterwards. The number of tests will be increased in the future to cover more PyNEMO functionality.
diff --git a/inputs/namelist_cmems.bdy b/inputs/namelist_cmems.bdy
index 20508eb..2883afb 100644
--- a/inputs/namelist_cmems.bdy
+++ b/inputs/namelist_cmems.bdy
@@ -52,11 +52,11 @@
! CMEMS Data Source Configuration
!------------------------------------------------------------------------------
ln_use_cmems = .true.
- ln_download_cmems = .false.
+ ln_download_cmems = .true.
sn_cmems_dir = '/Users/thopri/Projects/PyNEMO/inputs/' ! where to download CMEMS input files (static and variable)
ln_download_static = .false.
ln_subset_static = .false.
- nn_num_retry = 2 ! how many times to retry CMEMS download after non critical errors?
+ nn_num_retry = 4 ! how many times to retry CMEMS download after non critical errors?
!------------------------------------------------------------------------------
! CMEMS MOTU Configuration (for Boundary Data)
!------------------------------------------------------------------------------
diff --git a/pynemo/nemo_bdy_dl_cmems.py b/pynemo/nemo_bdy_dl_cmems.py
index 96be72a..e1642a7 100644
--- a/pynemo/nemo_bdy_dl_cmems.py
+++ b/pynemo/nemo_bdy_dl_cmems.py
@@ -4,7 +4,7 @@ Set of functions to download CMEMS files using FTP (for static mask data) and MO
"""
# import modules
-from subprocess import Popen, PIPE
+from subprocess import Popen, PIPE, CalledProcessError
import xml.etree.ElementTree as ET
import logging
import ftplib
@@ -16,8 +16,8 @@ import os
#local imports
from pynemo.utils import cmems_errors as errors
-logger = logging.getLogger('CMEMS Downloader')
-
+logger = logging.getLogger(__name__)
+# TODO: Fix double spacing issue on CMEMS download log entries.
'''
This function checks to see if the MOTU client is installed on the PyNEMO python environment. If it is not installed
error code 1 is returned . If it is installed the version number of the installed client is returned as a string
@@ -48,8 +48,8 @@ def get_static(args):
try:
from pynemo.utils import CMEMS_cred
except ImportError:
- logger.error('Unable to import CMEMS creditials, see Readme for instructions on adding to PyNEMO')
- return 'Unable to import credintials file'
+ logger.error('Unable to import CMEMS credentials, see Readme for instructions on adding to PyNEMO')
+ return 'Unable to import credential file, have you created one?'
try:
ftp = ftplib.FTP(host=args['ftp_server'], user=CMEMS_cred.user, passwd=CMEMS_cred.pwd)
except ftplib.error_temp:
@@ -247,21 +247,18 @@ def request_cmems(args, date_min, date_max):
with open(args['cmems_config'], 'w') as file:
file.write(filedata)
- stdout,stderr = Popen(['motuclient', '--size','--config-file', args['cmems_config']], stdout=PIPE, stderr=PIPE, universal_newlines=True).communicate()
- stdout = stdout.strip()
- stderr = stderr.strip()
- logger.info('checking size of request for variables '+' '.join(grids[key]))
-
- if 'ERROR' in stdout:
- idx = stdout.find('ERROR')
- return stdout[idx-1:-1]
-
- if 'Done' in stdout:
- logger.info('download of request xml file for variable ' + ' '.join(grids[key]) + ' successful')
-
- if len(stderr) > 0:
- return stderr
+ with Popen(['motuclient', '--size','--config-file', args['cmems_config']], stdout=PIPE, bufsize=1, universal_newlines=True) as p:
+ for line in p.stdout:
+ line = line.replace("[ INFO]", "")
+ logger.info(line)
+ if 'Error' in line:
+ return 'Error found in CMEMS download report, please check downloaded data'
+ if 'Done' in line:
+ logger.info('download of request xml file for variable ' + ' '.join(grids[key]) + ' successful')
+ if p.returncode != 0:
+ return str(p.returncode)
+ logger.info('checking size of request for variables '+' '.join(grids[key]))
xml = locs[key]+args['dl_prefix']+'_'+str(date_min)+'_'+str(date_max)+'_'+str(key)+ '.xml'
try:
root = ET.parse(xml).getroot()
@@ -272,19 +269,17 @@ def request_cmems(args, date_min, date_max):
if 'OK' in root.attrib['msg']:
logger.info('request valid, downloading now......')
- stdout,stderr = Popen(['motuclient', '--config-file', args['cmems_config']], stdout=PIPE, stderr=PIPE, universal_newlines=True).communicate()
- stdout = stdout.strip()
- stderr = stderr.strip()
-
- if 'ERROR' in stdout:
- idx = stdout.find('ERROR')
- return stdout[idx:-1]
-
- if 'Done' in stdout:
- logger.info('downloading of variables '+' '.join(grids[key])+' successful')
- if len(stderr) > 0:
- return stderr
+ with Popen(['motuclient', '--config-file', args['cmems_config']], stdout=PIPE, bufsize=1, universal_newlines=True) as p:
+ for line in p.stdout:
+ line = line.replace("[ INFO]", "")
+ logger.info(line)
+ if 'Error' in line:
+ return 'Error found in CMEMS download report, please check downloaded data'
+ if 'Done' in line:
+ logger.info('download of request xml file for variable ' + ' '.join(grids[key]) + ' successful')
+ if p.returncode != 0:
+ return str(p.returncode)
elif 'too big' in root.attrib['msg']:
return 1
diff --git a/pynemo/nemo_bdy_extr_tm3.py b/pynemo/nemo_bdy_extr_tm3.py
index 9693c19..cbaa0e6 100644
--- a/pynemo/nemo_bdy_extr_tm3.py
+++ b/pynemo/nemo_bdy_extr_tm3.py
@@ -847,7 +847,7 @@ class Extract:
try:
self.d_bdy[v][year]['data'] = intfn(np.arange(time_000, time_end, 86400))
except ValueError as e:
- logging.error('Value error in time_counter, does time horizon in data and bdy file match?')
+ self.logger.error('Value error in time_counter, does time horizon in data and bdy file match?')
raise ValueError('Value error in time_counter, does time horizon in data and bdy file match?') from e
else:
for v in self.var_nam:
@@ -857,7 +857,7 @@ class Extract:
try:
self.d_bdy[v].data[t::dstep, :, :] = intfn(np.arange(time_000,time_end, 86400))
except ValueError as e:
- logging.error('Value error in time_counter, does time horizon in data and bdy file match?')
+ self.logger.error('Value error in time_counter, does time horizon in data and bdy file match?')
raise ValueError('Value error in time_counter, does time horizon in data and bdy file match?') from e
self.time_counter = time_counter
diff --git a/pynemo/pynemo_exe.py b/pynemo/pynemo_exe.py
index 5c9aa96..1f8fb04 100644
--- a/pynemo/pynemo_exe.py
+++ b/pynemo/pynemo_exe.py
@@ -18,7 +18,7 @@ def main():
setup_file = ''
mask_gui = False
try:
- opts, dummy_args = getopt.getopt(sys.argv[1:], "hs:dg", ["help", "setup=", "download_cmems=", "mask_gui"])
+ opts, dummy_args = getopt.getopt(sys.argv[1:], "hs:d:g", ["help", "setup=", "download_cmems=", "mask_gui"])
except getopt.GetoptError:
print("usage: pynemo -g -s -d <namelist.bdy> ")
sys.exit(2)
diff --git a/pynemo/unit_test.py b/pynemo/pynemo_unit_test.py
similarity index 100%
rename from pynemo/unit_test.py
rename to pynemo/pynemo_unit_test.py
--
GitLab