CMEMS download progress now prints to log file

README.rst

@@ -64,6 +64,20 @@ Other commands include -d which downloads the specified CMEMS data in the nameli
 
     $ pynemo -d /PyNEMO/inputs/namelist_cmems.bdy
 
+To use the CMEMS download service an account needs to be created at http://marine.copernicus.eu/services-portfolio/access-to-products/
+Once created the user name and password need to be added to PyNEMO. To do this a file with the name CMEMS_cred.py in the utils folder
+needs to be created with two defined strings one called user and the other called pwd to define the user name and password.::
+
+    $ touch pynemo/utils/CMEMS_cred.py
+    $ vim pynemo/utils/CMEMS_cred.py
+    press i
+    user='username'
+    pwd='password'
+    press esc and then :q
+
+**IMPORTANT** This will create a py file in the right place with the parameters required to download CMEMS, the password is stored as plain text so please
+do not reuse any existing password!
+
 **Additional NOTES**
 
 The above path for Java Home was valid for a Macbook Pro 2015 with macOS Catalina and Java SDK 13.0.2
@@ -71,7 +85,6 @@ however for different java versions, operating systems etc this may be different
 
 The conda environment yaml file has been tested with Miniconda 3.7 and found to install the enironment correctly.
 
-
 Contribution guidelines
 -----------------------
 
@@ -100,7 +113,9 @@ Unit Tests
 To test operation of the PyNEMO module, running the PyTest script in the unit tests folder will perform a range of tests on different child grids,
 e.g. checking the interpolation of the source data on to the child grid. To do this the following command is required::
 
-    $ pytest -q pynemo/unit_test.py
+    $ pytest -q pynemo/pynemo_unit_test.py
+
+The results of the test will show if all tests pass or the errors that result from failed tests.
 
 Currently **(26/03/2020)** there are 7 tests that cover checking the interpolation results of different child grids. The input data is generated as part of the
 test and is removed afterwards. The number of tests will be increased in the future to cover more PyNEMO functionality.

inputs/namelist_cmems.bdy

@@ -52,11 +52,11 @@
 !  CMEMS Data Source Configuration
 !------------------------------------------------------------------------------
    ln_use_cmems             = .true.
-   ln_download_cmems        = .false.
+   ln_download_cmems        = .true.
    sn_cmems_dir             = '/Users/thopri/Projects/PyNEMO/inputs/' ! where to download CMEMS input files (static and variable)
    ln_download_static       = .false.
    ln_subset_static         = .false.
-   nn_num_retry             = 2 ! how many times to retry CMEMS download after non critical errors?
+   nn_num_retry             = 4 ! how many times to retry CMEMS download after non critical errors?
 !------------------------------------------------------------------------------
 !  CMEMS MOTU Configuration (for Boundary Data)
 !------------------------------------------------------------------------------

pynemo/nemo_bdy_dl_cmems.py

@@ -4,7 +4,7 @@ Set of functions to download CMEMS files using FTP (for static mask data) and MO
 
 """
 # import modules
-from subprocess import Popen, PIPE
+from subprocess import Popen, PIPE, CalledProcessError
 import xml.etree.ElementTree as ET
 import logging
 import ftplib
@@ -16,8 +16,8 @@ import os
 #local imports
 from pynemo.utils import cmems_errors as errors
 
-logger = logging.getLogger('CMEMS Downloader')
-
+logger = logging.getLogger(__name__)
+# TODO: Fix double spacing issue on CMEMS download log entries.
 '''
 This function checks to see if the MOTU client is installed on the PyNEMO python environment. If it is not installed
 error code 1 is returned . If it is installed the version number of the installed client is returned as a string
@@ -48,8 +48,8 @@ def get_static(args):
     try:
         from pynemo.utils import CMEMS_cred
     except ImportError:
-        logger.error('Unable to import CMEMS creditials, see Readme for instructions on adding to PyNEMO')
-        return 'Unable to import credintials file'
+        logger.error('Unable to import CMEMS credentials, see Readme for instructions on adding to PyNEMO')
+        return 'Unable to import credential file, have you created one?'
     try:
         ftp = ftplib.FTP(host=args['ftp_server'], user=CMEMS_cred.user, passwd=CMEMS_cred.pwd)
     except ftplib.error_temp:
@@ -247,21 +247,18 @@ def request_cmems(args, date_min, date_max):
         with open(args['cmems_config'], 'w') as file:
             file.write(filedata)
 
-        stdout,stderr = Popen(['motuclient', '--size','--config-file', args['cmems_config']], stdout=PIPE, stderr=PIPE, universal_newlines=True).communicate()
-        stdout = stdout.strip()
-        stderr = stderr.strip()
-        logger.info('checking size of request for variables '+' '.join(grids[key]))
-
-        if 'ERROR' in stdout:
-            idx = stdout.find('ERROR')
-            return stdout[idx-1:-1]
-
-        if 'Done' in stdout:
-            logger.info('download of request xml file for variable ' + ' '.join(grids[key]) + ' successful')
-
-        if len(stderr) > 0:
-            return stderr
+        with Popen(['motuclient', '--size','--config-file', args['cmems_config']], stdout=PIPE, bufsize=1, universal_newlines=True) as p:
+            for line in p.stdout:
+                line = line.replace("[ INFO]", "")
+                logger.info(line)
+                if 'Error' in line:
+                    return 'Error found in CMEMS download report, please check downloaded data'
+                if 'Done' in line:
+                    logger.info('download of request xml file for variable ' + ' '.join(grids[key]) + ' successful')
+        if p.returncode != 0:
+            return str(p.returncode)
 
+        logger.info('checking size of request for variables '+' '.join(grids[key]))
         xml = locs[key]+args['dl_prefix']+'_'+str(date_min)+'_'+str(date_max)+'_'+str(key)+ '.xml'
         try:
             root = ET.parse(xml).getroot()
@@ -272,19 +269,17 @@ def request_cmems(args, date_min, date_max):
 
         if 'OK' in root.attrib['msg']:
             logger.info('request valid, downloading now......')
-            stdout,stderr = Popen(['motuclient', '--config-file', args['cmems_config']], stdout=PIPE, stderr=PIPE, universal_newlines=True).communicate()
-            stdout = stdout.strip()
-            stderr = stderr.strip()
-
-            if 'ERROR' in stdout:
-                idx = stdout.find('ERROR')
-                return stdout[idx:-1]
-
-            if 'Done' in stdout:
-                logger.info('downloading of variables '+' '.join(grids[key])+' successful')
 
-            if len(stderr) > 0:
-                return stderr
+            with Popen(['motuclient', '--config-file', args['cmems_config']], stdout=PIPE, bufsize=1, universal_newlines=True) as p:
+                for line in p.stdout:
+                    line = line.replace("[ INFO]", "")
+                    logger.info(line)
+                    if 'Error' in line:
+                        return 'Error found in CMEMS download report, please check downloaded data'
+                    if 'Done' in line:
+                        logger.info('download of request xml file for variable ' + ' '.join(grids[key]) + ' successful')
+            if p.returncode != 0:
+                return str(p.returncode)
 
         elif 'too big' in root.attrib['msg']:
             return 1

pynemo/nemo_bdy_extr_tm3.py

@@ -847,7 +847,7 @@ class Extract:
                 try:
                     self.d_bdy[v][year]['data'] = intfn(np.arange(time_000, time_end, 86400))
                 except ValueError as e:
-                    logging.error('Value error in time_counter, does time horizon in data and bdy file match?')
+                    self.logger.error('Value error in time_counter, does time horizon in data and bdy file match?')
                     raise ValueError('Value error in time_counter, does time horizon in data and bdy file match?') from e
         else:
             for v in self.var_nam: 
@@ -857,7 +857,7 @@ class Extract:
                     try:
                         self.d_bdy[v].data[t::dstep, :, :] = intfn(np.arange(time_000,time_end, 86400))
                     except ValueError as e:
-                        logging.error('Value error in time_counter, does time horizon in data and bdy file match?')
+                        self.logger.error('Value error in time_counter, does time horizon in data and bdy file match?')
                         raise ValueError('Value error in time_counter, does time horizon in data and bdy file match?') from e
 
         self.time_counter = time_counter

pynemo/pynemo_exe.py

@@ -18,7 +18,7 @@ def main():
     setup_file = ''
     mask_gui = False
     try:
-        opts, dummy_args = getopt.getopt(sys.argv[1:], "hs:dg", ["help", "setup=", "download_cmems=", "mask_gui"])
+        opts, dummy_args = getopt.getopt(sys.argv[1:], "hs:d:g", ["help", "setup=", "download_cmems=", "mask_gui"])
     except getopt.GetoptError:
         print("usage: pynemo -g -s -d <namelist.bdy> ")
         sys.exit(2)