Skip to content

GitLab

Commit 66b16749 authored by thopri's avatar thopri
Browse files

added ability to automatically resize download CMEMS request in monthly, weekly or daily chunks

parent d3a76f69
Hide whitespace changes
Inline Side-by-side
Showing with 93 additions and 19 deletions
inputs/CMEMS.ncml
View file @ 66b16749
......@@ -5,11 +5,6 @@
<ns0:scan location="file://Users/thopri/Projects/PyNEMO/inputs/" regExp=".*T\.nc$" />
</ns0:aggregation>
</ns0:netcdf>
<ns0:netcdf>
<ns0:aggregation dimName="time" name="salinity" type="joinExisting">
<ns0:scan location="file://Users/thopri/Projects/PyNEMO/inputs/" regExp=".*T\.nc$" />
</ns0:aggregation>
</ns0:netcdf>
<ns0:netcdf>
<ns0:aggregation dimName="time" name="zonal_velocity" type="joinExisting">
<ns0:scan location="file://Users/thopri/Projects/PyNEMO/inputs/" regExp=".*U\.nc$" />
......@@ -27,7 +22,6 @@
</ns0:netcdf>
</ns0:aggregation>
<ns0:variable name="thetao" orgName="thetao" />
<ns0:variable name="so" orgName="so" />
<ns0:variable name="uo" orgName="uo" />
<ns0:variable name="vo" orgName="vo" />
<ns0:variable name="zos" orgName="zos" />
......
inputs/namelist_cmems.bdy
View file @ 66b16749
......@@ -65,7 +65,7 @@
!------------------------------------------------------------------------------
! CMEMS Static File Configuration
!------------------------------------------------------------------------------
ln_download_static = .true.
ln_download_static = .false.
ln_subset_static = .true.
sn_ftp_server = 'nrt.cmems-du.eu'
sn_static_dir = '/Core/GLOBAL_ANALYSIS_FORECAST_PHY_001_024/global-analysis-forecast-phy-001-024-statics'
......@@ -114,9 +114,9 @@
! Time information
!------------------------------------------------------------------------------
nn_year_000 = 2017 ! year start
nn_year_end = 2017 ! year end
nn_month_000 = 11 ! month start (default = 1 is years>1)
nn_month_end = 11 ! month end (default = 12 is years>1)
nn_year_end = 2018 ! year end
nn_month_000 = 01 ! month start (default = 1 is years>1)
nn_month_end = 12 ! month end (default = 12 is years>1)
sn_dst_calendar = 'gregorian' ! output calendar format
nn_base_year = 1960 ! base year for time counter
sn_tide_grid = './src_data/tide/grid_tpxo7.2.nc'
......
pynemo/nemo_bdy_dl_cmems.py
View file @ 66b16749
......@@ -9,6 +9,8 @@ import xml.etree.ElementTree as ET
import logging
import ftplib
import re
import pandas as pd
from datetime import datetime
# Need to add try excepts for files and folders being present
......@@ -84,6 +86,58 @@ def subset_static(args):
return stdout
return 0
def MWD_request_cmems(args,date_min,date_max,F):
if F == 'M':
month_start = pd.date_range(date_min, date_max,
freq='MS').strftime("%Y-%m-%d").tolist()
month_end = pd.date_range(date_min, date_max,
freq='M').strftime("%Y-%m-%d").tolist()
for m in range(len(month_end)):
mnth_dl = request_cmems(args, month_start[m], month_end[m])
if mnth_dl == 0:
logger.info('CMEMS month request ' + str((m + 1)) + 'of' + (str(len(month_end) + 1)) + ' successful')
if type(mnth_dl) == str:
logger.error(
'CMEMS month request ' + str((m + 1)) + 'of' + str((len(month_end) + 1)) + ' unsuccessful: Error Msg below')
logger.error(mnth_dl)
return
if mnth_dl == 1:
return 1
if F == 'W':
week_start = pd.date_range(date_min, date_max,
freq='W').strftime("%Y-%m-%d").tolist()
week_end = []
for w in range(len(week_start)):
week_end.append((datetime.strptime(week_start[w], '%Y-%m-%d')
+ datetime.timedelta(days=6)).strftime('%Y-%m-%d'))
for w in range(len(week_end)):
wk_dl = request_cmems(args, week_start[w], week_end[w])
if wk_dl == 0:
logger.info('CMEMS week request ' + str((w + 1)) + 'of' + str((len(week_end) + 1)) + ' successful')
if type(wk_dl) == str:
logger.error(
'CMEMS week request ' + str((m + 1)) + 'of' + str((len(week_end)) + 1) + ' unsuccessful: Error Msg below')
logger.error(mnth_dl)
return
if wk_dl == 1:
return 1
if F == 'D':
days = pd.date_range(date_min, date_max,
freq='D').strftime("%Y-%m-%d").tolist()
for d in range(len(days)):
dy_dl = request_cmems(args, days[d], days[d])
if dy_dl == 0:
logger.info('CMEMS day request ' + str((d + 1)) + 'of' + str((len(week_end) + 1)) + ' successful')
if dy_dl == 1:
logger.error('CMEMS day request still too big, please make domain smaller, or use less variables')
return
if type(dy_dl) == str:
logger.error('CMEMS day request ' + str((d + 1)) + 'of' + (
str(len(days) + 1)) + ' unsuccessful: Error Msg below')
logger.error(dy_dl)
return
return 0
def request_cmems(args, date_min, date_max):
try:
from pynemo.utils import CMEMS_cred
......@@ -135,7 +189,7 @@ def request_cmems(args, date_min, date_max):
filedata = filedata.replace('EI6GB1FHTMCIPOZC', str(args['depth_max']))
filedata = filedata.replace('4Y4LMQLAKP10YFUE', ','.join(grids[key]))
filedata = filedata.replace('QFCN2P56ZQSA7YNK', locs[key])
filedata = filedata.replace('YSLTB459ZW0P84GE', args['dl_prefix']+'_'+str(key)+'.nc')
filedata = filedata.replace('YSLTB459ZW0P84GE', args['dl_prefix']+'_'+str(date_min)+'_'+str(date_max)+'_'+str(key)+'.nc')
with open(args['cmems_config'], 'w') as file:
file.write(filedata)
......@@ -149,9 +203,9 @@ def request_cmems(args, date_min, date_max):
return stdout[idx-1:-1]
if 'Done' in stdout:
logger.info('downloading of variables ' + ' '.join(grids[key]) + ' successful')
logger.info('download of request xml file for variable ' + ' '.join(grids[key]) + ' successful')
xml = locs[key]+args['dl_prefix']+'_'+key+ '.xml'
xml = locs[key]+args['dl_prefix']+'_'+str(date_min)+'_'+str(date_max)+'_'+str(key)+ '.xml'
root = ET.parse(xml).getroot()
logger.info('size of request ' + root.attrib['size'])
......@@ -168,9 +222,7 @@ def request_cmems(args, date_min, date_max):
logger.info('downloading of variables '+' '.join(grids[key])+' successful')
elif 'too big' in root.attrib['msg']:
return 'file request too big reduce size of domain or length of time series'
return 1
else:
return 'unable to determine if size request is valid (too big or not)'
......
pynemo/profile.py
View file @ 66b16749
......@@ -57,6 +57,7 @@ from pynemo.gui.nemo_bdy_mask import Mask as Mask_File
from pynemo import nemo_bdy_dl_cmems as dl_cmems
from calendar import monthrange
import os
class Grid(object):
"""
......@@ -133,8 +134,8 @@ def process_bdy(setup_filepath=0, mask_gui=False):
logger.info('CMEMS Boundary data requested: starting download process....')
if settings['year_end'] - settings['year_000'] > 0:
date_min = settings['year_000']+'-01-01'
date_max = settings['year_end']+'-12-31'
date_min = str(settings['year_000'])+'-01-01'
date_max = str(settings['year_end'])+'-12-31'
elif settings['year_end'] - settings['year_000'] == 0:
......@@ -165,6 +166,32 @@ def process_bdy(setup_filepath=0, mask_gui=False):
if type(dl) == str:
logger.error(dl)
return
if dl == 1:
logger.warning('CMEMS request too large, try monthly downloads...(this may take awhile)')
mnth_dl = dl_cmems.MWD_request_cmems(settings,date_min,date_max,'M')
if mnth_dl == 0:
logger.info('CMEMS monthly request successful')
if type(mnth_dl) == str:
logger.error(mnth_dl)
return
if mnth_dl == 1:
logger.warning('CMEMS request still too large, trying weekly downloads...(this will take longer...)')
wk_dl = dl_cmems.MWD_request_cmems(settings,date_min,date_max,'W')
if wk_dl == 0:
logger.info('CMEMS weekly request successful')
if type(wk_dl) ==str:
logger.error(wk_dl)
return
if wk_dl == 1:
logger.warning('CMESM request STILL too large, trying daily downloads....(even longer.....')
dy_dl = dl_cmems.MWD_request_cmems(settings, date_min,date_max,'D')
if dy_dl == 0:
logger.info('CMEMS daily request successful')
if dy_dl == str:
logger.error(dy_dl)
return
# TODO: implement function to remove unneeded xml files.
#os.remove(file) for file in os.listdir(settings['cmems_dir']) if file.endswith('.xml')
if settings['download_cmems'] == False:
logger.info('no new data from CMEMS requested.......')
......
setup.py
View file @ 66b16749
......@@ -40,7 +40,8 @@ setup(
packages=['pynemo','pynemo.tests','pynemo.gui','pynemo.utils','pynemo.tide','pynemo.reader'],
install_requires=['netCDF4>=1.1.9','scipy','numpy','matplotlib', 'basemap', 'thredds_crawler', 'seawater'],
install_requires=['netCDF4>=1.1.9','scipy','numpy','matplotlib', 'basemap', 'thredds_crawler', 'seawater',
'pandas'],
include_package_data=True,
#The data files that needs to be included in packaging
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment