Skip to content

GitLab

Commit 4044a1b7 authored by Beartriz Recinos Rivas's avatar Beartriz Recinos Rivas
Browse files

formatted gather_stats_c99.py

parent 86078b5d
Hide whitespace changes
Inline Side-by-side
Showing with 13 additions and 15 deletions
tests/gather_stats_c99.py
View file @ 4044a1b7
...@@ -4,29 +4,28 @@ ...@@ -4,29 +4,28 @@
Gather field stats from CLIWOC c99 Gather field stats from CLIWOC c99
""" """
import os import os
import sys
sys.path.append('/home/users/brecinos/c3s_work')
import mdf_reader
import pandas as pd import pandas as pd
import numpy as np import numpy as np
import mdf_reader
import json
import pickle import pickle
from collections import defaultdict from collections import defaultdict
import sys
sys.path.append('/home/users/brecinos/c3s_work')
import mdf_reader
funPath = os.path.dirname(os.path.abspath(__file__)) funPath = os.path.dirname(os.path.abspath(__file__))
#data_path = os.path.join(funPath,'data/133-730/') # data_path = os.path.join(funPath,'data/133-730/')
#print(data_path) # print(data_path)
data_jasmin = '/gws/nopw/j04/glamod_marine/data/datasets/ICOADS_R3.0.0T/level0/133-730' data_jasmin = '/gws/nopw/j04/glamod_marine/data/datasets/ICOADS_R3.0.0T/level0/133-730'
print(data_jasmin) print(data_jasmin)
years = np.arange(1661,1895) years = np.arange(1661, 1895)
print(years) print(years)
output_path = '/home/users/brecinos/c3s_work/133-730/' output_path = '/home/users/brecinos/c3s_work/133-730/'
#i = 1 # i = 1
i = int(sys.argv[1]) i = int(sys.argv[1])
year = years[i] year = years[i]
...@@ -38,13 +37,13 @@ months = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12' ...@@ -38,13 +37,13 @@ months = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12'
paths_files = [] paths_files = []
for m in months: for m in months:
path = os.path.join(data_jasmin, str(year)+'-'+m+'.imma') path = os.path.join(data_jasmin, str(year)+'-'+m+'.imma')
#print(path) # print(path)
if os.path.exists(path): if os.path.exists(path):
paths_files.append(path) paths_files.append(path)
print(paths_files) print(paths_files)
schema_lib = os.path.join(os.path.dirname(funPath),'data_models','lib') schema_lib = os.path.join(os.path.dirname(funPath), 'data_models', 'lib')
print(schema_lib) print(schema_lib)
...@@ -72,7 +71,7 @@ releases = [] ...@@ -72,7 +71,7 @@ releases = []
for path in paths_files: for path in paths_files:
data = mdf_reader.read(path, data_model_path= model_path) data = mdf_reader.read(path, data_model_path=model_path)
names = os.path.split(path)[1][0:7] names = os.path.split(path)[1][0:7]
# Getting elements from voyage section # Getting elements from voyage section
...@@ -127,12 +126,11 @@ d['units_of_other_measurements'] = pd.concat(units_of_other_measurements, axis=1 ...@@ -127,12 +126,11 @@ d['units_of_other_measurements'] = pd.concat(units_of_other_measurements, axis=1
d['humidity_units'] = pd.concat(humidity_units, axis=1).sum(axis=1) d['humidity_units'] = pd.concat(humidity_units, axis=1).sum(axis=1)
d['releases'] = pd.concat(releases, axis=1).sum(axis=1) d['releases'] = pd.concat(releases, axis=1).sum(axis=1)
#print(d) # print(d)
fp = os.path.join(output_path, str(year) + '.pkl') fp = os.path.join(output_path, str(year) + '.pkl')
#print(fp) # print(fp)
with open(fp, 'wb') as f: with open(fp, 'wb') as f:
pickle.dump(d, f, protocol=-1) pickle.dump(d, f, protocol=-1)
print('Done!') print('Done!')
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment