Skip to content

GitLab

"README.rst" did not exist on "429b84b5fd99c86d7e84b557f15b06f2a53d2f9f"
Commit 8d7cd946 authored by iregon's avatar iregon
Browse files

First commit new architecture

parent 611a3444
Hide whitespace changes
Inline Side-by-side
Showing with 653 additions and 1 deletion
schemas/lib/td11.old/code_tables/ship_speed_knots.keys 0 → 100644
View file @ 8d7cd946
{
"('additional 6','S P D')" : ["('core1','YEAR')","('additional 6','S P D')"]
}
schemas/lib/td11.old/code_tables/swell_period.json 0 → 100644
View file @ 8d7cd946
{
"range_key(1750,1967)":
{
"0":"20-21 seconds",
"1":"over 21 seconds",
"2":"5 seconds or less",
"3":"6-7 seconds",
"4":"8-9 seconds",
"5":"10-11 seconds",
"6":"12-13 seconds",
"7":"14-15 seconds",
"8":"16-17 seconds",
"9":"18-19 seconds",
"-":"calm or no period determined"
},
"range_key(1968,yyyy)":
{
"0":"10 seconds",
"1":"11 seconds",
"2":"12 seconds",
"3":"13 seconds",
"4":"14 seconds or more",
"5":"5 seconds or less",
"6":"6 seconds",
"7":"7 seconds",
"8":"8 seconds",
"9":"9 seconds",
"-":"calm or period not determined"
}
}
schemas/lib/td11.old/code_tables/swell_period.keys 0 → 100644
View file @ 8d7cd946
{
"('core1','P E R SWELL')" : ["('core1','YEAR')","('core1','P E R SWELL')"]
}
schemas/lib/td11.old/code_tables/temperatures_ind.json 0 → 100644
View file @ 8d7cd946
{
"1":"",
"3":"",
"5":""
}
schemas/lib/td11.old/code_tables/visibility.json 0 → 100644
View file @ 8d7cd946
{
"0":
{"90":"<0.05 km","91":"0.05 km","92":"0.2 km","93":"0.5 km","94":"1 km","95":"2 km","96":"4 km","97":"10 km","98":"20 km","99":"50 km or more"},
"1":
{"90":"<0.05 km","91":"0.05 km","92":"0.2 km","93":"Fog present, no visibility reported","94":"1 km","95":"2 km","96":"4 km","97":"10 km","98":"20 km","99":"50 km or more"}
}
schemas/lib/td11.old/code_tables/visibility.keys 0 → 100644
View file @ 8d7cd946
{
"('core1','VIS')" : ["('core1','VIS I')","('core1','VIS')"]
}
schemas/lib/td11.old/code_tables/visibility_ind.json 0 → 100644
View file @ 8d7cd946
{
" ":"Not measured",
"0":"Measured",
"1":"Fog present"
}
schemas/lib/td11.old/code_tables/wave_direction_from.json 0 → 100644
View file @ 8d7cd946
{
"00":" ","01":" ","02":" ","03":" ","04":" ","05":" ","06":" ","07":" ","08":" ","09":" ",
"10":" ","11":" ","12":" ","13":" ","14":" ","15":" ","16":" ","17":" ","18":" ","19":" ",
"20":" ","21":" ","22":" ","23":" ","24":" ","25":" ","26":" ","27":" ","28":" ","29":" ",
"30":" ","31":" ","32":" ","33":" ","34":" ","35":" ","36":" ","49":" ","99":" "
}
schemas/lib/td11.old/code_tables/wave_height.json 0 → 100644
View file @ 8d7cd946
{
"00":" ","01":" ","02":" ","03":" ","04":" ","05":" ","06":" ","07":" ","08":" ","09":" ",
"10":" ","11":" ","12":" ","13":" ","14":" ","15":" ","16":" ","17":" ","18":" ","19":" ",
"20":" ","21":" ","22":" ","23":" ","24":" ","25":" ","26":" ","27":" ","28":" ","29":" ",
"30":" ","31":" ","32":" ","33":" ","34":" ","35":" ","36":" ","37":" ","38":" ","39":" ",
"40":" ","41":" ","42":" ","43":" ","44":" ","45":" ","46":" ","47":" ","48":" ","49":" ",
"50":" ","51":" ","52":" ","53":" ","54":" ","55":" ","56":" ","57":" ","58":" ","59":" ",
"60":" ","61":" ","62":" ","63":" ","64":" ","65":" ","66":" ","67":" ","68":" ","69":" ",
"70":" ","71":" ","72":" ","73":" ","74":" ","75":" ","76":" ","77":" ","78":" ","79":" ",
"80":" ","81":" ","82":" ","83":" ","84":" ","85":" ","86":" ","87":" ","88":" ","89":" ",
"90":" ","91":" ","92":" ","93":" ","94":" ","95":" ","96":" ","97":" ","98":" ","99":" "
}
schemas/lib/td11.old/code_tables/wave_period.json 0 → 100644
View file @ 8d7cd946
{
"0":"",
"1":"",
"2":"",
"3":"",
"4":"",
"5":"",
"6":"",
"7":"",
"8":"",
"9":"",
"-":""
}
schemas/lib/td11.old/code_tables/weather_past.json 0 → 100644
View file @ 8d7cd946
{
"0":"",
"1":"",
"2":"",
"3":"",
"4":"",
"5":"",
"6":"",
"7":"",
"8":"",
"9":""
}
schemas/lib/td11.old/code_tables/weather_present.json 0 → 100644
View file @ 8d7cd946
{
"00":" ","01":" ","02":" ","03":" ","04":" ","05":" ","06":" ","07":" ","08":" ","09":" ",
"10":" ","11":" ","12":" ","13":" ","14":" ","15":" ","16":" ","17":" ","18":" ","19":" ",
"20":" ","21":" ","22":" ","23":" ","24":" ","25":" ","26":" ","27":" ","28":" ","29":" ",
"30":" ","31":" ","32":" ","33":" ","34":" ","35":" ","36":" ","37":" ","38":" ","39":" ",
"40":" ","41":" ","42":" ","43":" ","44":" ","45":" ","46":" ","47":" ","48":" ","49":" ",
"50":" ","51":" ","52":" ","53":" ","54":" ","55":" ","56":" ","57":" ","58":" ","59":" ",
"60":" ","61":" ","62":" ","63":" ","64":" ","65":" ","66":" ","67":" ","68":" ","69":" ",
"70":" ","71":" ","72":" ","73":" ","74":" ","75":" ","76":" ","77":" ","78":" ","79":" ",
"80":" ","81":" ","82":" ","83":" ","84":" ","85":" ","86":" ","87":" ","88":" ","89":" ",
"90":" ","91":" ","92":" ","93":" ","94":" ","95":" ","96":" ","97":" ","98":" ","99":" "
}
schemas/lib/td11.old/code_tables/wind_direction.json 0 → 100644
View file @ 8d7cd946
{
" ":
{"00":" ","01":" ","02":" ","03":" ","04":" ","05":" ","06":" ","07":" ","08":" ","09":" ",
"10":" ","11":" ","12":" ","13":" ","14":" ","15":" ","16":" ","17":" ","18":" ","19":" ",
"20":" ","21":" ","22":" ","23":" ","24":" ","25":" ","26":" ","27":" ","28":" ","29":" ",
"30":" ","31":" ","32":" ","33":" ","34":" ","35":" ","36":" ","99":" "},
"0":
{"00":" ","01":" ","02":" ","03":" ","04":" ","05":" ","06":" ","07":" ","08":" ","09":" ",
"10":" ","11":" ","12":" ","13":" ","14":" ","15":" ","16":" ","17":" ","18":" ","19":" ",
"20":" ","21":" ","22":" ","23":" ","24":" ","25":" ","26":" ","27":" ","28":" ","29":" ",
"30":" ","31":" ","32":" ","99":" "},
"1":
{"00":" ","02":" ","05":" ","07":" ","09":" ",
"11":" ","14":" ","16":" ","18":" ",
"20":" ","23":" ","25":" ","27":" ","29":" ",
"32":" ","34":" ","36":" ","99":" "},
"2":
{"00":" ","02":" ","04":" ","06":" ","08":" ",
"10":" ","12":" ","14":" ","16":" ","18":" ",
"20":" ","22":" ","24":" ","26":" ","28":" ",
"30":" ","32":" ","99":" "}
}
schemas/lib/td11.old/code_tables/wind_direction.keys 0 → 100644
View file @ 8d7cd946
{
"('core1','WIND DIR')" : ["('core1','WIND DIR I')","('core1','WIND DIR')"]
}
schemas/lib/td11.old/code_tables/wind_direction_ind.json 0 → 100644
View file @ 8d7cd946
{
" ":"36 point scale",
"0":"32 point scale",
"1":"16 of 36 point scale",
"2":"16 of 32 point scale"
}
schemas/lib/td11.old/code_tables/wind_speed_ind.json 0 → 100644
View file @ 8d7cd946
{
" ":"Not measured",
"0":"Measured"
}
schemas/lib/td11.old/td11.json 0 → 100644
View file @ 8d7cd946
{
"header": {
"format": "fixed_width",
"parsing_order": [
{"s": ["core1"]},
{"e": [ "additional blank","additional 1", "additional 6","additional 8"]},
{"s": [ "core2","supplemental"]}],
"date_parser": {"section": "core1","elements": ["YEAR","MO","DA","HR"],"format": ["%Y","%m","%d","%H"]}
},
"sections": {
"core1": {
"header": {"sentinal": null,"sentinal_length": null,"length": 81},
"elements": {
"CARD DECK": {
"description": "Card deck number",
"field_length": 3,
"column_type": "str"
},
"MAR SQ": {
"description": "Marsden 10 degree square",
"field_length": 3,
"column_type": "int16",
"valid_max": 936,
"valid_min": 1
},
"SUB SQ": {
"description": "Marsden 1 degree sub-square",
"field_length": 2,
"column_type": "int8",
"valid_max": 99,
"valid_min": 0
},
"Q": {
"description": "Quadrant",
"field_length": 1,
"column_type": "key",
"codetable": "coordinates_quadrants"
},
"LAT": {
"description": "Latitude",
"field_length": 3,
"column_type": "float16",
"valid_max": 90.0,
"valid_min": 0.0,
"scale": 0.1,
"decimal_places": 1,
"units": "deg"
},
"LONG": {
"description": "Longitude",
"field_length": 4,
"column_type": "float16",
"valid_max": 180.0,
"valid_min": 0.0,
"scale": 0.1,
"decimal_places": 1,
"units": "deg"
},
"YEAR": {
"description": "Year",
"field_length": 4,
"column_type": "int16",
"valid_max": 1999,
"valid_min": 1800,
"units": "a"
},
"MO": {
"description": "Month",
"field_length": 2,
"column_type": "int8",
"valid_max": 12,
"valid_min": 1,
"units": "mon"
},
"DA": {
"description": "Day",
"field_length": 2,
"column_type": "int8",
"valid_max": 31,
"valid_min": 1,
"units": "d"
},
"HR": {
"description": "Hour GMT",
"field_length": 2,
"column_type": "int8",
"valid_max": 23,
"valid_min": 0,
"units": "h"
},
"WIND DIR I": {
"description": "Wind direction indicator",
"field_length": 1,
"column_type": "key",
"disable_white_strip": true,
"codetable": "wind_direction_ind"
},
"WIND DIR": {
"description": "Wind direction",
"field_length": 2,
"column_type": "key",
"codetable": "wind_direction"
},
"WIND SPD I": {
"description": "Wind speed indicator",
"field_length": 1,
"column_type": "key",
"disable_white_strip": true,
"codetable": "wind_speed_ind"
},
"WIND SPD": {
"description": "Wind speed",
"field_length": 3,
"column_type": "int16",
"valid_min": 0,
"valid_max": 199,
"units": "knots"
},
"VIS I": {
"description": "Visibility indicator",
"field_length": 1,
"column_type": "key",
"disable_white_strip": true,
"codetable": "visibility_ind"
},
"VIS": {
"description": "Visibility",
"field_length": 2,
"column_type": "key",
"codetable": "visibility"
},
"WX": {
"description": "Present weather",
"field_length": 2,
"column_type": "key",
"codetable": "weather_present"
},
"W": {
"description": "Past weather",
"field_length": 1,
"column_type": "key",
"codetable": "weather_past"
},
"PRESS": {
"description": "Sea level pressure",
"field_length": 5,
"column_type": "float16",
"valid_max": 1070.0,
"valid_min": 890.0,
"scale": 0.1,
"decimal_places": 1,
"units": "hPa"
},
"T I": {
"description": "Temperatures indicator",
"field_length": 1,
"column_type": "key",
"codetable": "temperatures_ind"
},
"AIR TMP": {
"description": "Air temperature",
"field_length": 3,
"column_type": "float16",
"encoding": "signed_overpunch",
"valid_max": 99.9,
"valid_min": -99.9,
"scale": 0.1,
"precision": "0.1"
},
"WET BLB": {
"description": "Wet bulb temperature",
"field_length": 3,
"column_type": "float16",
"encoding": "signed_overpunch",
"valid_max": 99.9,
"valid_min": -99.9,
"scale": 0.1,
"decimal_places": 1,
"units": "C"
},
"DEW PT": {
"description": "Dew point temperature",
"field_length": 3,
"column_type": "float16",
"encoding": "signed_overpunch",
"valid_max": 99.9,
"valid_min": -99.9,
"scale": 0.1,
"decimal_places": 1,
"units": "C"
},
"SEA TMP": {
"description": "Sea surface temperature",
"field_length": 3,
"column_type": "float16",
"encoding": "signed_overpunch",
"valid_max": 99.9,
"valid_min": -99.9,
"scale": 0.1,
"decimal_places": 1,
"units": "C"
},
"A-S DIFF": {
"description": "Air-sea temperature difference",
"field_length": 3,
"column_type": "float16",
"encoding": "signed_overpunch",
"valid_max": 99.9,
"valid_min": -99.9,
"scale": 0.1,
"decimal_places": 1,
"units": "C"
},
"CLOUDS N": {
"description": "Total cloud amount",
"field_length": 1,
"column_type": "key",
"codetable": "cloud_amount_oktas"
},
"CLOUDS Nh": {
"description": "Lower cloud amount",
"field_length": 1,
"column_type": "key",
"codetable": "cloud_amount_oktas"
},
"CLOUDS CL": {
"description": "Type of low cloud",
"field_length": 1,
"column_type": "key",
"codetable": "cloud_type_low"
},
"CLOUDS I": {
"description": "Cloud height indicator",
"field_length": 1,
"column_type": "key",
"disable_white_strip": true,
"codetable": "cloud_height_ind"
},
"CLOUDS h": {
"description": "Cloud height",
"field_length": 1,
"column_type": "key",
"codetable": "cloud_height_metres"
},
"CLOUDS CM": {
"description": "Type of middle cloud",
"field_length": 1,
"column_type": "key",
"codetable": "cloud_type_middle"
},
"CLOUDS CH": {
"description": "Type of high cloud",
"field_length": 1,
"column_type": "key",
"codetable": "cloud_type_high"
},
"WAVE DIR": {
"description": "Direction of waves",
"field_length": 2,
"column_type": "key",
"codetable": "wave_direction_from"
},
"P E R WAVES": {
"description": "Period of waves",
"field_length": 1,
"column_type": "key",
"codetable": "wave_period"
},
"WAVE HGT": {
"description": "Height of waves",
"field_length": 2,
"column_type": "key",
"codetable": "wave_height"
},
"SWL DIR": {
"description": "Direction of swell",
"field_length": 2,
"column_type": "key",
"codetable": "wave_direction_from"
},
"P E R SWELL": {
"description": "Period of swell",
"field_length": 1,
"column_type": "key",
"codetable": "swell_period"
},
"SWL HGT": {
"description": "Height of swell",
"field_length": 2,
"column_type": "key",
"codetable": "wave_height"
},
"OSV NO.": {
"description": "Ocean weather station number",
"field_length": 2,
"column_type": "key",
"codetable": "ocean_weather_station"
},
"C D": {
"description": "Card indicator",
"field_length": 1,
"column_type": "key",
"disable_white_strip": true,
"codetable": "card_indicator"
},
"SHP": {
"description": "Osv or ship indicator",
"field_length": 1,
"column_type": "key",
"disable_white_strip": true,
"codetable": "osv_ship_indicator"
}
}
},
"additional blank": {
"header": {"sentinal": " ", "sentinal_length": 1, "length": 7},
"elements": {
"ADD 0": {
"description": "No data",
"field_length": 1,
"column_type": "object",
"ignore": true
},
"Blank": {
"description": "No data",
"field_length": 6,
"column_type": "object",
"ignore": true
}
}
},
"additional 1": {
"header": {"sentinal": "1", "sentinal_length": 1,"length": 7},
"elements": {
"A D D": {
"description": "Additional data indicator",
"field_length": 1,
"column_type": "int8",
"ignore": true
},
"I C E": {
"description": "Type of ice",
"field_length": 1,
"column_type": "key",
"codetable": "ice_type"
},
"ICE THK": {
"description": "Thickness of ice in centimetres",
"field_length": 2,
"column_type": "int8",
"valid_max": 99,
"valid_min": 0,
"units": "cm"
},
"ACC": {
"description": "Rate of ice accretion",
"field_length": 1,
"column_type": "key",
"codetable": "ice_accretion_rate"
},
"Blank": {
"description": "Blank",
"field_length": 2,
"column_type": "object",
"ignore": true
}
}
},
"additional 6": {
"header": {"sentinal": "6","sentinal_length": 1,"length": 7},
"elements": {
"A D D": {
"description": "Additional data indicator",
"field_length": 1,
"column_type": "int8",
"ignore": true
},
"D I R": {
"description": "Ship direction",
"field_length": 1,
"column_type": "key",
"codetable": "ship_direction"
},
"S P D": {
"description": "Ship speed",
"field_length": 1,
"column_type": "key",
"codetable": "ship_speed_knots"
},
"a": {
"description": "Barometric tendency",
"field_length": 1,
"column_type": "key",
"codetable": "barometric_tendency"
},
"ppp": {
"description": "Amount of pressure change (3h, tenths of millibars)",
"field_length": 3,
"column_type": "float16",
"valid_max": 29.9,
"valid_min": 0.0,
"scale": 0.1,
"decimal_places": 1,
"units": "hPa"
}
}
},
"additional 8": {
"header": {"sentinal": "8","sentinal_length": 1,"length": 7},
"elements": {
"A D D": {
"description": "Additional data indicator",
"field_length": 1,
"column_type": "int8",
"ignore": true
},
"SIG N": {
"description": "Significant cloud amount",
"field_length": 1,
"column_type": "key",
"codetable": "cloud_amount_oktas_significant"
},
"SIG T": {
"description": "Significant cloud type",
"field_length": 1,
"column_type": "key",
"codetable": "cloud_type_significant"
},
"SIG HGT": {
"description": "Significant cloud height",
"field_length": 2,
"column_type": "key",
"codetable": "cloud_height_significant"
},
"Blank": {
"description": "Blank",
"field_length": 2,
"column_type": "object",
"ignore": true
}
}
},
"core2": {
"header": {"sentinal": null,"sentinal_length": null,"length": 5},
"elements": {
"I C E": {
"description": "Ice indicator",
"field_length": 1,
"column_type": "str"
},
"SHIP NO.": {
"description": "Ship number",
"field_length": 4,
"column_type": "str"
}
}
},
"supplemental": {
"header": {"sentinal": null,"sentinal_length": null,"length": null},
"elements": {
"supp": {
"description": "Supplemental data fields",
"column_type": "object"
}
}
}
}
}
schemas/schemas.py
View file @ 8d7cd946
......@@ -5,6 +5,11 @@ Created on Thu Sep 13 15:14:51 2018
Read data file format json schema to dictionary
Add schema validation:
- check mandatory are not null
- check fixed options
"""
......@@ -54,6 +59,22 @@ def copy_template(schema, out_dir = None,out_path = None):
print('\tValid names are: {}'.format(", ".join(schemas)))
return
def get_field_layout(field_layout_def,field_layout):
if not field_layout_def and not field_layout:
return None
elif not field_layout:
return field_layout_def
else:
return field_layout
def get_delimiter(delimiter_def,delimiter):
if not delimiter_def and not delimiter:
return None
elif not delimiter_def:
return delimiter
else:
return field_layout
def read_schema(schema_name = None, ext_schema_path = None):
if schema_name:
......@@ -65,6 +86,7 @@ def read_schema(schema_name = None, ext_schema_path = None):
else:
schema_path = os.path.abspath(ext_schema_path)
schema_name = os.path.basename(schema_path)
schema_file = os.path.join(schema_path, schema_name + '.json')
if not os.path.isfile(schema_file):
logging.error('Can\'t find input schema file {}'.format(schema_file))
......@@ -75,14 +97,31 @@ def read_schema(schema_name = None, ext_schema_path = None):
# FILL IN THE INITIAL SCHEMA TO "FULL COMPLEXITY"
# EXPLICITY ADD INFO THAT IS IMPLICIT TO GIVEN SITUATIONS/SUBFORMATS
# ---------------------------------------------------------------------------
# One report per record
# One report per record: make sure later changes are reflected in MULTIPLE
# REPORTS PER RECORD case below if we ever use it!
if not schema['header'].get('multiple_reports_per_line'):
# Make no section formats be 1 section format
if not schema.get('sections'):
schema['sections'] = {properties.dummy_level:{'header':{},'elements':schema.get('elements')}}
schema['header']['parsing_order'] = [{'s':[properties.dummy_level]}]
schema.pop('elements',None)
schema['sections'][properties.dummy_level]['header']['delimiter'] = schema['header'].get('delimiter')
schema['header'].pop('delimiter',None)
schema['sections'][properties.dummy_level]['header']['field_layout'] = schema['header'].get('field_layout')
schema['header'].pop('field_layout',None)
# Make parsing order explicit
if not schema['header'].get('parsing_order'):# assume sequential
schema['header']['parsing_order'] = [{'s':list(schema['sections'].keys())}]
# Make disable_read and field_layout explicit: this is ruled by delimiter or length being set,
# unless explicitly set
for section in schema['sections'].keys():
if schema['sections'][section]['header'].get('disable_read'):
continue
else:
schema['sections'][section]['header']['disable_read'] = False
if not schema['sections'][section]['header'].get('field_layout'):
delimiter = schema['sections'][section]['header'].get('delimiter')
schema['sections'][section]['header']['field_layout'] = 'delimited' if delimiter else 'fixed_width'
return schema
else:
# 1X: MULTIPLE REPORTS PER RECORD
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment