Skip to content

GitLab

Commit 0a2f4c89 authored by Irene Perez Gonzalez's avatar Irene Perez Gonzalez
Browse files

First commit

parents
Show whitespace changes
Inline Side-by-side
Showing with 1080 additions and 0 deletions
schemas/lib/td11/code_tables/weather_past.json 0 → 100644
View file @ 0a2f4c89
{
"0":"",
"1":"",
"2":"",
"3":"",
"4":"",
"5":"",
"6":"",
"7":"",
"8":"",
"9":""
}
schemas/lib/td11/code_tables/weather_present.json 0 → 100644
View file @ 0a2f4c89
{
"00":" ","01":" ","02":" ","03":" ","04":" ","05":" ","06":" ","07":" ","08":" ","09":" ",
"10":" ","11":" ","12":" ","13":" ","14":" ","15":" ","16":" ","17":" ","18":" ","19":" ",
"20":" ","21":" ","22":" ","23":" ","24":" ","25":" ","26":" ","27":" ","28":" ","29":" ",
"30":" ","31":" ","32":" ","33":" ","34":" ","35":" ","36":" ","37":" ","38":" ","39":" ",
"40":" ","41":" ","42":" ","43":" ","44":" ","45":" ","46":" ","47":" ","48":" ","49":" ",
"50":" ","51":" ","52":" ","53":" ","54":" ","55":" ","56":" ","57":" ","58":" ","59":" ",
"60":" ","61":" ","62":" ","63":" ","64":" ","65":" ","66":" ","67":" ","68":" ","69":" ",
"70":" ","71":" ","72":" ","73":" ","74":" ","75":" ","76":" ","77":" ","78":" ","79":" ",
"80":" ","81":" ","82":" ","83":" ","84":" ","85":" ","86":" ","87":" ","88":" ","89":" ",
"90":" ","91":" ","92":" ","93":" ","94":" ","95":" ","96":" ","97":" ","98":" ","99":" "
}
schemas/lib/td11/code_tables/wind_direction.json 0 → 100644
View file @ 0a2f4c89
{
" ":
{"00":" ","01":" ","02":" ","03":" ","04":" ","05":" ","06":" ","07":" ","08":" ","09":" ",
"10":" ","11":" ","12":" ","13":" ","14":" ","15":" ","16":" ","17":" ","18":" ","19":" ",
"20":" ","21":" ","22":" ","23":" ","24":" ","25":" ","26":" ","27":" ","28":" ","29":" ",
"30":" ","31":" ","32":" ","33":" ","34":" ","35":" ","36":" ","99":" "},
"0":
{"00":" ","01":" ","02":" ","03":" ","04":" ","05":" ","06":" ","07":" ","08":" ","09":" ",
"10":" ","11":" ","12":" ","13":" ","14":" ","15":" ","16":" ","17":" ","18":" ","19":" ",
"20":" ","21":" ","22":" ","23":" ","24":" ","25":" ","26":" ","27":" ","28":" ","29":" ",
"30":" ","31":" ","32":" ","99":" "},
"1":
{"00":" ","02":" ","05":" ","07":" ","09":" ",
"11":" ","14":" ","16":" ","18":" ",
"20":" ","23":" ","25":" ","27":" ","29":" ",
"32":" ","34":" ","36":" ","99":" "},
"2":
{"00":" ","02":" ","04":" ","06":" ","08":" ",
"10":" ","12":" ","14":" ","16":" ","18":" ",
"20":" ","22":" ","24":" ","26":" ","28":" ",
"30":" ","32":" ","99":" "}
}
schemas/lib/td11/code_tables/wind_direction.keys 0 → 100644
View file @ 0a2f4c89
{
"('core1','WIND DIR')" : ["('core1','WIND DIR I')","('core1','WIND DIR')"]
}
schemas/lib/td11/code_tables/wind_direction_ind.json 0 → 100644
View file @ 0a2f4c89
{
" ":"36 point scale",
"0":"32 point scale",
"1":"16 of 36 point scale",
"2":"16 of 32 point scale"
}
schemas/lib/td11/code_tables/wind_speed_ind.json 0 → 100644
View file @ 0a2f4c89
{
" ":"Not measured",
"0":"Measured"
}
schemas/lib/td11/td11.json 0 → 100644
View file @ 0a2f4c89
{
"header": {
"format": "fixed_width",
"parsing_order": [
{"s": ["core1"]},
{"e": [ "additional blank","additional 1", "additional 6","additional 8"]},
{"s": [ "core2","supplemental"]}],
"date_parser": {"section": "core1","elements": ["YEAR","MO","DA","HR"],"format": ["%Y","%m","%d","%H"]}
},
"sections": {
"core1": {
"header": {"sentinal": null,"sentinal_length": null,"length": 81},
"elements": {
"CARD DECK": {
"description": "Card deck number",
"field_length": 3,
"column_type": "str"
},
"MAR SQ": {
"description": "Marsden 10 degree square",
"field_length": 3,
"column_type": "int16",
"valid_max": 936,
"valid_min": 1
},
"SUB SQ": {
"description": "Marsden 1 degree sub-square",
"field_length": 2,
"column_type": "int8",
"valid_max": 99,
"valid_min": 0
},
"Q": {
"description": "Quadrant",
"field_length": 1,
"column_type": "key",
"codetable": "coordinates_quadrants"
},
"LAT": {
"description": "Latitude",
"field_length": 3,
"column_type": "float16",
"valid_max": 90.0,
"valid_min": 0.0,
"scale": 0.1,
"decimal_places": 1,
"units": "deg"
},
"LONG": {
"description": "Longitude",
"field_length": 4,
"column_type": "float16",
"valid_max": 180.0,
"valid_min": 0.0,
"scale": 0.1,
"decimal_places": 1,
"units": "deg"
},
"YEAR": {
"description": "Year",
"field_length": 4,
"column_type": "int16",
"valid_max": 1999,
"valid_min": 1800,
"units": "a"
},
"MO": {
"description": "Month",
"field_length": 2,
"column_type": "int8",
"valid_max": 12,
"valid_min": 1,
"units": "mon"
},
"DA": {
"description": "Day",
"field_length": 2,
"column_type": "int8",
"valid_max": 31,
"valid_min": 1,
"units": "d"
},
"HR": {
"description": "Hour GMT",
"field_length": 2,
"column_type": "int8",
"valid_max": 23,
"valid_min": 0,
"units": "h"
},
"WIND DIR I": {
"description": "Wind direction indicator",
"field_length": 1,
"column_type": "key",
"disable_white_strip": true,
"codetable": "wind_direction_ind"
},
"WIND DIR": {
"description": "Wind direction",
"field_length": 2,
"column_type": "key",
"codetable": "wind_direction"
},
"WIND SPD I": {
"description": "Wind speed indicator",
"field_length": 1,
"column_type": "key",
"disable_white_strip": true,
"codetable": "wind_speed_ind"
},
"WIND SPD": {
"description": "Wind speed",
"field_length": 3,
"column_type": "int16",
"valid_min": 0,
"valid_max": 199,
"units": "knots"
},
"VIS I": {
"description": "Visibility indicator",
"field_length": 1,
"column_type": "key",
"disable_white_strip": true,
"codetable": "visibility_ind"
},
"VIS": {
"description": "Visibility",
"field_length": 2,
"column_type": "key",
"codetable": "visibility"
},
"WX": {
"description": "Present weather",
"field_length": 2,
"column_type": "key",
"codetable": "weather_present"
},
"W": {
"description": "Past weather",
"field_length": 1,
"column_type": "key",
"codetable": "weather_past"
},
"PRESS": {
"description": "Sea level pressure",
"field_length": 5,
"column_type": "float16",
"valid_max": 1070.0,
"valid_min": 890.0,
"scale": 0.1,
"decimal_places": 1,
"units": "hPa"
},
"T I": {
"description": "Temperatures indicator",
"field_length": 1,
"column_type": "key",
"codetable": "temperatures_ind"
},
"AIR TMP": {
"description": "Air temperature",
"field_length": 3,
"column_type": "float16",
"encoding": "signed_overpunch",
"valid_max": 99.9,
"valid_min": -99.9,
"scale": 0.1,
"precision": "0.1"
},
"WET BLB": {
"description": "Wet bulb temperature",
"field_length": 3,
"column_type": "float16",
"encoding": "signed_overpunch",
"valid_max": 99.9,
"valid_min": -99.9,
"scale": 0.1,
"decimal_places": 1,
"units": "C"
},
"DEW PT": {
"description": "Dew point temperature",
"field_length": 3,
"column_type": "float16",
"encoding": "signed_overpunch",
"valid_max": 99.9,
"valid_min": -99.9,
"scale": 0.1,
"decimal_places": 1,
"units": "C"
},
"SEA TMP": {
"description": "Sea surface temperature",
"field_length": 3,
"column_type": "float16",
"encoding": "signed_overpunch",
"valid_max": 99.9,
"valid_min": -99.9,
"scale": 0.1,
"decimal_places": 1,
"units": "C"
},
"A-S DIFF": {
"description": "Air-sea temperature difference",
"field_length": 3,
"column_type": "float16",
"encoding": "signed_overpunch",
"valid_max": 99.9,
"valid_min": -99.9,
"scale": 0.1,
"decimal_places": 1,
"units": "C"
},
"CLOUDS N": {
"description": "Total cloud amount",
"field_length": 1,
"column_type": "key",
"codetable": "cloud_amount_oktas"
},
"CLOUDS Nh": {
"description": "Lower cloud amount",
"field_length": 1,
"column_type": "key",
"codetable": "cloud_amount_oktas"
},
"CLOUDS CL": {
"description": "Type of low cloud",
"field_length": 1,
"column_type": "key",
"codetable": "cloud_type_low"
},
"CLOUDS I": {
"description": "Cloud height indicator",
"field_length": 1,
"column_type": "key",
"disable_white_strip": true,
"codetable": "cloud_height_ind"
},
"CLOUDS h": {
"description": "Cloud height",
"field_length": 1,
"column_type": "key",
"codetable": "cloud_height_metres"
},
"CLOUDS CM": {
"description": "Type of middle cloud",
"field_length": 1,
"column_type": "key",
"codetable": "cloud_type_middle"
},
"CLOUDS CH": {
"description": "Type of high cloud",
"field_length": 1,
"column_type": "key",
"codetable": "cloud_type_high"
},
"WAVE DIR": {
"description": "Direction of waves",
"field_length": 2,
"column_type": "key",
"codetable": "wave_direction_from"
},
"P E R WAVES": {
"description": "Period of waves",
"field_length": 1,
"column_type": "key",
"codetable": "wave_period"
},
"WAVE HGT": {
"description": "Height of waves",
"field_length": 2,
"column_type": "key",
"codetable": "wave_height"
},
"SWL DIR": {
"description": "Direction of swell",
"field_length": 2,
"column_type": "key",
"codetable": "wave_direction_from"
},
"P E R SWELL": {
"description": "Period of swell",
"field_length": 1,
"column_type": "key",
"codetable": "swell_period"
},
"SWL HGT": {
"description": "Height of swell",
"field_length": 2,
"column_type": "key",
"codetable": "wave_height"
},
"OSV NO.": {
"description": "Ocean weather station number",
"field_length": 2,
"column_type": "key",
"codetable": "ocean_weather_station"
},
"C D": {
"description": "Card indicator",
"field_length": 1,
"column_type": "key",
"disable_white_strip": true,
"codetable": "card_indicator"
},
"SHP": {
"description": "Osv or ship indicator",
"field_length": 1,
"column_type": "key",
"disable_white_strip": true,
"codetable": "osv_ship_indicator"
}
}
},
"additional blank": {
"header": {"sentinal": " ", "sentinal_length": 1, "length": 7},
"elements": {
"ADD 0": {
"description": "No data",
"field_length": 1,
"column_type": "object",
"ignore": true
},
"Blank": {
"description": "No data",
"field_length": 6,
"column_type": "object",
"ignore": true
}
}
},
"additional 1": {
"header": {"sentinal": "1", "sentinal_length": 1,"length": 7},
"elements": {
"A D D": {
"description": "Additional data indicator",
"field_length": 1,
"column_type": "int8",
"ignore": true
},
"I C E": {
"description": "Type of ice",
"field_length": 1,
"column_type": "key",
"codetable": "ice_type"
},
"ICE THK": {
"description": "Thickness of ice in centimetres",
"field_length": 2,
"column_type": "int8",
"valid_max": 99,
"valid_min": 0,
"units": "cm"
},
"ACC": {
"description": "Rate of ice accretion",
"field_length": 1,
"column_type": "key",
"codetable": "ice_accretion_rate"
},
"Blank": {
"description": "Blank",
"field_length": 2,
"column_type": "object",
"ignore": true
}
}
},
"additional 6": {
"header": {"sentinal": "6","sentinal_length": 1,"length": 7},
"elements": {
"A D D": {
"description": "Additional data indicator",
"field_length": 1,
"column_type": "int8",
"ignore": true
},
"D I R": {
"description": "Ship direction",
"field_length": 1,
"column_type": "key",
"codetable": "ship_direction"
},
"S P D": {
"description": "Ship speed",
"field_length": 1,
"column_type": "key",
"codetable": "ship_speed_knots"
},
"a": {
"description": "Barometric tendency",
"field_length": 1,
"column_type": "key",
"codetable": "barometric_tendency"
},
"ppp": {
"description": "Amount of pressure change (3h, tenths of millibars)",
"field_length": 3,
"column_type": "float16",
"valid_max": 29.9,
"valid_min": 0.0,
"scale": 0.1,
"decimal_places": 1,
"units": "hPa"
}
}
},
"additional 8": {
"header": {"sentinal": "8","sentinal_length": 1,"length": 7},
"elements": {
"A D D": {
"description": "Additional data indicator",
"field_length": 1,
"column_type": "int8",
"ignore": true
},
"SIG N": {
"description": "Significant cloud amount",
"field_length": 1,
"column_type": "key",
"codetable": "cloud_amount_oktas_significant"
},
"SIG T": {
"description": "Significant cloud type",
"field_length": 1,
"column_type": "key",
"codetable": "cloud_type_significant"
},
"SIG HGT": {
"description": "Significant cloud height",
"field_length": 2,
"column_type": "key",
"codetable": "cloud_height_significant"
},
"Blank": {
"description": "Blank",
"field_length": 2,
"column_type": "object",
"ignore": true
}
}
},
"core2": {
"header": {"sentinal": null,"sentinal_length": null,"length": 5},
"elements": {
"I C E": {
"description": "Ice indicator",
"field_length": 1,
"column_type": "str"
},
"SHIP NO.": {
"description": "Ship number",
"field_length": 4,
"column_type": "str"
}
}
},
"supplemental": {
"header": {"sentinal": null,"sentinal_length": null,"length": null},
"elements": {
"supp": {
"description": "Supplemental data fields",
"column_type": "object"
}
}
}
}
}
schemas/lib/td14/td14.json 0 → 100644
View file @ 0a2f4c89
{
"header":
{
"format":"delimited",
"multiple_reports_per_line": true,
"reports_per_line": 6,
"encoding":"ascii"
},
"sections":
{
"header":
{
"header": {"sentinal": null, "sentinal_length": null, "length":null},
"elements":
{
"TAPE DECK": {"description":"Card deck number", "field_length":3,"column_type":"str"},
"STN NUMB": {"description":"Marsden 10 degree square", "field_length":3,"column_type":"int16","valid_max":936,"valid_min":1},
"YR": {"description":"Marsden 1 degree sub-square", "field_length":2,"column_type":"int8","valid_max":99,"valid_min":0},
"MO": {"description":"Quadrant", "field_length":1,"column_type":"int8","codetable":"coordinates_quadrants"},
"DY": {"description":"Latitude", "field_length":3,"column_type":"float16","valid_max":90.0,"valid_min":0.0,"scale":0.1,"precision":"0.1"}
}
},
"report":
{
"header": {"sentinal": null,"sentinal_length":null, "length":null},
"elements":
{
"HR": {"description":"Additional data indicator","field_length":1,"column_type":"int8","ignore":true},
"CEIL": {"description":"Type of ice","field_length":1,"column_type":"str","codetable":"ice_type"},
"VIS": {"description":"Thickness of ice in centimetres","field_length":2,"column_type":"int8","valid_max":99,"valid_min":0},
"WIND DR": {"description":"Rate of ice accretion","field_length":1,"column_type":"str","codetable":"ice_accretion_rate"},
"WIND SPD": {"description":"Blank","field_length":2,"column_type":"object","ignore":true}
}
}
}
}
schemas/lib/templates/code_tables/nested.json 0 → 100644
View file @ 0a2f4c89
{
"":
{
"":,
"":""
},
"":
{
"":,
"":""
}
}
schemas/lib/templates/code_tables/range_keyed_nested.json 0 → 100644
View file @ 0a2f4c89
{
"range_key(ini,end)":
{
"":"",
"":""
},
"range_key(ini,yyyy)":
{
"":"",
"":""
},
"":
{
"":"",
"":""
}
}
schemas/lib/templates/code_tables/range_keyed_simple.json 0 → 100644
View file @ 0a2f4c89
{
"range_key(ini,end)":"",
"range_key(ini,yyyy)":,
"":""
}
schemas/lib/templates/code_tables/simple.json 0 → 100644
View file @ 0a2f4c89
{
"":"",
"":""
}
schemas/lib/templates/schemas/delimited_basic.json 0 → 100644
View file @ 0a2f4c89
{
"header":
{
"format":"delimited",
"delimiter":"",
"date_parser":{"elements":[],"format":[]}
},
"elements":
{
"": {"description":"","field_length":,"column_type":"","missing_value":,"codetable":"","ignore":"","encoding":"","scale":,"offset":,"valid_max":,"valid_min":,"precision":},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","keep_blank":false},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","datetime_format":"%Y%m%d"}
}
}
schemas/lib/templates/schemas/delimited_sections.json 0 → 100644
View file @ 0a2f4c89
{
"header":
{
"format":"delimtied",
"delimiter":"",
"date_parser":{"section":"","elements":[],"format":[]}
},
"sections":
{
"section 1":
{
"elements":
{
"": {"description":"","field_length":,"column_type":"","missing_value":,"codetable":"","ignore":"","encoding":"","scale":,"offset":,"valid_max":,"valid_min":,"precision":},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","keep_blank":false},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","datetime_format":"%Y%m%d"}
}
},
"section N":
{
"elements":
{
"": {"description":"","field_length":,"column_type":"","missing_value":,"codetable":"","ignore":"","encoding":"","scale":,"offset":,"valid_max":,"valid_min":,"precision":},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","keep_blank":false},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","datetime_format":"%Y%m%d"}
}
}
}
}
schemas/lib/templates/schemas/fixed_width_basic.json 0 → 100644
View file @ 0a2f4c89
{
"header":
{
"format":"fixed_width",
"delimiter":null,
"date_parser":{"elements":[],"format":[]}
},
"elements":
{
"": {"description":"","field_length":,"column_type":"","missing_value":,"codetable":"","ignore":"","encoding":"","scale":,"offset":,"valid_max":,"valid_min":,"precision":},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","keep_blank":false},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","datetime_format":"%Y%m%d"}
}
}
schemas/lib/templates/schemas/fixed_width_complex_exc.json 0 → 100644
View file @ 0a2f4c89
{
"header":
{
"format":"fixed_width",
"delimiter":null,
"parsing_order":[{"s":["s1"]},
{"e":["e1","e1","eN"]},
{"s":["s2","s3"]}],
"date_parser":{"section":"","elements":[],"format":[]}
},
"sections":
{
"s1":
{
"header": {"sentinal": null, "sentinal_length": null, "length":},
"elements":
{
"": {"description":"","field_length":,"column_type":"","missing_value":,"codetable":"","ignore":"","encoding":"","scale":,"offset":,"valid_max":,"valid_min":,"precision":},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","keep_blank":false},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","datetime_format":"%Y%m%d"}
}
},
"e1":
{
"header": {"sentinal":, "sentinal_length":, "length":},
"elements":
{
"": {"description":"","field_length":,"column_type":"","missing_value":,"codetable":"","ignore":"","encoding":"","scale":,"offset":,"valid_max":,"valid_min":,"precision":},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","keep_blank":false},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","datetime_format":"%Y%m%d"}
}
},
"e2":
{
"header": {"sentinal":, "sentinal_length":, "length":},
"elements":
{
"": {"description":"","field_length":,"column_type":"","missing_value":,"codetable":"","ignore":"","encoding":"","scale":,"offset":,"valid_max":,"valid_min":,"precision":},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","keep_blank":false},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","datetime_format":"%Y%m%d"}
}
},
"eN":
{
"header": {"sentinal":, "sentinal_length":, "length":},
"elements":
{
"": {"description":"","field_length":,"column_type":"","missing_value":,"codetable":"","ignore":"","encoding":"","scale":,"offset":,"valid_max":,"valid_min":,"precision":},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","keep_blank":false},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","datetime_format":"%Y%m%d"}
}
},
"s2":
{
"header": {"sentinal": null, "sentinal_length": null, "length":},
"elements":
{
"": {"description":"","field_length":,"column_type":"","missing_value":,"codetable":"","ignore":"","encoding":"","scale":,"offset":,"valid_max":,"valid_min":,"precision":},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","keep_blank":false},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","datetime_format":"%Y%m%d"}
}
},
"s3":
{
"header": {"sentinal": null, "sentinal_length": null, "length":null},
"elements":
{
"": {"description":"","field_length":,"column_type":"","missing_value":,"codetable":"","ignore":"","encoding":"","scale":,"offset":,"valid_max":,"valid_min":,"precision":},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","keep_blank":false},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","datetime_format":"%Y%m%d"}
}
}
}
}
schemas/lib/templates/schemas/fixed_width_complex_opt.json 0 → 100644
View file @ 0a2f4c89
{
"header":
{
"format":"fixed_width",
"delimiter":null,
"parsing_order":[{"s":["s1"]},
{"o":["o1","oN"]},
{"s":["s2","s3"]}],
"date_parser":{"section":"","elements":[],"format":[]}
},
"sections":
{
"s1":
{
"header": {"sentinal": null, "sentinal_length": null, "length":},
"elements":
{
"": {"description":"","field_length":,"column_type":"","missing_value":,"codetable":"","ignore":"","encoding":"","scale":,"offset":,"valid_max":,"valid_min":,"precision":},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","keep_blank":false},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","datetime_format":"%Y%m%d"}
}
},
"o1":
{
"header": {"sentinal":, "sentinal_length":, "length":},
"elements":
{
"": {"description":"","field_length":,"column_type":"","missing_value":,"codetable":"","ignore":"","encoding":"","scale":,"offset":,"valid_max":,"valid_min":,"precision":},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","keep_blank":false},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","datetime_format":"%Y%m%d"}
}
},
"oN":
{
"header": {"sentinal":, "sentinal_length":, "length":},
"elements":
{
"": {"description":"","field_length":,"column_type":"","missing_value":,"codetable":"","ignore":"","encoding":"","scale":,"offset":,"valid_max":,"valid_min":,"precision":},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","keep_blank":false},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","datetime_format":"%Y%m%d"}
}
},
"s2":
{
"header": {"sentinal": null, "sentinal_length": null, "length":},
"elements":
{
"": {"description":"","field_length":,"column_type":"","missing_value":,"codetable":"","ignore":"","encoding":"","scale":,"offset":,"valid_max":,"valid_min":,"precision":},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","keep_blank":false},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","datetime_format":"%Y%m%d"}
}
},
"s3":
{
"header": {"sentinal": null, "sentinal_length": null, "length":null},
"elements":
{
"": {"description":"","field_length":,"column_type":"","missing_value":,"codetable":"","ignore":"","encoding":"","scale":,"offset":,"valid_max":,"valid_min":,"precision":},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","keep_blank":false},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","datetime_format":"%Y%m%d"}
}
}
}
}
schemas/lib/templates/schemas/fixed_width_sections.json 0 → 100644
View file @ 0a2f4c89
{
"header":
{
"format":"fixed_width",
"delimiter":null,
"date_parser":{"section":"","elements":[],"format":[]}
},
"sections":
{
"section 1":
{
"header": {"sentinal": null, "sentinal_length": null, "length":},
"elements":
{
"": {"description":"","field_length":,"column_type":"","missing_value":,"codetable":"","ignore":"","encoding":"","scale":,"offset":,"valid_max":,"valid_min":,"precision":},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","keep_blank":false},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","datetime_format":"%Y%m%d"}
}
},
"section N":
{
"header": {"sentinal": null, "sentinal_length": null, "length":null},
"elements":
{
"": {"description":"","field_length":,"column_type":"","missing_value":,"codetable":"","ignore":"","encoding":"","scale":,"offset":,"valid_max":,"valid_min":,"precision":},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","keep_blank":false},
"": {"description":"","field_length":,"column_type":"","missing_value":"","codetable":"","ignore":"","datetime_format":"%Y%m%d"}
}
}
}
}
schemas/lib/wmo_pub47_noc/wmo_pub47_noc.json 0 → 100644
View file @ 0a2f4c89
{
"header": {
"format": "delimited",
"delimiter": ";"
},
"elements": {
"call": {
"description": "Ship call sign (IMMA ID)",
"column_type": "object",
"missing_value": "MSNG"
},
"record": {
"description": "Record number for call",
"column_type": "int8"
},
"name": {
"description": "Name of ship",
"column_type": "object",
"missing_value": "MSNG"
},
"freq": {
"description": "Freq mapped to report duration from CDM (e.g. 8 obs per day mapped to 3 hour duration)",
"column_type": "int8",
"missing_value": "MSNG"
},
"vsslM": {
"description": "Type of meteorological reporting ship",
"column_type": "object",
"missing_value": "MSNG"
},
"vssl": {
"description": "Type of ship mapped to platform_sub_type from CDM",
"column_type": "int8",
"missing_value": "MSNG"
},
"atm": {
"description": "Automation status for ship, mapped to automation from CDM",
"column_type": "int8"
},
"rcnty": {
"description": "Recruiting country for ship",
"column_type": "object"
},
"valid_from": {
"description": "Date which record is valid from",
"column_type": "datetime",
"datetime_format": "%Y-%m-%d"
},
"valid_to": {
"description": "Date which record is valid to",
"column_type": "datetime",
"datetime_format": "%Y-%m-%d"
},
"uid": {
"description": "Internal unique ID for ship",
"column_type": "object"
},
"thmH1": {
"description": "Thermometer height",
"column_type": "float16",
"units": "m"
},
"platH": {
"description": "Platform height",
"column_type": "float16",
"units": "m"
},
"brmH1": {
"description": "Barometer height",
"column_type": "float16",
"units": "m"
},
"anmH": {
"description": "Anemometer height",
"column_type": "float16",
"units": "m"
},
"anHL1": {
"description": "Anemometer height above max load line",
"column_type": "float16",
"units": "m"
},
"wwH": {
"description": "Wind wave observing height",
"column_type": "float16",
"units": "m"
},
"sstD1": {
"description": "Depth of SST sensor",
"column_type": "float16",
"units": "m"
},
"th1": {
"description": "Air temperature sensor",
"column_type": "object"
},
"hy1": {
"description": "Wet bulb and dew point temperature sensor",
"column_type": "object"
},
"st1": {
"description": "Sea surface temperature sensor",
"column_type": "object"
},
"bm1": {
"description": "Sea level pressure sensor",
"column_type": "object"
},
"an1": {
"description": "Wind speed sensor",
"column_type": "object"
}
}
}
schemas/schemas.py 0 → 100644
View file @ 0a2f4c89
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Sep 13 15:14:51 2018
Read data file format json schema to dictionary
"""
from __future__ import print_function
from __future__ import absolute_import
# Import required libraries
import os
import sys
import json
import logging
import shutil
from copy import deepcopy
import glob
from .. import properties
if sys.version_info[0] >= 3:
py3 = True
else:
py3 = False
toolPath = os.path.dirname(os.path.abspath(__file__))
schema_lib = os.path.join(toolPath,'lib')
templates_path = os.path.join(schema_lib,'templates','schemas')
def templates():
schemas = glob.glob(os.path.join(templates_path,'*.json'))
return [ os.path.basename(x).split(".")[0] for x in schemas ]
def copy_template(schema, out_dir = None,out_path = None):
schemas = templates()
if schema in schemas:
schema_path = os.path.join(templates_path,schema + '.json')
schema_out = out_path if out_path else os.path.join(out_dir,schema + '.json')
shutil.copyfile(schema_path, schema_out)
if os.path.isfile( schema_out):
print('Schema template {0} copied to {1}'.format(schema, schema_out))
return
else:
print('copy_template ERROR:')
print('\tError copying schema template {0} copied to {1}'.format(schema, schema_out))
return
else:
print('copy_template ERROR:')
print('\tRequested template {} must be a valid name.'.format(schema))
print('\tValid names are: {}'.format(", ".join(schemas)))
return
def read_schema(schema_name = None, ext_schema_path = None):
if schema_name:
if schema_name not in properties.supported_file_formats:
print('ERROR: \n\tInput schema "{}" not supported. See mdf_reader.properties.supported_file_formats for supported file format schemas'.format(schema_name))
return
else:
schema_path = os.path.join(schema_lib,schema_name)
else:
schema_path = os.path.abspath(ext_schema_path)
schema_name = os.path.basename(schema_path)
schema_file = os.path.join(schema_path, schema_name + '.json')
if not os.path.isfile(schema_file):
logging.error('Can\'t find input schema file {}'.format(schema_file))
return
with open(schema_file) as fileObj:
schema = json.load(fileObj)
# ---------------------------------------------------------------------------
# FILL IN THE INITIAL SCHEMA TO "FULL COMPLEXITY"
# EXPLICITY ADD INFO THAT IS IMPLICIT TO GIVEN SITUATIONS/SUBFORMATS
# ---------------------------------------------------------------------------
# One report per record
if not schema['header'].get('multiple_reports_per_line'):
if not schema.get('sections'):
schema['sections'] = {properties.dummy_level:{'header':{},'elements':schema.get('elements')}}
schema['header']['parsing_order'] = [{'s':[properties.dummy_level]}]
schema.pop('elements',None)
if not schema['header'].get('parsing_order'):# assume sequential
schema['header']['parsing_order'] = [{'s':list(schema['sections'].keys())}]
return schema
else:
# 1X: MULTIPLE REPORTS PER RECORD
# !!!! NEED TO ADD SECTION LENS TO THE REPORT'S SECTION'S HEADER!!!
# CAN INFER FROM ELEMENTS LENGHT AND ADD, OR MAKE AS REQUIREMENT TO BE GIVEN
# global name_report_section
# Have to assess how the section splitting works when x sequential
# sections are declared, and only x-y are met.
if not schema['header'].get('reports_per_line'):
schema['header']['reports_per_line'] = 24
if not schema.get('sections'):
schema['sections'] = dict()
schema['header']['parsing_order'] = [{'s':[]}]
for i in range(1,schema['header']['reports_per_line'] + 1):
schema['sections'].update({str(i):{'header':{},'elements':deepcopy(schema.get('elements'))}})
else:
name_report_section = list(schema['sections'].keys())[-1]
schema['header']['name_report_section'] == name_report_section
schema['header']['parsing_order'] = [{'s':list(schema['sections'].keys())[:-1]}]
for i in range(1,schema['header']['reports_per_line'] + 1):
schema['sections'].update({str(i):schema['sections'].get(name_report_section)})
schema['sections'].pop(name_report_section,None)
for i in range(1,schema['header']['reports_per_line'] + 1):
schema['header']['parsing_order'][0]['s'].append(str(i))
return schema
def df_schema(df_columns, schema, data_model, supp_section = None, supp_schema = None, supp_model = None ):
def clean_schema(columns,schema,data_model):
# Could optionally add cleaning of element descriptors that only apply
# to the initial reading of the data model: field_length, etc....
for element in list(schema):
if element not in columns:
schema.pop(element)
else:
schema[element].update({'data_model': data_model})
return
flat_schema = dict()
# Flatten main model schema
for section in schema.get('sections'):
if section == properties.dummy_level:
flat_schema.update(schema['sections'].get(section).get('elements'))
else:
flat_schema.update( { (section, x): schema['sections'].get(section).get('elements').get(x) for x in schema['sections'].get(section).get('elements') })
clean_schema(df_columns, flat_schema, data_model)
# Here we are assuming supp data has no sections!
if supp_section:
flat_supp = dict()
flat_supp.update( { (supp_section, x): supp_schema['sections'].get(properties.dummy_level).get('elements').get(x) for x in supp_schema['sections'].get(properties.dummy_level).get('elements') })
clean_schema(df_columns, flat_supp, supp_model)
flat_schema.update(flat_supp)
return flat_schema
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment