#Copyright (c) 2013,Vienna University of Technology, Department of Geodesy and Geoinformation
#All rights reserved.
#Redistribution and use in source and binary forms, with or without
#modification, are permitted provided that the following conditions are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# * Neither the name of the Vienna University of Technology, Department of Geodesy and Geoinformation nor the
# names of its contributors may be used to endorse or promote products
# derived from this software without specific prior written permission.
#THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
#ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
#WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
#DISCLAIMED. IN NO EVENT SHALL VIENNA UNIVERSITY OF TECHNOLOGY,
#DEPARTMENT OF GEODESY AND GEOINFORMATION BE LIABLE FOR ANY
#DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
#(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
#LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
#ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
'''
Created on Aug 5, 2013
@author: Christoph Paulik Christoph.Paulik@geo.tuwien.ac.at
'''
import metadata_collector
import readers
import pytesmo.grid.nearest_neighbor as NN
import os
import numpy as np
[docs]class ISMNError(Exception):
pass
[docs]class ISMN_station(object):
"""
Knows everything about the station, like which variables are measured there in which depths
and in which files the data is stored. This is not completely true for the CEOP format
since depth_from and depth_to are not easily knowable without parsing the whole file.
For CEOP format depth_from and depth_to will only contain the phrase 'multiple' instead
of the actual depth
Attributes:
-----------
network : string
network the time series belongs to
station : string
station name the time series belongs to
latitude : float
latitude of station
longitude : float
longitude of station
elevation : float
elevation of station
variables : numpy.array
variables measured at this station
depth_from : numpy.array
shallower depth of layer the variable with same index was measured at
depth_to : numpy.array
deeper depth of layer the variable with same index was measured at
sensors : numpy.array
sensor names of variables
filenames : numpy.array
filenames in which the data is stored
Methods:
--------
get_variables()
returns the variables measured at this station
get_depths(variable)
get the depths in which a variable was measured at this station
get_sensors(variable,depth_from,depth_to)
get the sensors for the given variable, depth combination
read_variable(variable,depth_from=None,depth_to=None,sensor=None)
read the data for the given parameter combination
"""
def __init__(self,metadata):
"""
Goes through the passed metadata array and fills the attributes accordingly.
Parameters:
-----------
metadata : numpy.array
part or the structured array from metadata_collector.collect_from_folder()
which contains only fields for one station
"""
self.network = None
self.station = None
self.latitude = None
self.longitude = None
self.elevation = None
self.variables = []
self.depth_from = []
self.depth_to = []
self.sensors = []
self.filenames = []
for dataset in metadata:
if self.network is None:
self.network = dataset['network']
elif self.network != dataset['network']:
raise ISMNError("Networks in Station metadata are not the same")
if self.station is None:
self.station = dataset['station']
elif self.station != dataset['station']:
raise ISMNError("Station names in Station metadata are not the same")
self.latitude = dataset['latitude']
self.longitude = dataset['longitude']
self.elevation = dataset['elevation']
self.variables.append(dataset['variable'])
self.depth_from.append(dataset['depth_from'])
self.depth_to.append(dataset['depth_to'])
self.sensors.append(dataset['sensor'])
self.filenames.append(dataset['filename'])
self.variables = np.array(self.variables)
self.depth_from = np.array(self.depth_from)
self.depth_to = np.array(self.depth_to)
self.sensors = np.array(self.sensors)
self.filenames = np.array(self.filenames)
[docs] def get_variables(self):
"""
get a list of variables measured at this station
Returns:
--------
variables : numpy.array
array of variables measured at this station
"""
return np.unique(self.variables)
[docs] def get_depths(self,variable):
"""
get depths at which the given variable was measured at this station
Parameters:
-----------
variable : string
variable string best one of those returned by get_variables()
Returns:
--------
depth_from : numpy.array
depth_to : numpy.array
"""
if variable in self.variables:
index = np.where(variable == self.variables)
return self.depth_from[index],self.depth_to[index]
else:
return None,None
[docs] def get_sensors(self,variable,depth_from,depth_to):
"""
get the sensors at which the variable was measured at the
given depth
Parameters:
-----------
variable : string
variable abbreviation
depth_from : float
shallower depth of layer the variable was measured at
depth_to : float
deeper depth of layer the variable was measured at
Returns:
--------
sensors : numpy.array
array of sensors found for the given combination of variable and depths
Raises:
-------
ISMNError
if no sensor was found for the given combination of variable and depths
"""
ind_sensors = np.where((variable == self.variables) &
(depth_from == self.depth_from) &
(depth_to == self.depth_to))[0]
if ind_sensors.size == 0:
raise ISMNError("variable-depth_from-depth_to combination does not exist."
"Please check which depths do exist with get_depths_for_variable")
else:
return self.sensors[ind_sensors]
[docs] def read_variable(self,variable,depth_from=None,depth_to=None,sensor=None):
"""
actually reads the given variable from the file. Parameters are
required until any ambiguity is resolved. If there is only one depth for
the given variable then only variable is required. If there are multiple
depths at least depth_from is required. If there are multiple depth_to
possibilities for one variable-depth_from combination also depth_to has to
be specified. If 2 sensors are measuring the same variable in the same
depth then also the sensor has to be specified.
Parameters:
-----------
variable: string
variable to read
depth_from : float, optional
shallower depth of layer the variable was measured at
depth_to : float, optional
deeper depth of layer the variable was measured at
sensor : string, optional
name of the sensor
Returns:
--------
data : readers.ISMNTimeSeries
ISMNTimeSeries object containing the relevant metadata for the time series
as well as a .data pointing to a pandas.DataFrame
Raises:
-------
ISMNError:
if not all ambiguity was resolved by the given input parameters or
if no data was found for the given input parameters
"""
if depth_from is None:
depth_f,depth_t = self.get_depths_for_variable(variable)
if depth_f.size > 1:
raise ISMNError("there are multiple depths for this variable"
"Please specify the one you want to read")
elif depth_f.size == 1:
depth_from = depth_f[0]
elif depth_f.size == 0:
raise ISMNError("there are no depths for this variable"
"Something went wrong")
if depth_to is None:
depth_f,depth_t = self.get_depths_for_variable(variable)
if depth_t.size > 1:
raise ISMNError("there are multiple depths with the same depth_from value"
"Please specify the depth_to value you want to read")
elif depth_t.size == 1:
depth_to = depth_t[0]
elif depth_t.size == 0:
raise ISMNError("there are no depths for this variable"
"Something went wrong")
if sensor is None:
sensors = self.get_sensors_for_variable_depth(variable, depth_from, depth_to)
if sensors.size > 1:
raise ISMNError("there are multiple sensors for this combination of "
"variable, depth_to, depth_from. Please specify which one "
"you want to read")
elif sensors.size == 1:
sensor = sensors[0]
elif sensors.size == 0:
raise ISMNError("there are no sensors for this variable, depth_from, depth_to "
"combination. Please make sure you specified valid depths")
index_filename = np.where((variable == self.variables) &
(depth_from == self.depth_from) &
(depth_to == self.depth_to) &
(sensor == self.sensors))[0]
if index_filename.size != 1:
raise ISMNError("There is no data for this combination of variable, depth_from, "
"depth_to and sensor. Please check.")
else:
return readers.read_data(self.filenames[index_filename[0]])
[docs]class ISMN_Interface(object):
"""
class provides interface to ISMN data downloaded from the ISMN website
Attributes:
-----------
metadata : numpy.array
metadata array for all stations contained in the path given during initialization
kdTree : general.grid.nearest_neighbor.findGeoNN
kdTree for finding nearest insitu station for given lon lat
Methods:
--------
find_nearest_station(lon,lat)
find nearest station for given coordinates
"""
def __init__(self,path_to_data):
"""
collects metadata from all files in path_to_data and saves metadata information
in numpy file in folder path_to_data/python_metadata/
First initialization can take a minute or so if all ISMN
data is present in path_to_data
Parameters:
-----------
path_to_data : string
filepath to unzipped ISMN data containing the Network folders
"""
if not os.path.exists(os.path.join(path_to_data,'python_metadata','metadata.npy')):
self.metadata = metadata_collector.collect_from_folder(path_to_data)
os.mkdir(os.path.join(path_to_data,'python_metadata'))
np.save(os.path.join(path_to_data,'python_metadata','metadata.npy'), self.metadata)
#np.savetxt(os.path.join(path_to_data,'python_metadata','metadata.npy'), self.metadata,delimiter=',')
else:
self.metadata = np.load(os.path.join(path_to_data,'python_metadata','metadata.npy'))
self.kdTree = None
[docs] def find_nearest_station(self,lon,lat,return_distance=False):
"""
finds the nearest station available in downloaded data
Parameters:
-----------
lon : float
Longitude of point
lat : float
Latitude of point
return_distance : boolean, optional
if True also distance is returned
Returns:
--------
station : ISMN_station
ISMN_station object
distance : float, optional
distance to station in meters, measured in cartesian coordinates and not on
a great circle. Should be OK for small distances
"""
if self.kdTree is None:
self.kdTree = NN.findGeoNN(self.metadata['longitude'],self.metadata['latitude'])
d,index = self.kdTree.find_nearest_index(lon, lat)
all_index = np.where(self.metadata['station'] == self.metadata['station'][index])
if return_distance:
return ISMN_station(self.metadata[all_index]),d
else:
return ISMN_station(self.metadata[all_index])
[docs] def get_all_data_for_variable_depth_min_max(self,variable,min_depth,max_depth):
"""
function to go through all downloaded data and read the data
for the given variable if the depth it was measured at lies completely
between min_depth and max_depth
Parameters:
-----------
variable: string
variable to read
min_depth : float
depth_from of variable has to be >= min_depth in order to be
included.
max_depth : float
depth_to of variable has to be <= max_depth in order to be
included.
Yields:
-------
time_series : readers.ISMNTimeSeries
ISMNTimeSeries object containing data and metadata
"""
for time_series in self.metadata:
if time_series['variable'] != variable: continue
if ((time_series['depth_from'] >= min_depth) &
(time_series['depth_to'] <= max_depth)):
yield readers.read_data(time_series['filename'])
import matplotlib.pyplot as plt
#===============================================================================
# start = datetime.now()
#
rootdir = os.path.join(os.sep+'media','sf_D','small_projects','cpa_2013_07_ISMN_userformat_reader','ISMN_parser_test')
MyISMN = ISMN_Interface(rootdir)
#
# print datetime.now() - start
#
# lat,lon = 13.548,2.68
#
# Station,distance = MyISMN.find_nearest_station(lon, lat)
#
# print Station.latitude,Station.longitude
#
# print distance
# print Station.get_depths('sm')
# print Station.get_sensors('sm', 0.05, 0.05)
# data = Station.read_variable('sm',0.05,0.05,'CS616_1')
#===============================================================================
for time_series in MyISMN.get_all_data_for_variable_depth_min_max('sm', 0.5, 2):
print (time_series.network,time_series.station,
time_series.sensor)
time_series.plot()
plt.show()