Source code for pytesmo.io.ismn.interface

#Copyright (c) 2013,Vienna University of Technology, Department of Geodesy and Geoinformation
#All rights reserved.

#Redistribution and use in source and binary forms, with or without
#modification, are permitted provided that the following conditions are met:
#   * Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright
#      notice, this list of conditions and the following disclaimer in the
#      documentation and/or other materials provided with the distribution.
#    * Neither the name of the Vienna University of Technology, Department of Geodesy and Geoinformation nor the
#      names of its contributors may be used to endorse or promote products
#      derived from this software without specific prior written permission.

#THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
#ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
#WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
#DISCLAIMED. IN NO EVENT SHALL VIENNA UNIVERSITY OF TECHNOLOGY, 
#DEPARTMENT OF GEODESY AND GEOINFORMATION BE LIABLE FOR ANY
#DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
#(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
#LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
#ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

'''
Created on Aug 5, 2013

@author: Christoph Paulik Christoph.Paulik@geo.tuwien.ac.at
'''

import metadata_collector
import readers
import pytesmo.grid.nearest_neighbor as NN

import os
import numpy as np

[docs]class ISMNError(Exception): pass
[docs]class ISMN_station(object): """ Knows everything about the station, like which variables are measured there in which depths and in which files the data is stored. This is not completely true for the CEOP format since depth_from and depth_to are not easily knowable without parsing the whole file. For CEOP format depth_from and depth_to will only contain the phrase 'multiple' instead of the actual depth Attributes: ----------- network : string network the time series belongs to station : string station name the time series belongs to latitude : float latitude of station longitude : float longitude of station elevation : float elevation of station variables : numpy.array variables measured at this station depth_from : numpy.array shallower depth of layer the variable with same index was measured at depth_to : numpy.array deeper depth of layer the variable with same index was measured at sensors : numpy.array sensor names of variables filenames : numpy.array filenames in which the data is stored Methods: -------- get_variables() returns the variables measured at this station get_depths(variable) get the depths in which a variable was measured at this station get_sensors(variable,depth_from,depth_to) get the sensors for the given variable, depth combination read_variable(variable,depth_from=None,depth_to=None,sensor=None) read the data for the given parameter combination """ def __init__(self,metadata): """ Goes through the passed metadata array and fills the attributes accordingly. Parameters: ----------- metadata : numpy.array part or the structured array from metadata_collector.collect_from_folder() which contains only fields for one station """ self.network = None self.station = None self.latitude = None self.longitude = None self.elevation = None self.variables = [] self.depth_from = [] self.depth_to = [] self.sensors = [] self.filenames = [] for dataset in metadata: if self.network is None: self.network = dataset['network'] elif self.network != dataset['network']: raise ISMNError("Networks in Station metadata are not the same") if self.station is None: self.station = dataset['station'] elif self.station != dataset['station']: raise ISMNError("Station names in Station metadata are not the same") self.latitude = dataset['latitude'] self.longitude = dataset['longitude'] self.elevation = dataset['elevation'] self.variables.append(dataset['variable']) self.depth_from.append(dataset['depth_from']) self.depth_to.append(dataset['depth_to']) self.sensors.append(dataset['sensor']) self.filenames.append(dataset['filename']) self.variables = np.array(self.variables) self.depth_from = np.array(self.depth_from) self.depth_to = np.array(self.depth_to) self.sensors = np.array(self.sensors) self.filenames = np.array(self.filenames)
[docs] def get_variables(self): """ get a list of variables measured at this station Returns: -------- variables : numpy.array array of variables measured at this station """ return np.unique(self.variables)
[docs] def get_depths(self,variable): """ get depths at which the given variable was measured at this station Parameters: ----------- variable : string variable string best one of those returned by get_variables() Returns: -------- depth_from : numpy.array depth_to : numpy.array """ if variable in self.variables: index = np.where(variable == self.variables) return self.depth_from[index],self.depth_to[index] else: return None,None
[docs] def get_sensors(self,variable,depth_from,depth_to): """ get the sensors at which the variable was measured at the given depth Parameters: ----------- variable : string variable abbreviation depth_from : float shallower depth of layer the variable was measured at depth_to : float deeper depth of layer the variable was measured at Returns: -------- sensors : numpy.array array of sensors found for the given combination of variable and depths Raises: ------- ISMNError if no sensor was found for the given combination of variable and depths """ ind_sensors = np.where((variable == self.variables) & (depth_from == self.depth_from) & (depth_to == self.depth_to))[0] if ind_sensors.size == 0: raise ISMNError("variable-depth_from-depth_to combination does not exist." "Please check which depths do exist with get_depths_for_variable") else: return self.sensors[ind_sensors]
[docs] def read_variable(self,variable,depth_from=None,depth_to=None,sensor=None): """ actually reads the given variable from the file. Parameters are required until any ambiguity is resolved. If there is only one depth for the given variable then only variable is required. If there are multiple depths at least depth_from is required. If there are multiple depth_to possibilities for one variable-depth_from combination also depth_to has to be specified. If 2 sensors are measuring the same variable in the same depth then also the sensor has to be specified. Parameters: ----------- variable: string variable to read depth_from : float, optional shallower depth of layer the variable was measured at depth_to : float, optional deeper depth of layer the variable was measured at sensor : string, optional name of the sensor Returns: -------- data : readers.ISMNTimeSeries ISMNTimeSeries object containing the relevant metadata for the time series as well as a .data pointing to a pandas.DataFrame Raises: ------- ISMNError: if not all ambiguity was resolved by the given input parameters or if no data was found for the given input parameters """ if depth_from is None: depth_f,depth_t = self.get_depths_for_variable(variable) if depth_f.size > 1: raise ISMNError("there are multiple depths for this variable" "Please specify the one you want to read") elif depth_f.size == 1: depth_from = depth_f[0] elif depth_f.size == 0: raise ISMNError("there are no depths for this variable" "Something went wrong") if depth_to is None: depth_f,depth_t = self.get_depths_for_variable(variable) if depth_t.size > 1: raise ISMNError("there are multiple depths with the same depth_from value" "Please specify the depth_to value you want to read") elif depth_t.size == 1: depth_to = depth_t[0] elif depth_t.size == 0: raise ISMNError("there are no depths for this variable" "Something went wrong") if sensor is None: sensors = self.get_sensors_for_variable_depth(variable, depth_from, depth_to) if sensors.size > 1: raise ISMNError("there are multiple sensors for this combination of " "variable, depth_to, depth_from. Please specify which one " "you want to read") elif sensors.size == 1: sensor = sensors[0] elif sensors.size == 0: raise ISMNError("there are no sensors for this variable, depth_from, depth_to " "combination. Please make sure you specified valid depths") index_filename = np.where((variable == self.variables) & (depth_from == self.depth_from) & (depth_to == self.depth_to) & (sensor == self.sensors))[0] if index_filename.size != 1: raise ISMNError("There is no data for this combination of variable, depth_from, " "depth_to and sensor. Please check.") else: return readers.read_data(self.filenames[index_filename[0]])
[docs]class ISMN_Interface(object): """ class provides interface to ISMN data downloaded from the ISMN website Attributes: ----------- metadata : numpy.array metadata array for all stations contained in the path given during initialization kdTree : general.grid.nearest_neighbor.findGeoNN kdTree for finding nearest insitu station for given lon lat Methods: -------- find_nearest_station(lon,lat) find nearest station for given coordinates """ def __init__(self,path_to_data): """ collects metadata from all files in path_to_data and saves metadata information in numpy file in folder path_to_data/python_metadata/ First initialization can take a minute or so if all ISMN data is present in path_to_data Parameters: ----------- path_to_data : string filepath to unzipped ISMN data containing the Network folders """ if not os.path.exists(os.path.join(path_to_data,'python_metadata','metadata.npy')): self.metadata = metadata_collector.collect_from_folder(path_to_data) os.mkdir(os.path.join(path_to_data,'python_metadata')) np.save(os.path.join(path_to_data,'python_metadata','metadata.npy'), self.metadata) #np.savetxt(os.path.join(path_to_data,'python_metadata','metadata.npy'), self.metadata,delimiter=',') else: self.metadata = np.load(os.path.join(path_to_data,'python_metadata','metadata.npy')) self.kdTree = None
[docs] def find_nearest_station(self,lon,lat,return_distance=False): """ finds the nearest station available in downloaded data Parameters: ----------- lon : float Longitude of point lat : float Latitude of point return_distance : boolean, optional if True also distance is returned Returns: -------- station : ISMN_station ISMN_station object distance : float, optional distance to station in meters, measured in cartesian coordinates and not on a great circle. Should be OK for small distances """ if self.kdTree is None: self.kdTree = NN.findGeoNN(self.metadata['longitude'],self.metadata['latitude']) d,index = self.kdTree.find_nearest_index(lon, lat) all_index = np.where(self.metadata['station'] == self.metadata['station'][index]) if return_distance: return ISMN_station(self.metadata[all_index]),d else: return ISMN_station(self.metadata[all_index])
[docs] def get_all_data_for_variable_depth_min_max(self,variable,min_depth,max_depth): """ function to go through all downloaded data and read the data for the given variable if the depth it was measured at lies completely between min_depth and max_depth Parameters: ----------- variable: string variable to read min_depth : float depth_from of variable has to be >= min_depth in order to be included. max_depth : float depth_to of variable has to be <= max_depth in order to be included. Yields: ------- time_series : readers.ISMNTimeSeries ISMNTimeSeries object containing data and metadata """ for time_series in self.metadata: if time_series['variable'] != variable: continue if ((time_series['depth_from'] >= min_depth) & (time_series['depth_to'] <= max_depth)): yield readers.read_data(time_series['filename'])
import matplotlib.pyplot as plt #=============================================================================== # start = datetime.now() # rootdir = os.path.join(os.sep+'media','sf_D','small_projects','cpa_2013_07_ISMN_userformat_reader','ISMN_parser_test') MyISMN = ISMN_Interface(rootdir) # # print datetime.now() - start # # lat,lon = 13.548,2.68 # # Station,distance = MyISMN.find_nearest_station(lon, lat) # # print Station.latitude,Station.longitude # # print distance # print Station.get_depths('sm') # print Station.get_sensors('sm', 0.05, 0.05) # data = Station.read_variable('sm',0.05,0.05,'CS616_1') #=============================================================================== for time_series in MyISMN.get_all_data_for_variable_depth_min_max('sm', 0.5, 2): print (time_series.network,time_series.station, time_series.sensor) time_series.plot() plt.show()