from pathlib import Path
import os
import pandas as pd
from es_sfgtools.processing.operations.gnss_ops import rinex_to_kin,read_kin_data,plot_kin_results_wrms,get_wrms_from_res
from es_sfgtools.processing.pipeline.data_handler import DataHandler
pride_path = Path.home() / ".PRIDE_PPPAR_BIN"
os.environ["PATH"] += os.pathsep + str(pride_path)
Test 1: Validate PRIDE PPP_AR Processing
*Overview:
Process the same rinex file using different PRIDE PPP-AR installations on ES and UW software Files tested: the rinex2 file sfg11770.23o and the rinex3 file SFG100XXX_R_20231770000_01D_01S_MO.rnx Output files to compare: XYZ antenna positions logged in kin files generated by PRIDE Expected residuals: Very small, perhaps < millimeter level.
Other notes:
There is a bit of nuance here since there are two different formats of rinex data that may be used at this step in the processing chain, namely ES currently uses rinex v2 while UW currently uses rinex v3. For completeness, we will validate PRIDE runs using both rinex2 and rinex3 files derived from the same day of raw data. If we find that the PRIDE installations are generating significantly different outputs, this could be a sign that one of the softwares is using more appropriate processing flags when executing PRIDE, that there is a significant version difference in the two PRIDE installations, or that there is a major discrepancy in the orbital products used.
prideDir = Path("/Users/franklyndunbar/Project/SeaFloorGeodesy/Data/SFGMain/Pride")
writeDir = Path(
"/Users/franklyndunbar/Project/SeaFloorGeodesy/Data/GARPOS_Benchmarking_RINEX"
)
rnx_v2 = writeDir / "SFG100XXX_R_20231770000_01D_01S_MO.rnx"
rnx_v3 = writeDir / "sfg11770.23o"
kin_v2, res_v2 = rinex_to_kin(
source=rnx_v2, writedir=writeDir, pridedir=prideDir, site="TST1"
)
kin_v2_df = read_kin_data(kin_v2.local_path)
Converting RINEX file /Users/franklyndunbar/Project/SeaFloorGeodesy/Data/GARPOS_Benchmarking_RINEX/SFG100XXX_R_20231770000_01D_01S_MO.rnx to kin file 1;33mwarning:0m no observation for GNSS (E) 1;33mwarning:0m no observation for GNSS (C) 1;33mwarning:0m no observation for GNSS (J) 1;33mwarning:0m no satellite orbit for GNSS (J): WUM0MGXRAP_20231770000_01D_05M_ORB.SP3 1;33mwarning:0m no OSB for GNSS (J): L1 L2 Converted RINEX file /Users/franklyndunbar/Project/SeaFloorGeodesy/Data/GARPOS_Benchmarking_RINEX/SFG100XXX_R_20231770000_01D_01S_MO.rnx to kin file /Users/franklyndunbar/Project/SeaFloorGeodesy/Data/GARPOS_Benchmarking_RINEX/kin_2023177_tst1.kin Found PRIDE res file /Users/franklyndunbar/Project/SeaFloorGeodesy/Data/GARPOS_Benchmarking_RINEX/res_2023177_tst1.res
kin_v2_df = read_kin_data(kin_v2.local_path)
kin_v2_wrms_df = get_wrms_from_res(res_v2.local_path)
kin_v2_df_final = kin_v2_df.merge(
kin_v2_wrms_df, left_index=True, right_on="date", sort=True
)
plot_kin_results_wrms(kin_v2_df_final, title="rinex 2")
kin_v3, res_v3 = rinex_to_kin(
source=rnx_v3, writedir=writeDir, pridedir=prideDir, site="TST2"
)
kin_v3_df = read_kin_data(kin_v3.local_path)
Converting RINEX file /Users/franklyndunbar/Project/SeaFloorGeodesy/Data/GARPOS_Benchmarking_RINEX/sfg11770.23o to kin file 1;33mwarning:0m no observation for GNSS (E) 1;33mwarning:0m no observation for GNSS (C) 1;33mwarning:0m no observation for GNSS (J) 1;33mwarning:0m no satellite orbit for GNSS (J): WUM0MGXRAP_20231770000_01D_05M_ORB.SP3 1;33mwarning:0m no OSB for GNSS (J): L1 L2 Converted RINEX file /Users/franklyndunbar/Project/SeaFloorGeodesy/Data/GARPOS_Benchmarking_RINEX/sfg11770.23o to kin file /Users/franklyndunbar/Project/SeaFloorGeodesy/Data/GARPOS_Benchmarking_RINEX/kin_2023177_tst2.kin Found PRIDE res file /Users/franklyndunbar/Project/SeaFloorGeodesy/Data/GARPOS_Benchmarking_RINEX/res_2023177_tst2.res
kin_v3_wrms_df = get_wrms_from_res(res_v3.local_path)
kin_v3_df_final = kin_v3_df.merge(kin_v3_wrms_df, left_index=True, right_on="date", sort=True)
plot_kin_results_wrms(kin_v3_df_final, title="rinex 3")
Test 2: Validate rinex generation (Steps 1a and 1b)
*Overview: Generate daily rinex files from raw wave glider files and process 1Hz positions using PRIDE
Files tested: NOV770.raw files for day 177 in the 2023 NCL1 survey, full list below:
329653_001_20230625_232846_00282_NOV770.raw 329653_001_20230626_031035_00282_NOV770.raw 329653_001_20230626_065356_00282_NOV770.raw 329653_001_20230626_103028_00282_NOV770.raw 329653_001_20230626_142543_00282_NOV770.raw 329653_001_20230626_153909_00283_NOV770.raw 329653_001_20230626_192245_00283_NOV770.raw 329653_001_20230626_231837_00283_NOV770.raw
os.environ["DYLD_LIBRARY_PATH"] = os.environ.get("CONDA_PREFIX") + "/lib"
dh = DataHandler(prideDir.parent)
dh.change_working_station(network='cascadia-gorda',station='NCL1',campaign='2023_from_john')
raw_path = Path(
"/Users/franklyndunbar/Project/SeaFloorGeodesy/Data/Cascadia2023/NCL1/HR"
)
dh.discover_data_and_add_files(raw_path)
Building directory structure for cascadia-gorda NCL1 2023_from_john No date range set for cascadia-gorda, NCL1, 2023_from_john Building TileDB arrays for NCL1 Changed working station to cascadia-gorda NCL1 Found 155 files in /Users/franklyndunbar/Project/SeaFloorGeodesy/Data/Cascadia2023/NCL1/HR No date range set for cascadia-gorda, NCL1, 2023_from_john Building TileDB arrays for NCL1 Changed working station to cascadia-gorda NCL1 Found 155 files in /Users/franklyndunbar/Project/SeaFloorGeodesy/Data/Cascadia2023/NCL1/HR Added 0 out of 134 files to the catalog
ncl1_pipeline,ncl1_config = dh.get_pipeline_sv3()
ncl1_pipeline.run_pipeline()
shot_data_dates = dh.shotdata_tdb.get_unique_dates()
dfs = [dh.shotdata_tdb.read_df(x) for x in shot_data_dates.tolist()]
df = pd.concat(dfs)
position_df = df.loc[:,["triggerTime","east0","north0","up0","head0","pitch0","roll0"]].rename(columns={"triggerTime":"date","east0":"east","north0":"north","up0":"up","head0":"head","pitch0":"pitch","roll0":"roll"})
df_path = dh.station_dir / "position.csv"
position_df.to_csv(df_path,index=False)
gnss_dfs = [dh.gnss_tdb.read_df(x) for x in shot_data_dates.tolist()]
gnss_df = pd.concat(gnss_dfs)
gnss_df.to_csv(dh.station_dir / "gnss.csv",index=False)
def plot_kin_res(kin_path:Path,res_path:Path,title:str):
kin_df = read_kin_data(kin_path)
res_df = get_wrms_from_res(res_path)
kin_res_df = kin_df.merge(res_df,left_index=True,right_on="date",sort=True)
plot_kin_results_wrms(kin_res_df,title=title)
from es_sfgtools.processing.assets import AssetType
kin_files = dh.catalog.get_assets(network=dh.network,station=dh.station,campaign=dh.campaign,type=AssetType.KIN)
res_files = dh.catalog.get_assets(network=dh.network,station=dh.station,campaign=dh.campaign,type=AssetType.KINRESIDUALS)
get_doy_kin = lambda x: int(x.local_path.name.split("_")[1].replace("2023",""))
get_doy_res = lambda x: int(x.local_path.name.split("_")[1].replace("2023",""))
kin_files = sorted(kin_files,key=get_doy_kin)
res_files = sorted(res_files,key=get_doy_res)
for kin,res in zip(kin_files,res_files):
doy = get_doy_kin(kin)
title = f"Year 2023 DOY {doy}"
plot_kin_res(kin.local_path,res.local_path,title=title)
