# -*- coding: utf-8 -*- """ Created on Tue Feb 10 15:45:01 2026 The package is developed to monitor PlanetiQ L1A input files and RT BUFR products @author: jun.zhou@noaa.gov """ import matplotlib as mpl mpl.use('Agg') import matplotlib.pyplot as plt import matplotlib.dates as mdates from matplotlib.ticker import MultipleLocator import numpy as np import pandas as pd import os,sys from datetime import datetime, timezone from dataclasses import dataclass from zoneinfo import ZoneInfo import argparse @dataclass class plot_monitor: task: str now_utc: datetime fname_L2: str path_L2: str fname_L1A: str path_L1A: str def ingest(self): #print(self.task) L2_file = os.path.join(self.path_L2, self.fname_L2) self.df_L2 = pd.read_csv(L2_file) self.df_L2.columns = self.df_L2.columns.str.strip() self.df_L2["Total"] = pd.to_numeric(self.df_L2["Total"], errors="coerce") self.df_L2["QCed"] = pd.to_numeric(self.df_L2["QCed"], errors="coerce") self.df_L2["QC_Rate"] = pd.to_numeric(self.df_L2["QC_Rate"], errors="coerce") self.df_L2["UTC"] = pd.to_datetime(self.df_L2["UTC"], utc=True) self.df_L2["ET"] = pd.to_datetime(self.df_L2["ET"], utc=True) self.df_L2=self.df_L2.set_index('UTC') ## --- L1a_file = os.path.join(self.path_L1A, self.fname_L1A) self.df_L1A = pd.read_csv(L1a_file) self.df_L1A.columns = self.df_L1A.columns.str.strip() self.df_L1A["GN04"] = pd.to_numeric(self.df_L1A["GN04"], errors="coerce") self.df_L1A["GN05"] = pd.to_numeric(self.df_L1A["GN05"], errors="coerce") self.df_L1A["YM08"] = pd.to_numeric(self.df_L1A["YM08"], errors="coerce") self.df_L1A["total"] = pd.to_numeric(self.df_L1A["total"], errors="coerce") self.df_L1A["UTC"] = pd.to_datetime(self.df_L1A["UTC"], utc=True) self.df_L1A["ET"] = pd.to_datetime(self.df_L1A["ET"], utc=True) self.df_L1A=self.df_L1A.set_index('UTC') def plot_monitor(self): self.monitor_start = datetime(2026, 2, 3,0,0,0, tzinfo=timezone.utc) self.plot_L2_BUFR() self.plot_L1A_TAR() #======== BUFR ======================================================== def plot_L2_BUFR(self): font_jun={'size':45} mpl.rc('font',**font_jun) fig, ax = plt.subplots(nrows=4, ncols=1, figsize=(25,35)) now = self.now_utc #======== recent 30 hours ======================================================== end_time = now start_time = now - pd.Timedelta(hours=31) df_hours = self.df_L2.loc[ (self.df_L2.index >= start_time) & (self.df_L2.index <= end_time) ] df_hours_L1A = self.df_L1A.loc[ (self.df_L1A.index >= start_time) & (self.df_L1A.index <= end_time) ] UTC_min, UTC_max = start_time, end_time EST_min, EST_max = UTC_min - pd.Timedelta(hours=5), UTC_max - pd.Timedelta(hours=5) # ---------------- Panel 1: counts ---------------- ax[0].plot(df_hours.index, df_hours['Total'], 'k-', linewidth=2, label='Total') ax[0].plot(df_hours.index, df_hours['QCed'], 'g-', linewidth=2, label='QCed') ax[0].xaxis.set_major_locator(mdates.HourLocator(interval=3)) ax[0].xaxis.set_major_formatter(mdates.DateFormatter("%H:%M", tz=mdates.UTC)) ax[0].set_xlabel( f'UTC') ax[0].set_ylabel('Profile Number') ax[0].set_ylim(0,df_hours['Total'].max()+1) #ymin, ymax = ax[0].get_ylim() #print("Y-range:", ymin, ymax) ax[0].set_xlim(UTC_min, UTC_max) ax[0].tick_params(axis='x', direction='in', length=320, width=2, colors='darkgrey', labelcolor='k', pad=25) #--- second x-axis for Eastern time ------- ax_top0 = ax[0].twiny() ax_top0.plot(df_hours['ET'], df_hours['Total'], alpha=0) ax_top0.xaxis.set_major_locator(mdates.HourLocator(interval=3)) ax_top0.xaxis.set_major_formatter(mdates.DateFormatter("%H:%M")) ax_top0.xaxis.set_minor_locator(mdates.HourLocator(interval=1)) ax_top0.grid(which='minor', axis='x', linestyle='--', linewidth=2.0, color='0.7', alpha=1.0, zorder=0) ax_top0.set_xlabel('Eastern Time') ax_top0.set_xlim(EST_min, EST_max) ax_top0.set_ylim(0,df_hours['Total'].max()+1) ax_top0.tick_params(axis='x', pad=20) #--- second y-axis for L1A TAR files ------- ax_r0 = ax[0].twinx() ax_r0.plot( df_hours_L1A.index, df_hours_L1A['total'], color='tab:pink', marker='o', markersize=6, linewidth=3, label='L1A TAR') ax_r0.set_ylabel('L1A File Number', color='tab:pink') ax_r0.tick_params(axis='y', colors='tab:pink') ax_r0.set_ylim(0, df_hours_L1A['total'].max() + 1) ax_r0.grid(False) ax[0].text(-0.33, 0.5, 'Last 30 hours\n (30-min)', transform=ax[0].transAxes, fontsize=50) # ---------------- Panel 2: QC rate ---------------- ax[1].plot(df_hours.index, df_hours['QC_Rate'], 'r-', linewidth=2, label='QC Rate') ax[1].scatter(df_hours.index, df_hours['QC_Rate'], color='r', s=30) ax[1].xaxis.set_major_locator(mdates.HourLocator(interval=3)) ax[1].xaxis.set_major_formatter(mdates.DateFormatter("%H:%M", tz=mdates.UTC)) ax[1].set_xlabel( f'UTC') ax[1].set_ylabel('QC Rate') ax[1].set_ylim(50, 100) ax[1].set_xlim(UTC_min, UTC_max) # y-axis major ticks every 10 ax[1].yaxis.set_major_locator(MultipleLocator(10)) # enable grid on y-axis (major ticks) ax[1].grid(which='major', axis='y', linestyle='--', linewidth=1.5, color='0.7', alpha=1.0, zorder=0) ax[1].tick_params(axis='x', direction='in', length=320, width=2, colors='darkgrey', labelcolor='k', pad=25) #--- second x-axis for Eastern time ------- ax_top1 = ax[1].twiny() ax_top1.plot(df_hours['ET'], df_hours['Total'], alpha=0) ax_top1.xaxis.set_major_locator(mdates.HourLocator(interval=3)) ax_top1.xaxis.set_major_formatter(mdates.DateFormatter("%H:%M")) ax_top1.xaxis.set_minor_locator(mdates.HourLocator(interval=1)) ax_top1.grid(which='minor', axis='x', linestyle='--', linewidth=2.0, color='0.7', alpha=1.0, zorder=0) ax_top1.set_xlabel('Eastern Time') ax_top1.set_xlim(EST_min, EST_max) ax_top1.set_ylim(50, 100) ax_top1.tick_params(axis='x', pad=20) ax[1].text(-0.33, 0.5, 'Last 30 hours\n (30-min)', transform=ax[1].transAxes, fontsize=50) # ---- combined legend from left + right y-axes ---- h1, l1 = ax[0].get_legend_handles_labels() h2, l2 = ax_r0.get_legend_handles_labels() h_loc = 0.37 ax[0].legend( h1 + h2, l1 + l2, loc='upper center', bbox_to_anchor=(h_loc, 1.65), ncol=3, frameon=False,fontsize=50) ax[1].legend(loc='upper center', bbox_to_anchor=(h_loc+0.49, 3.4), ncol=1, frameon=False,fontsize=50) # ---- combined legend from left + right y-axes ---- #======== recent 30 days daily======================================================== day_end= self.now_utc day_start0=(day_end-pd.Timedelta(days=30)).replace(hour=0, minute=0, second=0, microsecond=0) day_start = max(day_start0, self.monitor_start) df_days = self.df_L2.loc[(self.df_L2.index >= day_start)& (self.df_L2.index <= day_end)] df_days=df_days.drop(columns=['ET']) df_daily=df_days.resample('D').sum() df_daily['QC_Rate']=df_daily['QCed']*100./df_daily['Total'] df_daily_L1A = self.df_L1A.loc[(self.df_L1A.index >= day_start) & (self.df_L1A.index <= day_end)] df_daily_L1A = df_daily_L1A.drop(columns=['ET']).resample('D').sum() ax[2].plot(df_daily.index,df_daily['Total'],'k-',linewidth=2, label='Total') ax[2].plot(df_daily.index,df_daily['QCed'],'g-',linewidth=2, label='QCed') ax[2].xaxis.set_major_locator(mdates.DayLocator(interval=3)) ax[2].xaxis.set_major_formatter(mdates.DateFormatter("%m-%d", tz=mdates.UTC)) ax[2].xaxis.set_minor_locator(mdates.DayLocator(interval=1)) ax[2].grid(which='minor', axis='x', linestyle='--', linewidth=2.0, color='0.7', alpha=1.0, zorder=0) ax[2].set_xlabel('Date') ax[2].set_ylabel('Profile Number') #ax[2].set_ylim(0,10000) ax[2].set_ylim(0,df_daily['Total'].max()+1000) ax[2].set_xlim(day_start0, df_daily.index.max()) #--- second y-axis for L1A TAR files ------- ax_r0 = ax[2].twinx() ax_r0.plot( df_daily_L1A.index, df_daily_L1A['total'], color='tab:pink', marker='o', markersize=6, linewidth=3, label='L1A TAR') ax_r0.set_ylabel('L1A File Number', color='tab:pink') ax_r0.tick_params(axis='y', colors='tab:pink') ax_r0.set_ylim(0, df_daily_L1A['total'].max() + 10) ax_r0.grid(False) ax[2].text(-0.33, 0.5, 'Last 30 days\n (Daily)',transform=ax[2].transAxes,fontsize=50) ax[2].tick_params(axis='x', direction='in',length=320,width=2,colors='darkgrey',labelcolor='k',rotation=10) ax[3].plot(df_daily.index,df_daily['QC_Rate'],'r-',linewidth=2, label='QC Rate') ax[3].scatter(df_daily.index,df_daily['QC_Rate'],color='r',s=30) ax[3].xaxis.set_major_locator(mdates.DayLocator(interval=3)) ax[3].xaxis.set_major_formatter(mdates.DateFormatter("%m-%d", tz=mdates.UTC)) ax[3].xaxis.set_minor_locator(mdates.DayLocator(interval=1)) ax[3].grid(which='minor', axis='x', linestyle='--', linewidth=2.0, color='0.7', alpha=1.0, zorder=0) # y-axis major ticks every 10 ax[3].yaxis.set_major_locator(MultipleLocator(10)) # enable grid on y-axis (major ticks) ax[3].grid(which='major', axis='y', linestyle='--', linewidth=1.5, color='0.7', alpha=1.0, zorder=0) ax[3].set_xlabel('Date') ax[3].set_ylabel('QC Rate') ax[3].set_ylim(50,100) ax[3].set_xlim(day_start0, df_daily.index.max()) ax[3].text(-0.33, 0.5, 'Last 30 days\n (Daily)',transform=ax[3].transAxes,fontsize=50) ax[3].tick_params(axis='x', direction='in',length=320,width=2,colors='darkgrey',labelcolor='k',rotation=10) # ---- combined legend from left + right y-axes ---- h1, l1 = ax[2].get_legend_handles_labels() h2, l2 = ax_r0.get_legend_handles_labels() v_loc = 1.35 h_loc = 0.37 ax[2].legend( h1 + h2, l1 + l2, loc='upper center', bbox_to_anchor=(h_loc, v_loc), ncol=3, frameon=False,fontsize=50) ax[3].legend(loc='upper center', bbox_to_anchor=(h_loc+0.49, v_loc+1.75), ncol=1, frameon=False,fontsize=50) # ---- combined legend from left + right y-axes ---- now_et = self.now_utc.replace(tzinfo=ZoneInfo("UTC")).astimezone( ZoneInfo("US/Eastern")) fig.suptitle("PlanetiQ Level-2 BUFR File Monitoring (Last update: "+ now_et.strftime('%Y-%m-%d %H:%M')+'ET)', y=1.01,fontsize=60) plt.subplots_adjust(left=0.1, bottom=0.1, right=0.99, top=0.9, wspace=0.3, hspace=0.75) plt.savefig(os.path.join(self.path_L2,'Monitor_L2_BUFR_30min_'+self.task+'.png'),dpi=300,bbox_inches='tight') print(os.path.join(self.path_L2,'Monitor_L2_BUFR_30min_'+self.task+'_star.png')) plt.close() #======== TAR ====================================================== def plot_L1A_TAR(self): font_jun={'size':40} mpl.rc('font',**font_jun) fig, ax = plt.subplots(nrows=3, ncols=1, figsize=(25,25)) now = self.now_utc #======= current day ======================================================== UTC_min_26 = now - pd.Timedelta(hours=27) UTC_min = now - pd.Timedelta(hours=25) UTC_max = now EST_min = UTC_min - pd.Timedelta(hours=5) EST_max = UTC_max - pd.Timedelta(hours=5) df_24h = self.df_L1A.loc[(self.df_L1A.index >= UTC_min_26) & (self.df_L1A.index <= UTC_max)] df_24h_plot = df_24h.copy() #df_24h_plot.index = df_24h_plot.index + pd.Timedelta(minutes=30) if len(df_24h) > 0: ax[0].plot(df_24h_plot.index, df_24h_plot['GN04'], 'b-', marker='o', markersize=6, linewidth=2, label='GN04') ax[0].plot(df_24h_plot.index, df_24h_plot['GN05'], 'g-', marker='o', markersize=6, linewidth=2, label='GN05') ax[0].plot(df_24h_plot.index, df_24h_plot['YM08'], 'r-', marker='o', markersize=6, linewidth=2, label='YM08') ax[0].plot(df_24h_plot.index, df_24h_plot['total'], 'k-', marker='o', markersize=6, linewidth=2, label='Total') ax[0].set_ylim(0, df_24h['total'].max() + 1) ax_top0 = ax[0].twiny() ax_top0.plot(df_24h['ET'], df_24h['total'], alpha=0) ax_top0.xaxis.set_major_locator(mdates.HourLocator(interval=3)) ax_top0.xaxis.set_major_formatter(mdates.DateFormatter("%H:%M", tz=mdates.UTC)) ax_top0.set_xlabel('Eastern Time') ax_top0.set_xlim(EST_min + pd.Timedelta(minutes=30), EST_max + pd.Timedelta(minutes=30)) ax[0].set_xlim(UTC_min + pd.Timedelta(minutes=30), UTC_max + pd.Timedelta(minutes=30)) ax[0].xaxis.set_major_locator(mdates.HourLocator(interval=3)) ax[0].xaxis.set_major_formatter(mdates.DateFormatter("%H:%M", tz=mdates.UTC)) ax[0].grid(which='major', axis='x', linestyle='-', linewidth=2.0, color='0.7', alpha=1.0,zorder=0) ax[0].xaxis.set_minor_locator(mdates.HourLocator(interval=1)) ax[0].grid(which='minor', axis='x', linestyle='--', linewidth=2.0, color='0.7', alpha=1.0, zorder=0) ax[0].set_xlabel('UTC') ax[0].set_ylabel('L1A Input File') ax[0].text(-0.26, 0.5, 'Last 24 hours\n (30-min)',transform=ax[0].transAxes, fontsize=45) #======== recent 7 days ======================================================== UTC_min = now - pd.Timedelta(days=7) UTC_max = now UTC_min_day = pd.Timestamp(UTC_min).floor('D') UTC_max_day = pd.Timestamp(UTC_max).ceil('D') df_7days = self.df_L1A.loc[(self.df_L1A.index >= UTC_min_day) & (self.df_L1A.index <= UTC_max_day)] df_6h = df_7days.drop(columns=['ET']).resample('6h').sum() #df_6h = ( df_7days.drop(columns=['ET']).resample('6h',origin='start_day', # anchor to 00:00 # offset='0h') .sum()) #df_6h = (df_7days.drop(columns=['ET']) .resample('6h', origin='start_day', offset='0h', # label='left', closed='left') .sum()) ax[1].plot(df_6h.index, df_6h['GN04'], 'b-', linewidth=2) ax[1].plot(df_6h.index, df_6h['GN05'], 'g-', linewidth=2) ax[1].plot(df_6h.index, df_6h['YM08'], 'r-', linewidth=2) ax[1].plot(df_6h.index, df_6h['total'], 'k-', linewidth=2) #ax[1].set_xlim(UTC_min, UTC_max) ax[1].set_xlim(UTC_min_day, UTC_max_day) ax[1].xaxis.set_major_locator(mdates.DayLocator(interval=1)) ax[1].xaxis.set_major_formatter(mdates.DateFormatter("%m-%d")) ax[1].grid(which='major', axis='x', linestyle='-', linewidth=2.5, color='0.6', alpha=1.0,zorder=0) ax[1].xaxis.set_minor_locator(mdates.HourLocator(interval=6)) ax[1].grid(which='minor', axis='x', linestyle='--', linewidth=2.0, color='0.7', alpha=1.0, zorder=0) ax[1].set_xlabel('Date') ax[1].set_ylabel('L1A Input File') ax[1].set_ylim(0, df_6h['total'].max() + 1) ax[1].text(-0.26, 0.5, 'Last 7 days\n (6-Hourly)',transform=ax[1].transAxes, fontsize=45) #=========== 30 day daily ========================== UTC_min = now - pd.Timedelta(days=30) UTC_max = now day_start0=(now - pd.Timedelta(days=30)).replace(hour=0, minute=0, second=0, microsecond=0) df_30days = self.df_L1A.loc[(self.df_L1A.index >= UTC_min) & (self.df_L1A.index <= UTC_max)] df_daily = df_30days.drop(columns=['ET']).resample('D').sum() ax[2].plot(df_daily.index, df_daily['GN04'], 'b-', linewidth=2) ax[2].plot(df_daily.index, df_daily['GN05'], 'g-', linewidth=2) ax[2].plot(df_daily.index, df_daily['YM08'], 'r-', linewidth=2) ax[2].plot(df_daily.index, df_daily['total'], 'k-', linewidth=2) #ax[2].set_xlim(df_daily.index.min(), df_daily.index.max()) ax[2].set_xlim(day_start0, df_daily.index.max()) ax[2].xaxis.set_major_locator(mdates.DayLocator(interval=3)) ax[2].xaxis.set_major_formatter(mdates.DateFormatter("%m-%d")) ax[2].xaxis.set_minor_locator(mdates.DayLocator(interval=1)) ax[2].grid(which='minor', axis='x', linestyle='--', linewidth=2.0, color='0.7', alpha=1.0, zorder=0) ax[2].set_xlabel('Date') ax[2].set_ylabel('L1A Input File') ax[2].set_ylim(0, df_daily['total'].max() + 1) ax[2].text(-0.26, 0.5, 'Last 30 days\n (Daily)', transform=ax[2].transAxes, fontsize=45) #=========== ax[2].tick_params(axis='x',direction='in',length=377,width=2,colors='darkgrey',labelcolor='k') ax[0].legend(loc='upper center', bbox_to_anchor=(0.5, 1.45), ncol=4, frameon=False,fontsize=45) now_et = self.now_utc.replace(tzinfo=ZoneInfo("UTC")).astimezone( ZoneInfo("US/Eastern")) fig.suptitle("PlanetiQ Level-1A TAR File Monitoring (Last update: "+ now_et.strftime('%Y-%m-%d %H:%M')+'ET)',y=1.03,fontsize=50) plt.subplots_adjust(left=0.06, bottom=0.1, right=0.95, top=0.9, wspace=0.3, hspace=0.4) plt.savefig(os.path.join(self.path_L1A,'Monitor_L1A_TAR_30min_'+self.task+'.png'),dpi=300,bbox_inches='tight') print(os.path.join(self.path_L1A,'Monitor_L1A_TAR_30min_'+self.task+'_star.png')) plt.close() parser = argparse.ArgumentParser() parser.add_argument("task", help="task name: rosdc_v3.11 / rosdc_v3.14 / nccf_v3.14 / rhw1271_v3.14") parser.add_argument("now_utc_str", help="current / 2026-02-11_22:21:00") args=parser.parse_args() task=args.task now_utc_str=args.now_utc_str if now_utc_str=='current': now_utc = datetime.now(timezone.utc) else: now_utc = datetime.strptime(now_utc_str, "%Y-%m-%d_%H:%M:%S").replace(tzinfo=timezone.utc) print(task) print(now_utc) #sys.exit() # rosdc--------------------------------------------------------- if task == 'rosdc_v3.11': path_L2='/data3/xinjiaz/data_GPSRO/RFSI_UMD/v3.11/planetiqrt/atmPrf/' path_L1A='/data3/xinjiaz/data_GPSRO/planetiq_scdr/GNOMES_PYXIS_L1A/' elif task == 'rosdc_v3.14': path_L2='/data3/xinjiaz/data_GPSRO/RFSI_UMD/v3.14/planetiqrt/atmPrf/' path_L1A='/data3/xinjiaz/data_GPSRO/planetiq_scdr/GNOMES_PYXIS_L1A/' elif task == 'rhw1271_v3.14': path_L2='/data3/xinjiaz/data_GPSRO/RFSI_UMD/v3.14/planetiqrt/atmPrf_rhw1271/' path_L1A='/data3/xinjiaz/data_GPSRO/planetiq_scdr/GNOMES_PYXIS_L1A/' elif task == 'nccf_v3.14': path_L2='/data/data504/xzhou/gps_pod/L1A_L1B_ExcessPhase_v1.3_20251216_addconfig2/systemd/' path_L1A='/data/data504/xzhou/gps_pod/L1A_L1B_ExcessPhase_v1.3_20251216_addconfig2/systemd/' fname_L1A='count_L1A_TAR_30min_nccf.log' fname_L2 ='count_L2_BUFR_30min_nccf_v3.14.log' else: print('Invalid task, pls check!') sys.exit() plot_monitor_obj=plot_monitor(task,now_utc,fname_L2,path_L2,fname_L1A,path_L1A) plot_monitor_obj.ingest() plot_monitor_obj.plot_monitor()