pro statistic_latency_UMD_atmPrf_new_name_convention ; ---- ; 1) observation time ; 2) L1A file ; 2.1) creation time ; 2.2) archive time, from the L1A_log files ; 3) L1B file ; 4) L2 file ; 4.1) UMD L2 ; 4.2) UCAR L2 RO_mission = 'Spire' ; --- version='v3.14' ; --- in_path_L1A = '/ROSDC_data3/xinjiaz/data_GPSRO/spire_scdr/LEMUR_L0_RO/' in_path_L1B_UMD = '/ROSDC_data3/xinjiaz/data_GPSRO/RFSI_UMD/'+version+'/spirert/atmPhs_nccf/' ;; NCCF in_path_L1B_UMD = '/ROSDC_data3/xinjiaz/data_GPSRO/RFSI_UMD/'+version+'/spirert/atmPhs_rhw1271/' ;; rhw1271 in_path_L1B_UMD = '/ROSDC_data3/xinjiaz/data_GPSRO/RFSI_UMD/'+version+'/spirert/atmPhs_rosdc/' ;; rosdc in_path_L2_UCAR = '/data2/xinjiaz/data_GPSRO/UCAR/spire/DO1/level2/atmPrf/' ; --- first_Date='2026-03-29' first_Date='2026-05-10' parts = STRSPLIT(first_Date, '-', /EXTRACT) & year = FIX(parts[0]) & month = FIX(parts[1]) & day = FIX(parts[2]) end_Date='2026-03-29' end_Date='2026-05-10' parts = STRSPLIT(end_Date, '-', /EXTRACT) & year2 = FIX(parts[0]) & month2= FIX(parts[1]) & day2 = FIX(parts[2]) ; --- filename='Latency_'+RO_mission+'_umd-tcliu_'+end_Date filename='Latency_'+RO_mission+'_umd-Xinjia_'+end_Date filename='Latency_'+RO_mission+'_nccf_'+end_Date filename='Latency_'+RO_mission+'_rhw1271_'+end_Date filename='Latency_'+RO_mission+'_rosdc_'+end_Date ;if !true then begin if !false then begin ; unit is day files_L1A =[] files_L1B_UMD =[] files_L2_UMD =[] files_L1B_UCAR=[] files_L2_UCAR =[] otime =[] mtime_L1A =[] ctime_L1A =[] mtime_L1B_UMD =[] mtime_L1B_UCAR=[] mtime_L2_UMD =[] mtime_L2_UCAR =[] ; Convert to Julian days for easy looping sjd = JULDAY(month, day, year) ejd = JULDAY(month2, day2, year2) FOR jd = sjd, ejd DO BEGIN CALDAT, jd, month, day, year this_Date = STRING(year, format='(I4)') + '-' + $ STRING(month,format='(I2.2)') + '-' + $ STRING(day, format='(I2.2)') ; --- searching for file_L1A --- CALDAT, jd-1, month_last, day_last, year_last last_Date = STRING(year_last, format='(I4)') + '-' + $ STRING(month_last,format='(I2.2)') + '-' + $ STRING(day_last, format='(I2.2)') ; --- doy for UCAR --- doy = jd - julday(1,0,year) this_Date2= string(year,format='(i4)')+'/'+string(doy, format='(i3.3)') CALDAT, jd-1, month, day, year doy = jd -1 - julday(1,0,year) last_Date2= string(year,format='(i4)')+'/'+string(doy, format='(i3.3)') CALDAT, jd+1, month, day, year doy = jd +1 - julday(1,0,year) next_Date2= string(year,format='(i4)')+'/'+string(doy, format='(i3.3)') file_list=file_search(in_path_L1B_UMD+this_Date+'/*nc',count=file_count) print,'file_count=',file_count if file_count eq 0 then begin print, 'non-exist:',in_path_L1B_UMD+this_Date+'/*nc' stop endif ; --- loop along L1B files, which has attribute to host the L1A file for file_loop = 0, file_count-1 do begin ;for file_loop = 18, file_count-1 do begin file_L1B_UMD = file_list[file_loop] print, file_loop,'/',strtrim(string(file_count),2), ', ', file_L1B_UMD ncfileid = NCDF_OPEN(file_L1B_UMD) NCDF_attget, ncfileid, 'year',year,/global NCDF_attget, ncfileid, 'month',month,/global NCDF_attget, ncfileid, 'day',day,/global NCDF_attget, ncfileid, 'hour',hour,/global NCDF_attget, ncfileid, 'minute',minute,/global NCDF_attget, ncfileid, 'second',second,/global NCDF_attget, ncfileid, 'fileStamp',fileStamp,/global & fileStamp=string(fileStamp) NCDF_attget, ncfileid, 'tarfile',tarfile,/global & tarfile=string(tarfile) ncdf_close, ncfileid doy = julday(month, day, year) - julday(1,0,year) jd_obs = julday(month, day, year, hour, minute, second) fileStamp_array = strsplit(fileStamp,'_',/extract) gnss=fileStamp_array[3] leo = fileStamp_array[0] ;if RO_mission eq 'Spire' then leo='FM'+strmid(leo,1) ; --- searching for file_L1A file_L1A=in_path_L1A+this_Date+'/'+tarfile if ~file_test(file_L1A) then file_L1A = in_path_L1A+last_Date+'/'+tarfile if ~file_test(file_L1A) then stop ; --- searching for file_L2_UMD file_L2_UMD = StrJoin(StrSplit(file_L1B_UMD,'atmPhs',/Regex,/Extract,/Preserve_Null), 'atmPrf') file_L2_UMD = StrJoin(StrSplit(file_L2_UMD,'L1A',/Regex,/Extract,/Preserve_Null), 'L1C') if ~file_test(file_L2_UMD) then begin print, '~file_test(file_L2_UMD): ',file_basename(file_L1B_UMD) file_L2_UMD = 'n/a' endif ; --- searching for file_L2_UCAR file_list_L2_UCAR = [] last_list_L2_UCAR=file_search(in_path_L2_UCAR+last_Date2+'/*'+leo+'*'+gnss+'*',count=file_count_last) if file_count_last gt 0 then file_list_L2_UCAR = [file_list_L2_UCAR, last_list_L2_UCAR] this_list_L2_UCAR=file_search(in_path_L2_UCAR+this_Date2+'/*'+leo+'*'+gnss+'*',count=file_count_this) if file_count_this gt 0 then file_list_L2_UCAR = [file_list_L2_UCAR, this_list_L2_UCAR] next_list_L2_UCAR=file_search(in_path_L2_UCAR+next_Date2+'/*'+leo+'*'+gnss+'*',count=file_count_next) if file_count_next gt 0 then file_list_L2_UCAR = [file_list_L2_UCAR, next_list_L2_UCAR] ;print, 'n_elements(file_list_L2_UCAR):',n_elements(file_list_L2_UCAR) file_L2_UCAR = [] for file_loop2= 0, n_elements(file_list_L2_UCAR)-1 do begin ; --- atmPrf_GN04.2026.001.01.15.C24_0001.0001_nc --- file_contents = strsplit(file_basename(file_list_L2_UCAR[file_loop2]),'.',/extract) year2=file_contents[1] doy2=file_contents[2] hour2=file_contents[3] minute2=file_contents[4] jd2 = julday(1, doy2, year2, hour2, minute2) ; --- 2025-11-20T03-25-55 --- ;file_contents = strsplit(file_basename(file_list_L2_UCAR[file_loop2]),'_',/extract) ;day2=strmid(file_contents[2], 8,2) ;hour2=strmid(file_contents[2], 11,2) ;minute2=strmid(file_contents[2], 14,2) ;second2=strmid(file_contents[2], 17,2) time_diff = (jd_obs - jd2) * 24 *60 ; unit is minute if abs( time_diff ) lt 10. then begin file_L2_UCAR = file_list_L2_UCAR[file_loop2] break ; stop endif ;if !true then begin ; for debug if !false then begin ; for debug print, file_basename(file_L1B_UMD) print, file_basename(file_list_L2_UCAR[file_loop2]) print, float(doy) , float(hour) ,float(minute) , float(doy2) , float(hour2) ,float(minute2) print, abs(time_diff) stop ; for debug endif endfor if n_elements(file_L2_UCAR) ge 1 then begin ; --- searching for file_L1B_UMD file_L1B_UCAR = StrJoin(StrSplit(file_L2_UCAR,'level2',/Regex,/Extract,/Preserve_Null), 'level1b') file_L1B_UCAR = StrJoin(StrSplit(file_L1B_UCAR,'atmPrf',/Regex,/Extract,/Preserve_Null), 'conPhs') if ~file_test(file_L1B_UCAR) then stop endif else begin print, 'n_elements(file_L2_UCAR) eq 0: ',file_basename(file_L1B_UMD) file_L2_UCAR = 'n/a' file_L1B_UCAR= 'n/a' endelse ; ---- ; the IDL info time is UTC based info = FILE_INFO(file_L1A) time_L1A_m = info.mtime / 86400.d + julday(1,1,1970,0,0,0) ; about 1 minute later than on NCCF time_L1A_c = info.ctime / 86400.d + julday(1,1,1970,0,0,0) ; scdr available time info = FILE_INFO(file_L1B_UMD) time_L1B_UMD = info.mtime / 86400.d + julday(1,1,1970,0,0,0) if file_L2_UMD eq 'n/a' then begin time_L2_UMD = !values.f_nan endif else begin info = FILE_INFO(file_L2_UMD) time_L2_UMD= info.mtime / 86400.d + julday(1,1,1970,0,0,0) endelse if file_L1B_UCAR eq 'n/a' then begin time_L1B_UCAR = !values.f_nan time_L2_UCAR = !values.f_nan endif else begin info = FILE_INFO(file_L1B_UCAR) time_L1B_UCAR = info.mtime / 86400.d + julday(1,1,1970,0,0,0) info = FILE_INFO(file_L2_UCAR) time_L2_UCAR= info.mtime / 86400.d + julday(1,1,1970,0,0,0) endelse ; ---- files_L1A = [files_L1A, file_L1A] files_L1B_UMD = [files_L1B_UMD, file_L1B_UMD] files_L2_UMD = [files_L2_UMD, file_L2_UMD] files_L1B_UCAR= [files_L1B_UCAR,file_L1B_UCAR] files_L2_UCAR = [files_L2_UCAR, file_L2_UCAR] otime = [otime, jd_obs ] mtime_L1A = [mtime_L1A, time_L1A_m ] ctime_L1A = [ctime_L1A, time_L1A_c ] mtime_L1B_UMD = [mtime_L1B_UMD, time_L1B_UMD ] mtime_L2_UMD = [mtime_L2_UMD , time_L2_UMD ] mtime_L1B_UCAR= [mtime_L1B_UCAR,time_L1B_UCAR] mtime_L2_UCAR = [mtime_L2_UCAR, time_L2_UCAR ] endfor ; files in one day endfor ; jd_list save, first_Date, end_Date, $ files_L1A, $ files_L1B_UMD, files_L2_UMD, $ files_L1B_UCAR,files_L2_UCAR,$ otime, $ mtime_L1A, ctime_L1A, $ mtime_L1B_UMD, mtime_L2_UMD, $ mtime_L1B_UCAR, mtime_L2_UCAR,$ filename=filename+'.sav' print, filename+'.sav' endif else begin restore,filename+'.sav' endelse ;------------------ if !true then begin ;if !false then begin minus_sign = '!Z(u2212)' ; minus sign if first_Date eq end_Date then title2 = first_Date $ else title2 = first_Date+' to '+ end_Date ;indices = WHERE( STRPOS(files_L1B_UMD, '_') GE 0 ) & title1= RO_mission+'(All)' ;indices = WHERE( STRPOS(files_L1B_UMD, 'GN04') GE 0 ) & title1=RO_mission+'(GN04)' ;indices = WHERE( STRPOS(files_L1B_UMD, 'GN05') GE 0 ) & title1=RO_mission+'(GN05)' ;indices = WHERE( STRPOS(files_L1B_UMD, 'YM08') GE 0 ) & title1=RO_mission+'(YM08)' title1= RO_mission title1= 'UCAR '+RO_mission title1= 'STAR (rhw9163, 8/8 CPU)' title1= 'NCCF (c6g.xlarge, 6/6 CPU)' title1= 'UCAR' title1= 'STAR (rhw1271, 24/28 CPU)' title1= 'NCCF (c7a.4xlarge, 8/16 CPU)' title1= 'UMD (ROSDC, 8/96 CPU)' subtrahend = 'otime' subtrahend = 'ctime_L1A' ; L1A TAR file archive time, for rhw1271/rosdc subtrahend = 'mtime_L1A' ; L1A TAR file creation time, for nccf if subtrahend eq 'otime' then begin ;XRange=[0, 180] & xticks = 6 XRange=[0, 240] & xticks = 6 YRange=[0, 400] XRange=[0, 900] & xticks = 6 YRange=[0, 20] ;XTitle = 'L0 Creation Time '+minus_sign+' Observation Time (Minute)' ;time_diff = (mtime_L1A-otime)*24.*60 XTitle = 'L1A Creation Time '+minus_sign+' Observation Time (Minute)' time_diff = (mtime_L1B_UMD-otime)*24.*60 ;XTitle = 'L1C Creation Time '+minus_sign+' Observation Time (Minute)' ;time_diff = (mtime_L2_UMD-otime)*24.*60 ;XRange=[0, 240] & xticks = 6 ;YRange=[0, 100] ;XTitle = 'L1B Creation Time '+minus_sign+' Observation Time (Minute)' ;time_diff = (mtime_L1B_UCAR-otime)*24.*60 ;XTitle = 'L2 Creation Time '+minus_sign+' Observation Time (Minute)' ;time_diff = (mtime_L2_UCAR-otime)*24.*60 endif else if subtrahend eq 'mtime_L1A' then begin XRange=[-60, 60] XRange=[-30, 90] XRange=[0, 60] XRange=[-100, 100] & xticks = 8 XRange=[-90, 90] & xticks = 6 XRange=[-5, 25] & xticks = 3 XRange=[0, 30] & xticks = 3 YRange=[0, 4000] YRange=[0, 2000] ;XTitle = 'L1A Archive Time on SCDR '+minus_sign+' L1A Creation Time (Minute)' ;time_diff = (mtime_L1A_log-mtime_L1A)*24.*60 ;XTitle = 'Time (FSI Creation '+minus_sign+' L1A Creation), Minute' ;XTitle = 'Time (Python L1B Creation '+minus_sign+' L1A Archive on NCCF), Minute' ;time_diff = (mtime_L1B_UMD-mtime_L1A)*24.*60 XTitle = 'Time (UCAR L1B Creation '+minus_sign+' L1A Archive on NCCF), Minute' time_diff = (mtime_L1B_UCAR-mtime_L1A)*24.*60 ;XTitle = 'Time (FSI Creation '+minus_sign+' L1A Creation), Minute' ;XTitle = 'Time (UCAR Creation '+minus_sign+' L1A Creation), Minute' ;XTitle = 'Time (FSI L2 Creation '+minus_sign+' L1A Archive on NCCF), Minute' ;XTitle = 'Time (UCAR L2 Creation '+minus_sign+' L1A Archive on NCCF), Minute' ;time_diff = (mtime_L2_UMD-mtime_L1A)*24.*60 ;time_diff = (mtime_L2_UCAR-mtime_L1A)*24.*60 ;time_diff = time_diff[where(time_diff gt -10 and time_diff lt 20)] ;time_diff = time_diff[where(time_diff gt 0 and time_diff lt 30)] ;time_diff = time_diff[where(time_diff gt 0 and time_diff lt 50)] ;time_diff = time_diff[where(time_diff gt 0 and time_diff lt 20)] endif else if subtrahend eq 'ctime_L1A' then begin ;XRange=[-10, 50] ;XRange=[-120, 120] XRange=[0, 60] & xticks = 6 XRange=[-90, 0] & xticks = 6 XRange=[-100, 1500] & xticks = 4 XRange=[-100, 20] & xticks = 6 XRange=[-90, 90] & xticks = 6 XRange=[-60, 60] XRange=[0, 30] & xticks = 3 YRange=[0, 4000] YRange=[0, 8000] YRange=[0, 1500] YRange=[0, 150] YRange=[0, 300] YRange=[0, 600] YRange=[0, 2000] ;XTitle = 'L1B Creation Time '+minus_sign+' L1A Archive Time on SCDR (Minute)' ;XTitle = 'L2 Creation Time '+minus_sign+' L1A Archive Time on SCDR (Minute)' XRange=[0, 5] & xticks = 5 YRange=[0, 1000] XRange=[0, 3*60] & xticks = 6 YRange=[0, 50] ;XTitle = 'L1A Creation Time '+minus_sign+' L0 Archive Time on SCDR, Minute' ;XTitle = 'L1A Creation Time '+minus_sign+' L0 Archive Time on SCDR, Second' ;time_diff = (mtime_L1B_UMD-ctime_L1A)*24.*60 ;time_diff = (mtime_L1B_UMD-ctime_L1A)*24.*60*60 ;XTitle = 'L1C Creation Time '+minus_sign+' L0 Archive Time on SCDR, Minute' ;XTitle = 'L1C Creation Time '+minus_sign+' L0 Archive Time on SCDR, Second' ;time_diff = (mtime_L2_UMD-ctime_L1A)*24.*60 ;time_diff = (mtime_L2_UMD-ctime_L1A)*24.*60*60 XTitle = 'L1A Creation Time '+minus_sign+' L0 Archive Time on SCDR, Minute' ;XTitle = 'L1A Creation Time '+minus_sign+' L0 Archive Time on SCDR, Second' time_diff = (mtime_L1B_UCAR-ctime_L1A)*24.*60 ;time_diff = (mtime_L1B_UCAR-ctime_L1A)*24.*60*60 ;XTitle = 'L1C Creation Time '+minus_sign+' L0 Archive Time on SCDR, Minute' ;time_diff = (mtime_L2_UCAR-ctime_L1A)*24.*60 ;time_diff = time_diff[where(time_diff ge 0)] ;time_diff = time_diff[where(time_diff lt 20)] ;time_diff = time_diff[where(time_diff ge 0 and time_diff lt 20)] print, min(time_diff,id) print, files_L2_UMD[id] print, files_L1B_UMD[id] print, files_L1A[id] ;stop if !false then begin ;if !true then begin this_id = where(time_diff lt 10*60) print, 'files_L1A: ',files_L1A[this_id[0]] print, 'files_L1B_UMD: ',files_L1B_UMD[this_id[0]] print, 'files_L2_UCAR: ',files_L2_UCAR[this_id[0]] ;stop time_diff = time_diff[this_id] endif endif else if subtrahend eq 'mtime_L1B' then begin XRange=[0, 60] XRange=[-60, 60] YRange=[0, 10000] XTitle = 'L2 Creation Time '+minus_sign+' L1B Creation Time (Minute)' time_diff = (mtime_L2_UCAR-mtime_L1B_UCAR)*24.*60 endif ; --- ;indices = where(time_diff lt -100, check_num) indices = where(time_diff gt 20, check_num) if check_num gt 0 then begin print, 'time_diff gt 20', check_num print, files_L1A[indices[0]] print, files_L1B_UMD[indices[0]] print, files_L2_UMD[indices[0]] print, files_L1B_UCAR[indices[0]] print, files_L2_UCAR[indices[0]] endif ; XTitle = 'EUMT processing time '+minus_sign+' SCDR archive time (Minute)' binsize = 1 pdf_all = HISTOGRAM(time_diff, binsize=binsize, min=XRange[0], max=XRange[1]+binsize, locations=xbin_all) size_this = fix((XRange[1]-XRange[0])/binsize) xs=600 ys=600 res = 4 color_table, R, G, B Set_Plot, 'Z', /COPY DEVICE, SET_RESOLUTION=[xs*res, ys*res], Z_BUFFER=0 ; + Start PS operations !P.POSITION=[0.2,0.16,0.94,0.90] !P.POSITION=[0.18,0.16,0.94,0.90] !P.COLOR=1 !P.background=0 !P.THICK=3.0 !P.FONT=1 !X.THICK=3.0 !Y.THICK=3.0 TVLCT, R, G, B print,' max(pdf_all):', max(pdf_all) ;title = 'Sentinel6, 01/01/2021 to 03/31/2021' YTitle = 'Number of files' PLOT, [0,1], [0,1], TICKLEN=-0.012, /YNOZERO, /NOCLIP, $ XTitle=XTitle, YTitle=YTitle, $ ;title=title1+', '+title2, $ ;title=title1, $ charSize=1.9*res,charthick=res*1.5, $ ; xticks = 6, xtickname=['0','20','40','60','1"20','1"40','2'],$ xticks = xticks, $; 120 hours, 5 days,$ ;xticks = 5, $; 120 hours, 5 days,$ ;xticks = 4, $; 48 hours, 2 days,$ ; --- ;xminor = 10, $ xminor = 6, $ ;xminor = 5, $ ;xminor = 3, $ ;XTicks=n_elements(xtick_loc)-1, XTickv=xtick_loc , xtickname=xtick_name, XMinor=XMinor, $ ;YTicks=ROUND((maxYvalDsp - minYvalDsp)/ygridstep), YMinor=1, $ ;XRange=[floor(min(year_all)), ceil(max(year_all))],XSTYLE=1, xCHARSIZE=1.1, $ XRange=XRange,XSTYLE=1, xCHARSIZE=1.1, $ YRange=YRange,YSTYLE=1, yCHARSIZE=1.1, $ xthick=res, ythick=res, $ /NoErase, NoData=1 n_pdf = N_ELEMENTS(pdf_all[0:size_this-1]) data = pdf_all[0:size_this-1] x = INDGEN(n_pdf)*binsize x = xbin_all ; Build multi-segment arrays xx = REFORM( [[x], [x]], 2, N_ELEMENTS(x) ) yy = REFORM( [[INTARR(N_ELEMENTS(data))], [data]], 2, N_ELEMENTS(data) ) ; Draw spikes in ONE call ;width = binsize/2. ; bar half-width FOR i=0, N_ELEMENTS(data)-1 DO BEGIN ;polyx = [x[i]-width, x[i]+width, x[i]+width, x[i]-width] polyx = [x[i], x[i]+binsize, x[i]+binsize, x[i]] polyy = [0, 0, data[i], data[i]] POLYFILL, polyx, polyy, COLOR=10 ENDFOR ALIGNMENT = 1 ;xyouts, 0.93, 0.85, 'Mean = '+strtrim(string(mean(time_diff,/nan),format='(f8.1)'),2)+' min', ALIGNMENT=ALIGNMENT,CHARSIZE=2.2*res,CHARTHICK=2.2*res, /normal ;xyouts, 0.93, 0.81, 'Max = '+strtrim(string(max(time_diff),format='(f8.1)'),2)+' min', ALIGNMENT=ALIGNMENT,CHARSIZE=2.2*res,CHARTHICK=2.2*res, /normal xyouts, 0.93, 0.85, 'Mean = '+strtrim(string(mean(time_diff,/nan),format='(f8.1)'),2)+' sec', ALIGNMENT=ALIGNMENT,CHARSIZE=2.2*res,CHARTHICK=2.2*res, /normal xyouts, 0.93, 0.81, 'Max = '+strtrim(string(max(time_diff),format='(f8.1)'),2)+' sec', ALIGNMENT=ALIGNMENT,CHARSIZE=2.2*res,CHARTHICK=2.2*res, /normal ;xyouts, 0.93, 0.81, 'Max = 14.9 min', ALIGNMENT=ALIGNMENT,CHARSIZE=2.2*res,CHARTHICK=2.2*res, /normal ;xyouts, 0.93, 0.77, 'Min = '+strtrim(string(min(time_diff),format='(f8.1)'),2)+' min', ALIGNMENT=ALIGNMENT,CHARSIZE=2.2*res,CHARTHICK=2.2*res, /normal ;xyouts, 0.9, 0.5, RO_mission+' '+title2, ALIGNMENT=ALIGNMENT,CHARSIZE=2.2*res,CHARTHICK=2.2*res, /normal ;xyouts, 0.93, 0.65, RO_mission+' '+title2, ALIGNMENT=ALIGNMENT,CHARSIZE=2.2*res,CHARTHICK=2.2*res, /normal print, 'mean(time_diff):',mean(time_diff,/nan) print, 'max(time_diff):',max(time_diff) print, 'min(time_diff):',min(time_diff) im = TVRD() DEVICE, /Close ;DEVICE, /Close_File im24 = BYTARR(3, xs*res, ys*res) im24[0, *, *] = r[im] im24[1, *, *] = g[im] im24[2, *, *] = b[im] im3 = REBIN(im24, 3, xs, ys) ; Default is for bilinear interpolation print, filename+'_barplot.png' WRITE_PNG, filename+'_barplot.png', im3 endif stop END