# -*- coding: utf-8 -*- import sys,os import glob,time,subprocess import uuid,shutil from pathlib import Path from datetime import datetime,timedelta class batch_process_class: def __init__(self,year,doy,work_dir,input_dir,output_dir,check_leoid): self.year,self.doy,self.work_dir,self.input_dir,self.output_dir=year,doy,work_dir,input_dir,output_dir self.check_leoid=check_leoid ############################### self.process_log = os.path.join( self.output_dir, f'processed_{self.year}_{self.doy}_{self.check_leoid}.log' ) self.processed_files = self.load_processed_files() ############################### self.opnGns_output=os.path.join(self.output_dir,'opnGns',self.year,self.doy) self.podCrx_output=os.path.join(self.output_dir,'podCrx',self.year,self.doy) self.scnRaw_output=os.path.join(self.output_dir,'scnRaw',self.year,self.doy) self.opnScn_output=os.path.join(self.output_dir,'opnScn',self.year,self.doy) self.goxSOH_output=os.path.join(self.output_dir,'goxSOH',self.year,self.doy) os.makedirs(self.opnGns_output, exist_ok=True) os.makedirs(self.podCrx_output, exist_ok=True) os.makedirs(self.scnRaw_output, exist_ok=True) os.makedirs(self.opnScn_output, exist_ok=True) os.makedirs(self.goxSOH_output, exist_ok=True) ############################### def load_processed_files(self): processed = set() if os.path.exists(self.process_log): with open(self.process_log,'r') as f: for line in f: gps = line.strip().split() if len(gps) >= 3: processed.add(gps[-1]) return processed def record_processed_file(self,fname): nowstr = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') base = os.path.basename(fname) with open(self.process_log,'a') as f: f.write(f'{nowstr} {base}\n') self.processed_files.add(base) ############################### def sort_key(self,fname): base=os.path.basename(fname) gps=base.split('.') hr,minute,leoid=int(gps[2]),int(gps[3]),int(gps[4]) return (hr,minute,leoid) def decode_L0_L1A(self): if self.check_leoid=='all': pattern = os.path.join( self.input_dir, '*', f'trgLv0_{self.year}.{self.doy}.*.*.*.*_tlm.gz') else: pattern = os.path.join( self.input_dir, '*', f'trgLv0_{self.year}.{self.doy}.*.*.{self.check_leoid}.*_tlm.gz') fnames = glob.glob(pattern) #print(fnames) #sys.exit() if len(fnames)==0: print('No L0 files available, pls check!') #sys.exit() return fnames_sorted=sorted(fnames,key=self.sort_key) #--------------------------------------------------------- # skip already processed files #--------------------------------------------------------- fnames_sorted = [ f for f in fnames_sorted if os.path.basename(f) not in self.processed_files ] #--------------------------------------------------------- for ifile,fname in enumerate(fnames_sorted): #if ifile>0: return print(fname,'=================================') # --- create unique temp directory --- tmp_suffix = uuid.uuid4().hex[:8] tmp_opnGns = f"{self.opnGns_output}.tmp_{tmp_suffix}" tmp_podCrx = f"{self.podCrx_output}.tmp_{tmp_suffix}" tmp_scnRaw = f"{self.scnRaw_output}.tmp_{tmp_suffix}" tmp_opnScn = f"{self.opnScn_output}.tmp_{tmp_suffix}" tmp_goxSOH = f"{self.goxSOH_output}.tmp_{tmp_suffix}" for d in [tmp_opnGns, tmp_podCrx, tmp_scnRaw, tmp_opnScn, tmp_goxSOH]: os.makedirs(d, exist_ok=True) # --- create unique temp directory --- command=f'/usr/bin/perl {self.work_dir}/trigZero2one_star.pl \ {fname} \ {tmp_opnGns} \ {tmp_podCrx} \ {tmp_scnRaw} \ {tmp_opnScn} \ {tmp_goxSOH}' #ret=os.system(command) ret = subprocess.run(command, shell=True) # check success #if ret == 0: if ret.returncode == 0: # atomic replace for tmp, final in [ (tmp_opnGns, self.opnGns_output), (tmp_podCrx, self.podCrx_output), (tmp_scnRaw, self.scnRaw_output), (tmp_opnScn, self.opnScn_output), (tmp_goxSOH, self.goxSOH_output), ]: # sync files rsync_ret = os.system(f"rsync -a {tmp}/ {final}/") # record the filename in LOG self.record_processed_file(fname) # only remove tmp if rsync succeeded if rsync_ret == 0: shutil.rmtree(tmp) else: print(f"rsync failed for {tmp}, keeping temp for debug") else: print("Perl command failed, temp dirs kept for debug") msg="""command to process current day: python kick_trigZero2one_star.py 0 command to process yesterday: python kick_trigZero2one_star.py -1 command to process n days before today: python kick_trigZero2one_star.py -n by default: process all LEOs, you can also specify which LEO satellite you would like to process for a specific day: command to process a specified day for one LEO: python kick_trigZero2one_star.py 1 2025-12-16 002 """ if len(sys.argv)==2: #python kick_trigZero2one_star.py -2 offset = int(sys.argv[1]) dt = datetime.utcnow() + timedelta(days=offset) year = dt.strftime("%Y") mm = dt.strftime("%m") day = dt.strftime("%d") doy = dt.strftime("%j") check_leoid='all' elif len(sys.argv)==3: #python kick_trigZero2one_star.py 1 2025-12-16 year,mm,day=sys.argv[2].split('-') dt = datetime(int(year), int(mm), int(day)) doy = dt.strftime("%j") #doy=datetime(int(year),int(mm),int(day)).strftime("%j") check_leoid='all' elif len(sys.argv)==4: #python kick_trigZero2one_star.py 1 2025-12-16 002 year,mm,day=sys.argv[2].split('-') dt = datetime(int(year), int(mm), int(day)) doy = dt.strftime("%j") #doy=datetime(int(year),int(mm),int(day)).strftime("%j") check_leoid=sys.argv[3] else: print('input arguments are incorrect, pls check!') print(msg) sys.exit(1) print(f'Processing year={year} doy={doy} leoid={check_leoid}') #path='/data3/jzhou128/GNSSRO_Archive/COSMIC2_L0_to_L1A/' #work_dir=os.path.join(path,'star_cosmic_GOX_TRIG_reader_1.24') work_dir=os.path.join('/data3/xinjiaz/gps_pod/TRIG_reader/','star_cosmic_GOX_TRIG_reader_1.26_rt') #cosmic2_L0='/GNSSRO_data2/xinjiaz/data_GPSRO/UCAR/cosmic2/level0/trgLv0/' #cosmic2_L0='/GNSSRO_data2/xinjiaz/data_GPSRO/TACC/level0/trgLv0/' #input_dir=os.path.join(cosmic2_L0,year,doy) cosmic2_L0='/data2/xinjiaz/data_GPSRO/cosmic2_nccf/level0/trgLv0/' input_dir=os.path.join(cosmic2_L0,year,mm,day) #output_dir=os.path.join('/data3/xinjiaz/data_GPSRO/','NCCF_unpack_STAR','cosmic2','v1.26_rt','level1a') output_dir=os.path.join('/data2/xinjiaz/data_GPSRO/','cosmic2_nccf','level1a') batch_process_obj=batch_process_class(year,doy,work_dir,input_dir,output_dir,check_leoid) batch_process_obj.decode_L0_L1A()