; Comments added by Erik Johnson 7/3/07
; cleaner.pro, compiles and runs with idl
; needs strip.pro to run
; input must be raw MACCS data file(s)
; output is a cleaned binary version of the raw data with filetype .s2
; example: if input file is CD05.4, output file is CD05004.s2
; to compile & run: cleaner, "path_and_name_of_raw_data_file(s)" (you can use * and ?)
; default output directory is e:\
; to set output directory: out_dir="output_directory"
; to display a window with a graph: view="view"
; to create summary text file(s) containg the command line output: summary="summary"
; to save the .s2 file(s) in a subdirectory folder (which must already exist): month_str="sub_dir_name"
; to move the raw file(s) after creating the clean file(s): move="path" (if you used the month_str parameter, the file(s) will be moved to the subdirectory within the folder you specified for the move directory) Ex: if parameters are month_str="Jan05/", move="/Volumes/MACCS_DATA/" then raw file(s) will be moved to /Volumes/MACCS_DATA/Jan05/
; the click parameter should not be used unless changes are made to the program
; Example: cleaner, "/Volumes/physics_data/Ftp/MACCS_DATA/Raw/CD/2005/CD05.4", out_dir="/Users/johnson8/maccs/", summary="summary", month_str="Jan05/", move="/Users/johnson8/Desktop/", view="view"
pro cleaner,search,view=view,summary=summary,month_str=month_str,$
calc_mon=calc_mon,out_dir=out_dir,click=click,move=move
!p.multi=[0,0,4]
if not keyword_set(month_str) then month_str =
flagged_record = byte([32767000L,32767000L,32767000L,32767000L,32767000L,32767000L],0,1,24)
x = 35 ;bytes
clean_data = bytarr(28,86400)
raw_files = findfile(search,count=num_raw)
for i = 0, n_elements(raw_files)-1 do begin ; Loop over number of files to be processed
if keyword_set(view) then window,0,xsize=600,ysize=700
for ii = 0L,86399L do clean_data(0,ii) = [0B,0B,0B,0B,reform(flagged_record)]
print,'done'
print,raw_files(i)
dot_pos = strpos(raw_files(i),'.')
date = string(fix(strmid(raw_files(i),dot_pos+1,3)),format='(i3.3)')
if keyword_set(calc_mon) then date_var,yrday=fix(strmid(raw_files(i),dot_pos-2,2)+date),month_str=month_str
if not keyword_set(out_dir) then out_dir = 'e:\'
; OLD clean_file = out_dir+month_str+'\'+strmid(raw_files(i),dot_pos-4,4)+date+'.s2'
clean_file = FILEPATH(strmid(raw_files(i),dot_pos-4,4)+date+'.s2', ROOT_DIR = out_dir, SUBDIR = [month_str])
; OLD text_file = out_dir+month_str+'\'+strmid(raw_files(i),dot_pos-4,4)+date+'.txt'
text_file = FILEPATH(strmid(raw_files(i),dot_pos-4,4)+date+'.txt', ROOT_DIR = out_dir, SUBDIR = [month_str])
print,clean_file,text_file
; Open "summary" text file is summary keyword is selected
if keyword_set(summary) then openw,summary_file,text_file,/get_lun
; Open and read raw data file
openr, unit, raw_files(i), error=err, /get_lun
if err eq 0 then begin
file_info = fstat(unit)
print,file_info.isatty
y = file_info.size / x
PRINT, 'Y ', y
print,raw_files(i)+' has '+strip(y)+' records.'
if keyword_set(summary) then printf,summary_file,raw_files(i)+' has '+strip(y)+' records.'
temp = assoc(unit, bytarr(x, y), 0)
raw_data = temp(0)
close,unit
free_lun, unit
endif else begin
print,'An error has occured reading '+raw_files(i)
stop
endelse
good_time = where((raw_data(30,*) ne 0) and (raw_data(31,*) ne 1),n_good_time)
no_time = where((raw_data(30,*) eq 0) and (raw_data(31,*) eq 1),n_no_time)
print,n_no_time,' records without time.'
if n_no_time gt 0 then begin
raw_data = raw_data(*,good_time)
y = n_good_time
endif
; Read data into arrays
header= raw_data( 0,*)
dd = fix (raw_data( 1: 2,*),0,1,y)
hh = raw_data( 3,*)
mm = raw_data( 4,*)
ss = raw_data( 5,*)
bx1 = long(raw_data( 6: 9,*),0,1,y)/1000.
by1 = long(raw_data(10:13,*),0,1,y)/1000.
bz1 = long(raw_data(14:17,*),0,1,y)/1000.
bx2 = long(raw_data(18:21,*),0,1,y)/1000.
by2 = long(raw_data(22:25,*),0,1,y)/1000.
bz2 = long(raw_data(26:29,*),0,1,y)/1000.
tf1 = raw_data(30,*)
tf2 = raw_data(31,*)
block = raw_data(32,*)
status= raw_data(33,*)
house = raw_data(34,*)
; Plot if keyword "view" is selected
if keyword_set(view) then begin
; window,1,xsize=600,ysize=600
seconds = hh*3600L + mm*60L + ss
diff = seconds(1:n_elements(seconds)-1) - seconds(0:n_elements(seconds)-2)
backward = where(diff lt 0,count)
plot,seconds,/ystyle,title=raw_files(i)
plot,bx1,max_val=median(bx1)+500,min_val=median(bx1)-500,/ystyle
plot,by1,max_val=median(by1)+500,min_val=median(by1)-500,/ystyle
plot,bz1,max_val=median(bz1)+500,min_val=median(bz1)-500,/ystyle
; if (keyword_set(click) and count gt 0) then cursor,cx,cy,/wait,/up,/norm
; print,cx,cy
; if cy lt .5 then goto,skip
; print,'noskip',cy
endif
; Check day
same_day = where(((dd gt 0) and (dd le 366)) and (dd ne dd(0)) and (tf1 eq 49) and (tf2 eq 48),count)
if count ne 0 then begin
print,raw_files(i)+' has more than one day.'
; for jk=0,count-1 do begin
; print,same_day(jk)
; for kk = same_day(jk)-2, same_day(jk)+2 do $
; print,header(kk),dd(kk),hh(kk),mm(kk),ss(kk),tf1(kk),tf2(kk),block(kk)
; endfor
stop
endif
; Find Gaps
; flagged = where((tf1 ne 49) or (tf2 ne 57),n_flagged)
; good = where((tf1 eq 49) and (tf2 eq 57),n_good)
flagged = where((tf1 ne 49) or (tf2 ne 48),n_flagged)
good = where((tf1 eq 49) and (tf2 eq 48),n_good)
; if n_good eq 0 then begin
; print,'n_good',n_good
; goto,skip
; endif
delta_good = good(1:n_good-1) - good(0:n_good-2)
good_gaps = where(delta_good ne 1,n_good_gap)
seconds = hh * 3600. + mm * 60. + ss
seconds = reform(seconds)
; Find first good record
print,'first good time at ',hh(good(0)),':',mm(good(0)),':',ss(good(0))
if keyword_set(summary) then printf,summary_file,'first good time at ',hh(good(0)),':',mm(good(0)),':',ss(good(0))
clean_data(0,0:seconds(good(0))) = 2b
for ii = 0L,seconds(good(0)) do clean_data(04:27,ii) = flagged_record
; Write good data into clean_data
clean_time = findgen(86400)
clean_data(0,seconds(good)) = 0b
clean_data(1,*) = byte(floor(clean_time/3600.))
clean_data(2,*) = byte(floor((clean_time-clean_data(1,*)*3600.)/60.))
clean_data(3,*) = byte(clean_time-clean_data(1,*)*3600.-clean_data(2,*)*60.)
clean_data(04:27,seconds(good)) = raw_data(06:29,good)
; Fix Gaps
for jj = 0L,n_good_gap-1 do begin
gap_start = good(good_gaps(jj))
gap_stop = good(good_gaps(jj)+1)
gap_delta_t = seconds(gap_stop) - seconds(gap_start)
gap_delta_b = block(gap_stop) - block(gap_start)
gap_length = gap_stop-gap_start
print,'gap ',strip(jj),' at ',hh(gap_start),':',mm(gap_start),':',ss(gap_start),$
' has ',strip(gap_delta_t),' seconds and ',strip(gap_length),' data points'
if keyword_set(summary) then printf,summary_file,'gap ',strip(jj),' at ',hh(gap_start),$
':',mm(gap_start),':',ss(gap_start),' has ',strip(gap_delta_t),' seconds and ',strip(gap_length),' data points'
; for kk = gap_start, gap_stop do begin
; print,dd(kk),hh(kk),mm(kk),ss(kk),bz1(kk),block(kk),tf1(kk),tf2(kk)
; endfor
if (gap_length eq gap_delta_t) and (gap_length eq gap_delta_b) then begin
clean_data(0,seconds(gap_start):seconds(gap_stop)) = 1b
clean_data(04:27,seconds(gap_start):seconds(gap_stop)) = raw_data(06:29,gap_start:gap_stop)
if gap_delta_t gt 3600 then stop
endif else begin
if gap_delta_t gt 5 then begin
clean_data(0,seconds(gap_start):seconds(gap_stop)) = 2b
for ii = seconds(gap_start),seconds(gap_stop-1) do clean_data(04:27,ii) = flagged_record
gap_flagged = where((flagged gt gap_start) and (flagged lt gap_stop), n_gap_flagged)
print,'* ',strip(n_gap_flagged),' flagged values in gap.'
if keyword_set(summary) then printf,summary_file,'* ',strip(n_gap_flagged),' flagged values in gap.'
if n_gap_flagged/gap_length gt 0.5 then stop
endif else begin
; clean_data(0,seconds(gap_start):seconds(gap_stop-1)) = 2b
clean_data(0,seconds(gap_start):seconds(gap_stop)) = 2b
for ii = seconds(gap_start),seconds(gap_stop-1) do clean_data(04:27,ii) = flagged_record
endelse
endelse
endfor
; Switch from bx1,by1,bz1,bx2,by2,bz2 to bx1,bx2,by1,by2,bz1,bz2
old_bytes = [04,05,06,07,16,17,18,19,08,09,10,11,20,21,22,23,12,13,14,15,24,25,26,27]
clean_data(04:27,*) = clean_data(old_bytes,*)
; Write clean data to file
openw,unit,clean_file,/get_lun
writeu,unit,clean_data
close,unit
free_lun,unit
if keyword_set(move) then begin
print,'move '+raw_files(i)+' '+move+month_str+'\'+strmid(raw_files(i),dot_pos-4,8)
spawn,'move '+raw_files(i)+' '+move+month_str+'\'+strmid(raw_files(i),dot_pos-4,8)
; wait,5
endif
; Close "summary" text file
; skip:
if keyword_set(summary) then close,summary_file
if keyword_set(summary) then free_lun,summary_file
endfor
!p.multi=0
end
