Viewing contents of file '../idllib/user_contrib/knight/ddread.pro'
;+
; Name:
; ddread
; Purpose:
; This routine reads data in formatted (or unformatted) rows and columns.
; The name stands for Data Dump Read. By default, comments are
; skipped and the number of columns is sensed. Many options
; exist, e.g., selecting rows and columns, reading binary data,
; and selecting non-default data type and delimiters.
;
; Examples:
; junk = ddread(/help) ; get information only
; array = ddread(file) ; read ASCII data
; array = ddread(file,/formatted) ; ditto
; array = ddread(file,object=object) ; read binary data
; array = ddread(file,columns=[0,3]) ; get only 1st & 4th columns
; array = ddread(file,rows=lindgen(10)+10); get only 2nd 10 rows
; array = ddread(file,offset=10,last=19) ; get rows (10,19)
; array = ddread(file,/countall) ; count comment lines
; array = ddread(file,/verbose) ; echo comment lines
; array = ddread(file,type=1) ; return bytes, not floats or longs
; array = ddread(file,range=['start text','stop text']) ; text delimiters
;
; ; Place the detailed output from a Lowtran run in a 2-D array---wow!
; output = ddread('lowtran.out',range=['(CM-1) (MICRN)','0INTEGRATED ABSORPTION'])
; % DDREAD: Read 69 data lines selecting 14 of 14 columns; skipped 395 comment lines.
; Usage:
; array = ddread([file][,options][,/help])
; Optional Inputs:
; file = file with data; if omitted, then call pickfile.
; Keywords:
; /formatted, /unformatted = flags to tell IDL whether data format is
; binary or ASCII. ddread tries to determine the type
; of data but it's not foolproof.
; object = a string containing the IDL declaration for one instance
; of the object in an unformatted file, e.g.,
; 'fltarr(4)'
; or
; '{struct,dwell:0.,pitch:0.,yaw:0.,roll:0.}'
; rows = an array to select a subset of the rows in a formatted file
; Does not count comment lines, unless /countallrows is set!
; columns = likewise for columns
; type = data type of the output D=float (if '.' appears) or long
; delimiter = column separater, D=whitespace
; /help = flag to print header
; range = start and stop row or strings,
; e.g. range = ['substring in 1st line','substring in last line']
; offset = start row (read to end of file, unless last set)
; last = stop row (read from start of file, unless offset set)
; /countallrows = flag to count comment rows as well as data rows (D=0)
; /verbose = flag to echo comments to screen
; Outputs:
; array = array of data from the lines (ASCII) or objects (binary)
; Common blocks:
; none
; Procedure:
; After deciding on ASCII or binary, read file and return array.
;
; Restrictions:
; - Comments can be either an entire line or else an end of a line, e.g.,
; /* C comment. */
; ; IDL comment
; Arbitrary text as a comment
; Comment in Fortran
; The next line establishes # of columns (4) & data type (float):
; 6. 7 8 9
; This line and the next are both considered comments.
; 6 comment because only one of 4 columns appears
; 1 2 3 4 but this line has valid data and will be read as data
;
; - Even if a range of lines is selected with offset, range or last, all
; lines are read. This could be avoided.
;
; - Other routines needed:
; pickfile.pro - to choose file if none is given
; nlines.pro - to count lines in a file
; nbytes.pro - to count bytes in a variable
; replicas.pro - to replicate arrays (not scalars as in replicate.pro)
; typeof.pro - to obtain the type of a variable
;
; Modification history:
; write, 22-26 Feb 92, F.K.Knight (knight@ll.mit.edu)
; allow reading with arbitrary delimiter using reads, 23 Mar 92, FKK
; add countallrows keyword and modify loop to read as little
; data as possible, 20 May 92, FKK
; correct bug if /formatted set, 6 Jul 92, FKK
; add verbose keyword to print comments, 6 July 92, FKK
; correct bug if /rows=...,/countall set, 6 July 92, FKK & EJA
; add a guard against a blank line being converted to a
; number, 21 Aug 92, FKK
; allow parital line just before the EOF. Possibly this isn't the
; right thing to do, but I decided to allow it. If the final line
; is incomplete, the values are still read and the remainder of
; the line is filled with zeroes. 26 Oct 92, FKK
; allow range keyword to be a string array, 2 Dec 92, FKK
; make default for countallrows be true if range is present, 2 Dec 92, FKK
; add new function (typeof); called in a few places, 2 Dec 92, FKK
;-
function ddread,help=help,file,formatted=formatted,unformatted=unformatted $
,rows=rows,columns=columns,object=object,type=type,delimiter=delimiter $
,offset=offset,last=last,range=range,countallrows=countallrows $
,verbose=verbose
;
; =====>> HELP
;
on_error,2
if keyword_set(help) then begin & doc_library,'ddread' & return,0 & endif
;
; =====>> SETUP
;
on_ioerror,IOERROR
if n_elements(file) eq 0 then file = pickfile()
if n_elements(countallrows) eq 0 then countallrows = n_elements(range) gt 0
gotobject = n_elements(object) gt 0
if keyword_set(formatted) then begin
if gotobject then message,/inform,'You should not specify an object for formatted data. Try tmp=ddread(/help). Ignoring object.'
goto,ASCII
endif
if gotobject then begin
status = execute('obj = '+object)
if not status then message,'Your object '+object+' is incorrect.'
goto,BINARY
endif
if keyword_set(unformatted) and (not gotobject) then message,'For unformatted dump, you need to supply an object. Try tmp=ddread(/help).'
goto,ASCII ; The default
;
; =====>> READ UNFORMATTED DATA USING OBJ
;
BINARY:
openr,lun,file,/get_lun
stat = fstat(lun)
nb = nbytes(obj) ; # of bytes in one object
all = stat.size/nb ; the entire file
if n_elements(offset) eq 0 then offset = 0 ; user-defined offset
if n_elements(last) eq 0 then last = all - 1 ; user-defined end
if n_elements(range) eq 2 then begin ; range takes precedence
if typeof(range) ge 7 then message,'For binary data, range keyword must have 2 numbers.'
offset = range(0)
last = range(1)
endif
array = replicas(obj,last-offset+1) ; output array
point_lun,lun,nb*offset ; locate file ptr at offset
readu,lun,array
close,lun
free_lun,lun
return,array
;
; =====>> OPEN FILE FOR FORMATTED DATA
;
ASCII:
nlines = nlines(file) ; COUNT LINES IN FILE
openr,lun,file,/get_lun
;
; =====>> SET THE LIMITS OF THE READ IF POSSIBLE (countallrows=1)
;
lbegin = 0 ; DEFAULT START
lend = nlines-1 ; DEFAULT END
if countallrows then begin
if n_elements(range) eq 2 then begin
if typeof(range) eq 7 then begin ; IF STRING, SEARCH FILE.
point_lun,lun,0 ; GO TO START OF FILE
count = 0 ; LINE COUNTER
rdbuf = ' '
while not eof(lun) do begin ; LOOP & SET lbegin & lend
readf,lun,rdbuf
if (strpos(rdbuf,range(0)) ge 0) and (lbegin eq 0) then begin
lbegin = count
stat = fstat(lun)
data_start = stat.cur_ptr ; SAVE PTR TO START OF DATA
endif
if (strpos(rdbuf,range(1)) ge 0) then lend = count
count = count + 1
endwhile
endif else begin
lbegin = range(0)
lend = range(1)
endelse
endif
if n_elements(rows) gt 0 then begin
lbegin = min(rows)
lend = max(rows)
endif
if n_elements(offset) eq 1 then lbegin = offset
if n_elements(last) eq 1 then lend = last
endif
;
; =====>> SKIP TO START OF DATA
;
if typeof(data_start) eq 0 then begin ; IF UNDEFINED, SKIP LINES.
point_lun,lun,0 ; GO TO START OF FILE
rdbuf = ' '
if lbegin gt 0 then begin ; READ TO START OF DATA
for i=0,lbegin-1 do readf,lun,rdbuf
endif
stat = fstat(lun)
data_start = stat.cur_ptr ; SAVE PTR TO START OF DATA
endif
point_lun,lun,data_start ; GO TO START OF DATA
;
; =====>> SETUP FOR READING FORMATTED DATA
;
on_ioerror,COMMENT ; READY TO SKIP COMMENTS
ncols = 0 ; ASSUME NO COLUMNS
tmp = ''
COMMENT: ; FIND A VALID LINE-ONE STARTING WITH
readf,lun,tmp ; A VALID FLOAT
if n_elements(delimiter) eq 1 then $ ; CHANGE DELIMITER TO SPACE
while (((i = strpos(tmp,delimiter))) ne -1) do strput,tmp,' ',i
if strlen(tmp) eq 0 then goto,COMMENT
line = float(tmp)
tmp = ' ' + strcompress(strtrim(tmp,2)) ; KILL EXCESS WHITESPACE
for i = 0,strlen(tmp)-1 do $ ; LOOP OVER CHARACTERS
ncols = ncols + (strpos(tmp,' ',i) eq i) ; COUNT SPACES: ONE/COLUMN
if ncols eq 0 then message,'No valid lines in file.'
if n_elements(columns) eq 0 then columns = lindgen(ncols); SELECT COLUMNS
ncolumns = n_elements(columns)
if n_elements(type) eq 0 then $
if strpos(tmp,'.') ge 0 then type = 4 else type = 3 ; USE FLOATS OR LONGS.
;
; =====>> DEFINE DATA ARRAYS FOR READING AND OUTPUT
;
nrows = lend - lbegin + 1
array = make_array(size=[2,ncolumns,nrows,type,ncolumns*nrows])
tmp = make_array(size=[1,ncols,type,ncols]) ; READ BUFFER
linectr = lonarr(nrows) ; 1(0) = DATA(COMMENT) LINE
;
; =====>> READ FORMATTED DATA.
; =====>> IF DELIMITER SPECIFIED, THEN CHANGE EACH DELIMITER TO BLANK.
; =====>> TO AVOID OVERHEAD OF TRANSLATING DELIMITER, MAKE TWO LOOPS: ONE
; =====>> FOR WHITESPACE AND THE OTHER FOR USER-SPECIFIED DELIMITER
;
point_lun,lun,data_start ; RETURN TO START OF DATA
if n_elements(delimiter) eq 1 then begin ; HERE FOR USER-SPECIFIED DELIMITER
on_ioerror, SKIP1
for l = 0,nrows-1 do begin ; LOOP OVER LINES
readf,lun,rdbuf ; READ AS ASCII
while (((i = strpos(rdbuf,delimiter))) ne -1) do strput,rdbuf,' ',i
rdbuf = strcompress(strtrim(rdbuf,2)) ; KILL EXCESS WHITESPACE
reads,rdbuf,tmp ; CONVERT FROM STRING TO NUMBER
array(0,l) = tmp(columns)
linectr(l) = 1
goto,ENDLOOP1
SKIP1:
if keyword_set(verbose) then message,/inform,rdbuf
if eof(lun) and (strlen(rdbuf) gt 0) then begin ; HANDLE PARTIAL LINE BEFORE EOF
tmp(*) = 0 ; FILL FIRST
on_ioerror, GOTEOF1
reads,rdbuf,tmp
GOTEOF1:
array(0,l) = tmp(columns)
linectr(l) = 1
endif
ENDLOOP1:
endfor
endif else begin
on_ioerror, SKIP0
for l = 0,nrows-1 do begin ; LOOP OVER LINES
readf,lun,rdbuf
reads,rdbuf,tmp ; CONVERT FROM STRING TO NUMBER
array(0,l) = tmp(columns)
linectr(l) = 1
goto,ENDLOOP0
SKIP0:
if keyword_set(verbose) then message,/inform,rdbuf
if eof(lun) and (strlen(rdbuf) gt 0) then begin ; HANDLE PARTIAL LINE BEFORE EOF
tmp(*) = 0 ; FILL FIRST
on_ioerror, GOTEOF0
reads,rdbuf,tmp
GOTEOF0:
array(0,l) = tmp(columns)
linectr(l) = 1
endif
ENDLOOP0:
endfor
endelse
nl = long(total(linectr)) ; # OF DATA LINES
ns = long(nrows - nl) ; # OF COMMENT LINES
message,/inform,'Read '+strtrim(nl,2)+' data lines selecting '+strtrim(n_elements(columns),2) +' of '+strtrim(ncols,2) $
+ ' columns; skipped '+strtrim(ns,2)+' comment lines.'
close,lun
free_lun,lun
;
; =====>> SELECT LINES FROM FORMATTED FILE
;
if nl eq 0 then return,0
if keyword_set(countallrows) then begin
if n_elements(rows) eq 0 then begin ; SELECT LINES
return,array(*,where(linectr))
endif else begin
return,array(*,where(linectr(rows-lbegin)))
endelse
endif else begin
if n_elements(rows) eq 0 then rows = lindgen(nl) ; SELECT DATA LINES
array = array(*,where(linectr))
return,array(*,rows)
endelse
;
; =====>> I/O error
;
IOERROR:
close,lun
free_lun,lun
return,0
end