Home > atmlab > deprecated > collocation_read.m

collocation_read

PURPOSE ^

collocation_read Read collocations for indicated period

SYNOPSIS ^

function [M, cols_out, limmat, filters] = collocation_read(sat1, sensor1, sat2, sensor2,start_date, end_date, cols_in, limits, filters)

DESCRIPTION ^

 collocation_read Read collocations for indicated period

 This m-file reads all the collocations for the indicated period, between
 start_date and end_date, for the indicated collocations type (like
 collocations between CloudSat and MHS or between MHS and MHS).

 FORMAT

   [M, cols] = collocation_read(...
       sat1, sensor1, sat2, sensor2, ...
       start_date, end_date, cols_in[, limits[, filters]])

 IN

   sat1        string          primary satellite
   sensor1     string          sensor on satellite; passed on to reader
                               function which can still return data from
                               other sensors; for example,
                               cloudsat/cpr+noaa18/mhs can also return
                               data from amsua or hirs.
   sat2        string          secondary satellite
   sensor2     string          sensor on satellite
   start_date  3x1 array       Starting date for which to read collocations
   end_date    3x1 array       Ending date for which to read collocations
   cols_in     cell array      Column names. This is passed on to the
                               reader function.
   limits      structure       Describes acceptable ranges for the
                               different columns requested. Data outside
                               these ranges will not be returned. Note:
                               [-inf, inf] will still get rid of nans.
                               Note2: if limits=nan, no filtering is done
                               AT ALL.
   filters     cell array      Cell array of cell arrays. Each member of
                               the cell array is {@filter, {COL1, ...,
                               COLN}, {'foo', 'bar', ..., 'baz'}}.
                               The filter is called according to
                               filter(M(COL1, :), ..., M(COLN, :), 'foo',
                               'bar', ..., 'baz') and shall return logical
                               true wherever the filter is passed. See
                               examples on website (see below).

 OUT

   M           NxM matrix  Matrix containing collected info
   cols        structure   Structure describing the names of the columns

 For examples, see <a href="http://www.sat.ltu.se/docs/datasets/collocations/collocations_atmlab.php">online help on the satpage</a>

 $Id$

CROSS-REFERENCE INFORMATION ^

This function calls: This function is called by:

DOWNLOAD ^

collocation_read.m

SOURCE CODE ^

0001 function [M, cols_out, limmat, filters] = collocation_read(sat1, sensor1, sat2, sensor2, ...
0002     start_date, end_date, cols_in, limits, filters)
0003 
0004 % collocation_read Read collocations for indicated period
0005 %
0006 % This m-file reads all the collocations for the indicated period, between
0007 % start_date and end_date, for the indicated collocations type (like
0008 % collocations between CloudSat and MHS or between MHS and MHS).
0009 %
0010 % FORMAT
0011 %
0012 %   [M, cols] = collocation_read(...
0013 %       sat1, sensor1, sat2, sensor2, ...
0014 %       start_date, end_date, cols_in[, limits[, filters]])
0015 %
0016 % IN
0017 %
0018 %   sat1        string          primary satellite
0019 %   sensor1     string          sensor on satellite; passed on to reader
0020 %                               function which can still return data from
0021 %                               other sensors; for example,
0022 %                               cloudsat/cpr+noaa18/mhs can also return
0023 %                               data from amsua or hirs.
0024 %   sat2        string          secondary satellite
0025 %   sensor2     string          sensor on satellite
0026 %   start_date  3x1 array       Starting date for which to read collocations
0027 %   end_date    3x1 array       Ending date for which to read collocations
0028 %   cols_in     cell array      Column names. This is passed on to the
0029 %                               reader function.
0030 %   limits      structure       Describes acceptable ranges for the
0031 %                               different columns requested. Data outside
0032 %                               these ranges will not be returned. Note:
0033 %                               [-inf, inf] will still get rid of nans.
0034 %                               Note2: if limits=nan, no filtering is done
0035 %                               AT ALL.
0036 %   filters     cell array      Cell array of cell arrays. Each member of
0037 %                               the cell array is {@filter, {COL1, ...,
0038 %                               COLN}, {'foo', 'bar', ..., 'baz'}}.
0039 %                               The filter is called according to
0040 %                               filter(M(COL1, :), ..., M(COLN, :), 'foo',
0041 %                               'bar', ..., 'baz') and shall return logical
0042 %                               true wherever the filter is passed. See
0043 %                               examples on website (see below).
0044 %
0045 % OUT
0046 %
0047 %   M           NxM matrix  Matrix containing collected info
0048 %   cols        structure   Structure describing the names of the columns
0049 %
0050 % For examples, see <a href="http://www.sat.ltu.se/docs/datasets/collocations/collocations_atmlab.php">online help on the satpage</a>
0051 %
0052 % $Id$
0053 
0054 %% prepare configuration things
0055 
0056 no = number_sats_in_dataset(['collocation_' sensor1 '_' sensor2]);
0057 
0058 cols = colloc_constants(['cols_' sensor1 '_' sensor2]);
0059 
0060 if ~exist('filters', 'var')
0061     filters = {};
0062 end
0063 
0064 switch no
0065     case 1
0066         s = sat2;
0067     case 2
0068         s = {sat1, sat2};
0069 end
0070 
0071 %% check legacy format
0072 if iscell(cols_in{1})
0073     warning('atmlab:collocation_read', ...
0074         ['fields should be {''a'', ''b'', ''c'', ''d'', ...}, ' ...
0075          'not {{''a'', ''b'', ...}, {''c'', ''d'', ...}}']);
0076      cols_in = [cols_in{:}];
0077 end
0078 
0079 %% ALWAYS get rid of doubles
0080 % FIXME: at some point in the future, this should be redundant
0081 if isfield(cols.overlap, 'filter_double')
0082     filters = [filters ...
0083         {{@colloc_select_good_lines, cols.overlap.filter_double{1}, {sat1, sensor1}}, ...
0084         {@colloc_select_good_lines, cols.overlap.filter_double{2}, {sat2, sensor2}}}];
0085     % but only where filter_double has a value
0086     filters(cellfun(@(c) isempty(c{2}), filters)) = [];
0087     
0088     % make sure the fields are asked for
0089     for c = horzcat(cols.overlap.filter_double{:})
0090         % cannot use union here, because it wants a cellstr :(
0091         if ~any(cellfun(@(d) isequal(c, d), cols_in))
0092             cols_in = [cols_in c]; %#ok<AGROW>
0093         end
0094     end
0095 else
0096     logtext(atmlab('OUT'), 'Not filtering doubles, hopefully this was done when processing\n');
0097 end
0098 %% get name_struct
0099 
0100 name_struct = cols_cell_to_cols_struct(cols, cols_in);
0101 % ncols: highest value
0102 C = struct2cell(name_struct);
0103 ncols = max(horzcat(C{:}));
0104 
0105 %% convert limits structure to matrix as wanted by restrain_collocations
0106 
0107 dolimits = true;
0108 if exist('limits', 'var') 
0109     if isequalwithequalnans(limits, nan)
0110         dolimits = false;
0111         limmat = zeros(0, 3);
0112     else
0113         limmat = limstruct2limmat(limits, name_struct);
0114     end
0115 else
0116     limmat = zeros(0, 3);
0117 end
0118 
0119 % for the filters, convert column names to column numbers, using the
0120 % earlier obtained name_struct
0121 if ~isempty(filters)
0122     for i = 1:length(filters)
0123         filters{i}{2} = cellfun(@(s) name_struct.(s)', filters{i}{2}, 'UniformOutput', false);
0124     end
0125 end
0126 
0127 %% check if this can be done via hdf5 (much faster!)
0128 
0129 years = start_date(1):end_date(1);
0130 paths_hdf5 = cell(size(years));
0131 for i = 1:length(years)
0132     year = years(i);
0133     try
0134         paths_hdf5{i} = find_datafile_by_date([year 1 1], s, ['collocation_' sensor1 '_' sensor2 '_hdf5']);
0135         assert(~~exist(paths_hdf5{i}, 'file'), ...
0136             'atmlab:collocation_read', 'HDF5-file does not exist: %s', ...
0137             paths_hdf5{i});
0138     catch ME
0139         switch (ME.identifier)
0140             case {'atmlab:find_datadir_by_date', 'atmlab:collocation_read', 'atmlab:find_granules_by_date', 'atmlab:input:undefined'}
0141                 if iscell(s)
0142                     s_p = horzcat(s{:});
0143                 else
0144                     s_p = s;
0145                 end
0146                 logtext(atmlab('OUT'), ...
0147                     'Note: no HDF5 @ %d %s %s %s\n', ...
0148                     year, s_p, sensor1, sensor2)
0149                 continue
0150             otherwise
0151                 ME.rethrow()
0152         end
0153     end
0154 end
0155 
0156 % if any(cellfun(@length, paths_hdf5))
0157 %     %% do this stuff via HDF5
0158 %     if ~all(cellfun(@length, C))
0159 %         error('atmlab:collocation_read', ...
0160 %             'Period PARTIALLY covered by HDF-5, this is not supported yet');
0161 %     end
0162 %     logtext(atmlab('OUT'), 'Found HDF5\n');
0163 %     % Convert limits-matrix to limits-strings
0164 %     baselimstrs = {'', ''};
0165 %     fieldnames_both = {union(fieldnames(cols.overlap), fieldnames(cols.data)), fieldnames(cols.meandata)};
0166 %     for i = 1:2
0167 %
0168 %         limcellstr = cellfun(@(v) ...
0169 %             ['(' v ' >= ' num2str(limits.(v)(1)) ') & (' ...
0170 %             v ' <= ' num2str(limits.(v)(2)) ') & '], ...
0171 %             intersect(fieldnames_both{i}, fieldnames(limits)), ...
0172 %             'UniformOutput', false);
0173 %         baselimstrs{i} = horzcat(limcellstr{:});
0174 %     end
0175 %     for i = 1:length(years)
0176 %         year = years(i);
0177 %         hdf5file = paths_hdf5{i};
0178 %         % and limit dates
0179 %         global_start = date2unixsecs(start_date(1), start_date(2), start_date(3));
0180 %         this_year_start = date2unixsecs(year, 1, 1);
0181 %         this_start = max(global_start, this_year_start);
0182 %         global_end = date2unixsecs(end_date(1), end_date(2), end_date(3));
0183 %         this_year_end = date2unixsecs(year, 12, 31, 24);
0184 %         this_end = min(global_end, this_year_end);
0185 %         % add datelimits to limstr
0186 %         limstr = [baselimstrs{1} sprintf(' (B_TIME >= %d) & (B_TIME <= %d)', ...
0187 %             this_start, this_end) ' MEANSEP ' baselimstrs{2}];
0188 %         limstr = deblank(limstr);
0189 %         if strcmp(limstr(end), '&')
0190 %             limstr = deblank(limstr(1:end-2));
0191 %         end
0192 %         M = collocation_read_frompipe(hdf5file, cols_in, limstr);
0193 %         cols_out = name_struct;
0194 %         return
0195 %     end
0196 % end
0197 
0198 %% pre-allocate at least zero rows
0199 
0200 M = zeros(0, ncols);
0201 
0202 %% loop through all the dates
0203 
0204 dates = daterange(start_date, end_date);
0205 for i = 1:size(dates, 1)
0206     date = dates(i, :);
0207     %% read collocations for date
0208 
0209     path = find_datafile_by_date(date, s, ...
0210         ['collocation_' sensor1 '_' sensor2]);
0211     
0212     try
0213         collocations_day = read_collocs_data_mean(path, cols_in, cols);
0214 
0215     catch ME
0216         switch (ME.identifier)
0217             case {'MATLAB:load:couldNotReadFile', 'MATLAB:nonExistentField', ...
0218                     'MATLAB:gunzip:invalidFilename','MATLAB:netcdf:open:noSuchFile', ...
0219                     'atmlab:exec_system_cmd:shell'}
0220                 logtext(atmlab('ERR'), 'Problem for %04d-%02d-%02d: %s\n', ...
0221                     date(1), date(2), date(3), ME.message);
0222                 continue
0223             otherwise
0224                 ME.rethrow();
0225         end
0226     end
0227     if isempty(collocations_day)
0228         logtext(atmlab('OUT'), 'no collocations\n');
0229         continue
0230     end
0231     
0232     %% apply limitations
0233     if dolimits
0234         lim = collocation_restrain(collocations_day, limmat, filters);
0235         collocations_day = collocations_day(lim, :);
0236     end
0237     
0238     %% add to total
0239     L = size(M, 1);
0240     N = size(collocations_day, 1);
0241     M((L+1):(L+N), :) = collocations_day;
0242 end
0243 
0244 cols_out = name_struct;

Generated on Mon 15-Sep-2014 13:31:28 by m2html © 2005