collocation_read_frompipe

 collocation_read_frompipe Low-level, read collocations as client to pytables server

 EXPERIMENTAL! Function in α/β and subject to change.

 FORMAT

 IN

   hdf5file    string      full, absolute path to hdf5file. The absolute
                           path is required for the correct calculation
                           of the path to the lockfile.
   fields      cellstr
   query


 TODO:
   - select particular columns from AMSU/MHS etc.
   - read meandata and avhrrdata correctly
   - convert limmat to query-string
   - do not-exactly-one-year correctly
   - handle errors
   - have a better way to treat path_to_server

 $Id: collocation_read_frompipe.m 7553 2012-04-27 19:08:16Z gerrit $

0001 function M = collocation_read_frompipe(hdf5file, fields, query)
0002 
0003 % collocation_read_frompipe Low-level, read collocations as client to pytables server
0004 %
0005 % EXPERIMENTAL! Function in α/β and subject to change.
0006 %
0007 % FORMAT
0008 %
0009 % IN
0010 %
0011 %   hdf5file    string      full, absolute path to hdf5file. The absolute
0012 %                           path is required for the correct calculation
0013 %                           of the path to the lockfile.
0014 %   fields      cellstr
0015 %   query
0016 %
0017 %
0018 % TODO:
0019 %   - select particular columns from AMSU/MHS etc.
0020 %   - read meandata and avhrrdata correctly
0021 %   - convert limmat to query-string
0022 %   - do not-exactly-one-year correctly
0023 %   - handle errors
0024 %   - have a better way to treat path_to_server
0025 %
0026 % $Id: collocation_read_frompipe.m 7553 2012-04-27 19:08:16Z gerrit $
0027 
0028 %% initialise
0029 path_to_server = '/storage4/home/gerrit/svn/sources/python/scripts/collocation_read.py';
0030 
0031 % note: make sure this matches Pythons calculations
0032 lockfile = fullfile('/tmp', strrep(hdf5file, '/', '_'));
0033 % convert fields cellstr to fields "A B C"-string
0034 fields = cellfun(@(c) [c ' '], fields, 'UniformOutput', false);
0035 fields = horzcat(fields{:});
0036 fields = deblank(fields);
0037 scan_format = '%g';
0038 
0039 if ~exist(lockfile, 'file') % Start server
0040     logtext(atmlab('OUT'), 'Initialising program\n');
0041     c = num2str(round(100*rand));
0042     pipe_in_name = fullfile(atmlab('WORK_AREA'), ['pipereader' strrep(hdf5file, '/', '_') c]);
0043     pipe_out_name = fullfile(atmlab('WORK_AREA'), ['pipewriter' strrep(hdf5file, '/', '_') c]);
0044     log_name = fullfile(atmlab('WORK_AREA'), ['pipelogger' strrep(hdf5file, '/', '_') c]);
0045 
0046     command = sprintf([path_to_server ' -vd -i %s -o %s -f %s -l %s'], ...
0047         pipe_in_name, pipe_out_name, hdf5file, log_name);
0048     system(command);
0049 end
0050 
0051 %% open files
0052 wait_for_existence(lockfile, 2, 0.1);
0053 f = fileopen(lockfile, 'r');
0054 [~] = fgets(f); % first line is pid
0055 pipe_in_name = deblank(fgets(f));
0056 pipe_out_name = deblank(fgetl(f));
0057 log_name = deblank(fgetl(f));
0058 
0059 wait_for_existence(pipe_out_name, 2, 0.1);
0060 pipe_out = fileopen(pipe_out_name, 'r');
0061 wait_for_existence(log_name, 2, 0.1);
0062 logfile = fileopen(log_name, 'r');
0063 fseek(logfile, -1, 'eof');
0064 wait_for_existence(pipe_in_name, 2, 0.1);
0065 pipe_in = fileopen(pipe_in_name, 'w');
0066 
0067 %% communicate
0068 logtext(1, 'Sending query: %s ''FIELDS'' %s\n', query, fields);
0069 % send command to program
0070 fprintf(pipe_in, [query ' FIELDS ' fields '\n']);
0071 % read output until '.'
0072 i = 0;
0073 while true
0074     i = i + 1;
0075     if mod(i, 100) == 0
0076         % do a 'tail -f' of the logfile
0077         fwrite(1, fread(logfile, inf, '*char'));
0078     end
0079     line = fgets(pipe_out);
0080     if strcmp(deblank(line), '.')
0081         break
0082     else
0083         data_in_line = sscanf(line, scan_format);
0084         if i == 1
0085             data_total = data_in_line.';
0086         else
0087             data_total(i, :) = data_in_line;
0088         end
0089         if i == size(data_total, 1) % pre-allocate some more
0090             data_total = [data_total; nan(size(data_total))]; %#ok<AGROW>
0091             logtext(1, 'Doubling allocation to %d lines\n', size(data_total, 1))
0092         end
0093     end
0094 end
0095 fwrite(1, fread(logfile, inf, '*char'));
0096 %% finalise
0097 
0098 M = data_total(~isnan(data_total(:, 1)), :);
0099 
0100 end