function [points,endTimes] = readLoadHistories(mode,logFileTLD,scanFileTLD,expStart,expEnd,scanDataType)
%points = readLoadHistories(mode, logFileTLD, scanFileTLD, expStart, expEnd, scanDataType)
%==========================================================================
% Read and plot load histories for APS testing
% Piece together load histories from scans and log files to get a
% complete(ish) load-displacement curve (or stress-strain). "Ish" because
% sensor errors (lack of readings) are removed). In the end, this should
% report any missing log files, but just skip missing single points.
%
% --- INPUT ---
% expStart - int, starting experiment number (the initial scan)
% expEnd - int, final experiment number, not including the fracture surface
%          scans
% logFileTLD - string, top level directory for the log files; assume that the
%              log files for all experiments are stored in one folder, and
%              each is held in folder a folder and named
%              "Exp###_loading_history/load_history.log". This is the path
%              to the folder that contains all the experiment folders.
% scanFileTLD - sting, top level directory for scan files; assume all
%               experiments are in one folder, in subfolders named
%               "Exp###..." which contain one file: "proj_####.hdf. This is
%               the path to the folder that contains all the experiment folders
% mode - string, the type of test, discrete or continuous; data needs to be
%        collected differently depending on the mode. For mode ==
%        'continuous' the expEnd value is ignored.
% scanDataType - string either 'none','hdf', or 'mat' depending on the type
%                of scan data to look for in the scanFileTLD.
%
% --- OUTPUT ---
% points - a 1xn struct with an element for each datapoint, and 3 fields:
%          time of aquisition, displacement at that time, and load at that time.
%          Accessing "points(1).load" will be the very first load data point
%          take for that specimen.
% Output formats: points.disp - double
%                 pounts.load - double
%                 points.time - integer; posix time
%                                        (seconds from 1 Jan 1970 00:00:00)
% -------------------------------------------------------------------------
% History - Orion L. Kafka
% 11-23-2017:11-29-2017 - written
% 11-29-2017:12-06-2017 - added code to handle June 2017 data
% 12-06-2017:12-13-2017 - added scan file timestamp handling and code for
%                         different types of scan file data
%
%==========================================================================

% Debug - for running as a script
if nargin == 0
    
    clear all; close all; clc
    
    mode = 'discrete';
    scanDataType='mat';
    
    % Exp start num
    expStart = 177;
    
    % Exp end num (not including the scans of fractured surfaces)
    expEnd =181;
    
    % Prefix for path to log files
    logFileTLD = 'C:\Users\ender\Desktop\Northwestern\Research\APS\loadDataProcessing\data\2017_06\AllLoadHistories_2017_06';
    
    % Prefix for path to scan files
    scanFileTLD = 'C:\Users\ender\Desktop\Northwestern\Research\APS\loadDataProcessing\data\2017_06\scanFileData';
    
end

switch lower(mode) % make sure it's case insensitive
    
    case 'discrete'
        pointStart = 1;
        for expNum = expStart:expEnd
            if expNum ~= expStart
                % Open the exp log file
                logFile = fullfile(logFileTLD,...
                    ['Exp',padWithZeros(3,expNum),'_loading_history'],'load_history.log');
                
                %debug
                disp(['Exp num is: ',num2str(expNum)])
                disp(['Log file is: ',logFile]);
                
                if exist(logFile,'file') == 2
                    logData = importdata(logFile,' ');
                    % Find all the time stamps, these start each record
                    if strcmp(logData.textdata{1,5},'CST')
                        dataPointInd = find(strcmp(logData.textdata(:,5),'CST'));
                    elseif strcmp(logData.textdata{1,5},'CDT')
                        dataPointInd = find(strcmp(logData.textdata(:,5),'CDT'));
                    else
                        disp('Not a valid time stamp')
                    end
                    
                    % COUNTERS and iterators:
                    % exp - current experiment iterator
                    % count2 - a loop counter within each exp
                    % ii - running count over all exps
                    % ind - current index within the exp (iterator)
                    % pointStart - running count of datapoints - increments through all
                    % datapoints taken throuhgout the specimen (note the
                    % distinction between specimen and experiment)
                    % dataPointInd - list (array) of all known time stamps
                    % for the experiment
                    
                    % the sensor might not read/report a value sometimes, some we need
                    % to make the data-reading routine robust to that. This should mark
                    % any missing from either the load sensor or the crosshead with
                    % "NaN".
                    % This way we can easily filter it out later.
                    
                    count2 = 1;
                    for ii = pointStart:pointStart + length(dataPointInd) - 1
                        ind = dataPointInd(count2);
                        count2 = count2+1;
                        % timeString = [logData.textdata{ind,3},'-',logData.textdata{ind,4}];
                        pointTime = datetime(...
                            [logData.textdata{ind,1:end-1},num2str(logData.data(ind))],'InputFormat','eeeMMMddHH:mm:ssuuuu');
                        points(ii).time = posixtime(pointTime);
                        
                        if length(logData.textdata)> ind+2 && ~strcmp(logData.textdata{ind+2,2},'cell')
                            points(ii).disp = str2double(logData.textdata{ind+2,2});
                            
                            if ( length(logData.textdata) > ind+6 && ...
                                    ~strcmp(logData.textdata{ind+6,2},'Nov') && ...
                                    ~strcmp(logData.textdata{ind+5,2},'Nov') && ...
                                    ~strcmp(logData.textdata{ind+4,2},'Nov') && ...
                                    ~strcmp(logData.textdata{ind+3,2},'Nov') && ...
                                    strcmp(logData.textdata{ind,2},'Nov') )
                                points(ii).load = str2double(logData.textdata{ind+6,2});
                            elseif ( length(logData.textdata) > ind+4 && ...
                                    ~strcmp(logData.textdata{ind+4,2},'Jun') && ...
                                    ~strcmp(logData.textdata{ind+3,2},'Jun') && ...
                                    ~strcmp(logData.textdata{ind+2,2},'Jun') && ...
                                    ~strcmp(logData.textdata{ind+1,2},'Jun') && ...
                                    strcmp(logData.textdata{ind,2},'Jun') )
                                points(ii).load = str2double(logData.textdata{ind+4,2});
                            else
                                points(ii).load = NaN;
                            end
                        else
                            points(ii).disp = NaN;
                            if ( length(logData.textdata) > ind+6 && ...
                                    ~strcmp(logData.textdata{ind+6,2},'Nov') && ...
                                    ~strcmp(logData.textdata{ind+5,2},'Nov') && ...
                                    ~strcmp(logData.textdata{ind+4,2},'Nov') && ...
                                    ~strcmp(logData.textdata{ind+3,2},'Nov') && ...
                                    strcmp(logData.textdata{ind,2},'Nov')  )
                                points(ii).load = str2double(logData.textdata{ind+6,2});
                            elseif ( length(logData.textdata) > ind+4 && ...
                                    ~strcmp(logData.textdata{ind+4,2},'Jun') && ...
                                    ~strcmp(logData.textdata{ind+3,2},'Jun') && ...
                                    ~strcmp(logData.textdata{ind+2,2},'Jun') && ...
                                    ~strcmp(logData.textdata{ind+1,2},'Jun') && ...
                                    strcmp(logData.textdata{ind,2},'Jun')  )
                                points(ii).load = str2double(logData.textdata{ind+4,2});
                            else
                                points(ii).load = NaN;
                            end % end inside length if
                        end % end outside length if
                    end % end "pointStart loop
                    endTimes(expNum) = points(end).time;
                    %debug
                    %disp('the log start time is: ')
                    %datetime(points(pointStart).time,'ConvertFrom','posixtime');
                    
                    pointStart = ii+1;
                    
                    % SCAN INFORMATION
                    switch lower(scanDataType)
                        case 'hdf'
                            % grab load, time, and disp data from the hdf file
                            fileName = dir(fullfile(scanFileTLD,['Exp',padWithZeros(3,expNum),'*']));
                            scanFile = fullfile(scanFileTLD,fileName.name,...
                                ['proj_',padWithZeros(4,expNum),'.hdf']);
                            
                            if exist(scanFile,'file') == 2
                                [loadDuringScan, startTime, timeDuringScan] =...
                                    getScanHDFdata(scanFile);
                                
                                scanLength = length(loadDuringScan);
                                dispDuringScan = points(ii-1).disp.*ones(scanLength,1);
                                
                                % plug the scan data into the big "points" file that keeps
                                % track of the entire life of the specimen
                                count = 1;
                                for ii = pointStart : pointStart + length(loadDuringScan) - 1
                                    points(ii).load = loadDuringScan(count);
                                    points(ii).disp = dispDuringScan(count);
                                    points(ii).time = startTime + timeDuringScan(count);
                                    count = count + 1;
                                end
                                pointStart = ii + 1;
                            else
                                disp('Scan file for Exp ',num2str(expNum),' missing')
                            end
                            
                        case 'mat'
                            scanFile = fullfile(scanFileTLD,['Exp',padWithZeros(3,expNum),'.mat']);
                            %debug
                            %disp(['scan file is: ',scanFile]);
                            if exist(scanFile,'file') == 2
                                scanData = load(scanFile);
                                loadDuringScan = scanData.loadDuringScan;
                                timeDuringScan = scanData.timeDuringScan;
                                startTime = scanData.startTime;
                                
                                %debug
                                %disp('scan start is ')
                                %datetime(startTime,'ConvertFrom','posixtime');
                                
                                scanLength = length(loadDuringScan);
                                dispDuringScan = points(ii-1).disp.*ones(scanLength,1);
                                
                                count = 1;
                                for ii = pointStart : pointStart + length(loadDuringScan) - 1
                                    points(ii).load = loadDuringScan(count);
                                    points(ii).disp = dispDuringScan(count);
                                    points(ii).time = startTime + timeDuringScan(count);
                                    count = count + 1;
                                end
                                pointStart = ii + 1;
                            else
                                disp(['Scan file for Exp ',num2str(expNum),' missing'])
                            end
                            
                        case 'none'
                            %skip looking for scans entirely
                    end % end scan data type switch for not first exp scans
                else
                    disp(['Missing log file for exp ',num2str(expNum)])
                    continue
                end
            else % for the first exp, we just have the scan, no log file
                % Since there's not a log file, we need to handle getting
                % the initial displacement differently
                % SCAN INFORMATION
                switch lower(scanDataType)
                    case 'hdf'
                        % grab load, time, and disp data from the hdf file
                        fileName = dir(fullfile(scanFileTLD,['Exp',padWithZeros(3,expNum),'*']));
                        scanFile = fullfile(scanFileTLD,fileName.name,...
                            ['proj_',padWithZeros(4,expNum),'.hdf']);
                        
                        if exist(scanFile,'file') == 2
                            [loadDuringScan, startTime, timeDuringScan] =...
                                getScanHDFdata(scanFile);
                            
                            scanLength = length(loadDuringScan);
                            dispDuringScan = getInitDisp(scanLength,logFileTLD,expNum+1);
                            
                            %dispDuringScan = points(ii-1).disp.*ones(scanLength,1);
                            
                            % plug the scan data into the big "points" file that keeps
                            % track of the entire life of the specimen
                            count = 1;
                            for ii = pointStart : pointStart + length(loadDuringScan) - 1
                                points(ii).load = loadDuringScan(count);
                                points(ii).disp = dispDuringScan(count);
                                points(ii).time = startTime + timeDuringScan(count);
                                count = count + 1;
                            end
                            pointStart = ii + 1;
                        else
                            disp(['Scan file for Exp ',num2str(expNum),' missing'])
                        end
                        
                    case 'mat'
                        scanFile = fullfile(scanFileTLD,['Exp',padWithZeros(3,expNum),'.mat']);
                        %debug
                        %disp(['scan file is: ',scanFile]);
                        if exist(scanFile,'file') == 2
                            scanData = load(scanFile);
                            loadDuringScan = scanData.loadDuringScan;
                            timeDuringScan = scanData.timeDuringScan;
                            startTime = scanData.startTime;
                            
                            %debug
                            %disp('scan start is ')
                            %datetime(startTime,'ConvertFrom','posixtime');
                            
                            scanLength = length(loadDuringScan);
                            dispDuringScan = getInitDisp(scanLength,logFileTLD,expNum+1);
                            
                            count = 1;
                            for ii = pointStart : pointStart + length(loadDuringScan) - 1
                                points(ii).load = loadDuringScan(count);
                                points(ii).disp = dispDuringScan(count);
                                points(ii).time = startTime + timeDuringScan(count);
                                count = count + 1;
                            end
                            pointStart = ii + 1;
                        else
                            disp(['Scan file for Exp ',num2str(expNum),' missing'])
                        end
                        
                    case 'none'
                        %skip looking for scans entirely
                end % end scan data type switch for not first exp scans
            end % end if...else for first scan
        end % end the "expStart to expEnd" loop
        
    case 'continuous'
        
        expNum = expStart;
        pointStart = 1;
        
        switch lower(scanDataType)
            case 'hdf'
                % Data from the first scan, before loading (relaxation under
                % preload)
                fileName = dir(fullfile(scanFileTLD,['Exp',padWithZeros(3,expNum),'*']));
                scanFile = fullfile(scanFileTLD,fileName.name,['proj_',padWithZeros(4,expNum),'.hdf']);
                
                if exist(scanFile,'file') == 2
                    [loadDuringScan, startTime, timeDuringScan] =...
                        getScanHDFdata(scanFile);
                    
                    scanLength = length(loadDuringScan);
                    dispDuringScan = getInitDisp(scanLength,logFileTLD,expNum+1);
                    
                    % plug the scan data into the big "points" file that keeps
                    % track of the entire life of the specimen
                    count = 1;
                    for ii = pointStart : pointStart + length(loadDuringScan) - 1
                        points(ii).load = loadDuringScan(count);
                        points(ii).disp = dispDuringScan(count);
                        points(ii).time = startTime + timeDuringScan(count);
                        count = count + 1;
                    end
                    pointStart = ii + 1;
                else
                    disp(['Scan file for Exp ',num2str(expNum),' missing'])
                end
            case 'mat'
                scanFile = fullfile(scanFileTLD,['Exp',padWithZeros(3,expNum-1),'.mat']);
                if exist(scanFile,'file') == 2
                    scanData = load(scanFile);
                    loadDuringScan = scanData.loadDuringScan;
                    timeDuringScan = scanData.timeDuringScan;
                    startTime = scanData.startTime;
                    
                    scanLength = length(scanData.loadDuringScan);
                    dispDuringScan = getInitDisp(scanLength,logFileTLD,expNum);
                    
                    count = 1;
                    for ii = pointStart : pointStart + length(loadDuringScan) - 1
                        points(ii).load = loadDuringScan(count);
                        points(ii).disp = dispDuringScan(count);
                        points(ii).time = startTime + timeDuringScan(count);
                        count = count + 1;
                    end
                    pointStart = ii + 1;
                else
                    disp(['Scan file for Exp ',num2str(expNum),' missing'])
                end
            case 'none'
                % skip the scan
        end % end scan data type switch for "continuous"
        if exist('points','var')
            endTimes(1) = points(end).time;
        else % if there was no scan
            endTimes(1) = 0;
        end
        
        % Open the exp log file
        logFile = fullfile(logFileTLD,...
            ['Exp',padWithZeros(3,expNum),'_loading_history'],'load_history.log');
        if exist(logFile,'file') == 2
            logData = importdata(logFile,' ');
            % Find all the time stamps, these start each record
            if strcmp(logData.textdata{1,5},'CST')
                dataPointInd = find(strcmp(logData.textdata(:,5),'CST'));
            elseif strcmp(logData.textdata{1,5},'CDT')
                dataPointInd = find(strcmp(logData.textdata(:,5),'CDT'));
            else
                disp('Not a valid time stamp')
            end
            
            count2 = 1;
            for ii = pointStart:pointStart + length(dataPointInd) - 1
                ind = dataPointInd(count2);
                count2 = count2+1;
                pointTime = datetime(...
                    [logData.textdata{ind,1:end-1},num2str(logData.data(ind))],'InputFormat','eeeMMMddHH:mm:ssuuuu');
                points(ii).time = posixtime(pointTime);
                
                if length(logData.textdata)> ind+2 && ~strcmp(logData.textdata{ind+2,2},'cell')
                    points(ii).disp = str2double(logData.textdata{ind+2,2});
                    
                    if ( length(logData.textdata) > ind+6 && ...
                            ~strcmp(logData.textdata{ind+6,2},'Nov') && ...
                            ~strcmp(logData.textdata{ind+5,2},'Nov') && ...
                            ~strcmp(logData.textdata{ind+4,2},'Nov') && ...
                            ~strcmp(logData.textdata{ind+3,2},'Nov') && ...
                            strcmp(logData.textdata{ind,2},'Nov')  )
                        points(ii).load = str2double(logData.textdata{ind+6,2});
                    elseif ( length(logData.textdata) > ind+4 && ...
                            ~strcmp(logData.textdata{ind+4,2},'Jun') && ...
                            ~strcmp(logData.textdata{ind+3,2},'Jun') && ...
                            ~strcmp(logData.textdata{ind+2,2},'Jun') && ...
                            ~strcmp(logData.textdata{ind+1,2},'Jun') && ...
                            strcmp(logData.textdata{ind,2},'Jun') )
                        points(ii).load = str2double(logData.textdata{ind+4,2});
                    else
                        points(ii).load = NaN;
                    end
                else
                    points(ii).disp = NaN;
                    
                    if ( length(logData.textdata) > ind+6 && ...
                            ~strcmp(logData.textdata{ind+6,2},'Nov') && ...
                            ~strcmp(logData.textdata{ind+5,2},'Nov') && ...
                            ~strcmp(logData.textdata{ind+4,2},'Nov') && ...
                            ~strcmp(logData.textdata{ind+3,2},'Nov') && ...
                            strcmp(logData.textdata{ind,2},'Nov') )
                        points(ii).load = str2double(logData.textdata{ind+6,2});
                    elseif ( length(logData.textdata) > ind+4 && ...
                            ~strcmp(logData.textdata{ind+4,2},'Jun') && ...
                            ~strcmp(logData.textdata{ind+3,2},'Jun') && ...
                            ~strcmp(logData.textdata{ind+2,2},'Jun') && ...
                            ~strcmp(logData.textdata{ind+1,2},'Jun') && ...
                            strcmp(logData.textdata{ind,2},'Jun')  )
                        points(ii).load = str2double(logData.textdata{ind+4,2});
                    else
                        points(ii).load = NaN;
                    end % end inside if
                end % end outside if
            end % end "pointStart loop
        else
            disp(['Missing log file for exp ',num2str(expNum)])
        end
        endTimes(2) = points(end).time;
end


if exist('points','var')
    % filter out points with missing data; since we took so many this is
    % probably inconsequential. But we still need to make sure the
    % images match up, so matching time stamps, not just count, will be
    % important
    points(isnan([points.load])) = [];
    points(isnan([points.disp])) = [];
else
    points.nan = 'NaN';
end

%debug
% plot stuff
%for ii = 1:length(points), a(ii) = points(ii).load; end
%for ii = 1:length(points), b(ii) = points(ii).disp; end
%plot(b,a)