Add Link-replacement feature

.gitignore

@@ -26,7 +26,13 @@ codegen/
 
 # Personal test files
 test*.m
+test123_data.h5
+
+# files that are created in example.m
+testdata_2.h5
+test_data.mat
 export/*
 
 # Octave session info
 octave-workspace
+test_data.mat 

CI_files/default_test.m

+function [result] = default_test()
+%UNTITLED2 This is a simple test if Plot ID works for the default settings
+%   Detailed explanation goes here
+
+clear; clc; close all;
+addpath('./fcn_core','./fcn_help');
+
+ProjectID = 'Test01';
+%% Data
+% some random data
+x = linspace(0,7);
+y = rand(1,100)+2;
+dataset1 = 'test_data.mat';
+save('CI_files/test_data.mat','x','y');
+% some data as .h5
+x1 = linspace(0,2*pi);
+y1 = sin(x1)+2;
+
+% define file path & name
+fpath = "CI_files/testdata_2.h5";
+dataset2 = 'testdata_2.h5';
+
+% create hdf5 file and dataset > write data to hdf5 file / dataset
+h5create(fpath, "/x1", size(x1), "Datatype", class(x1))
+h5create(fpath, "/y1", size(y1), "Datatype", class(y1))
+h5write(fpath, "/x1", x1)
+h5write(fpath, "/y1", y1)
+
+%% Plotting
+
+fig(1) =figure('visible','off');
+plot(x,y,'-k');
+hold on
+plot(x1,y1,'-r');
+
+%% Tag the plot
+try
+    [figs, ID] = TagPlot(fig, ProjectID);
+
+    %% call a dummy function
+    a=1;
+    a = example_fcn(a);
+
+    %% publishing
+
+    % The functions needs the file location, the location of the data and the
+    % figure
+    path.script = mfilename('fullpath'); % filename of the m.script
+
+    % file name of the data
+    path.rdata =  {dataset1,dataset2} ; % don't forget the extension
+
+    Publish(path, ID, figs, 'Location', 'CI-Test')
+    
+    % clean up 
+    delete CI_files/export/* CI_files/*.mat CI_files/*.h5 
+    rmdir('CI_files/export','s'); 
+    result = true;
+    clc;
+catch
+    result = false;
+    warning('simple_test failed');
+end
+
+end
+

example.m

-% test skript
+%% Example Script
+% This Script is meant to demonstrate the capabilities of the PlotID tool.
+
+%% Clear Environment
 clear; clc; close all;
 addpath('fcn_core','fcn_help');
+addpath('CI_files'); % Test scripts
 
 try
     delete testdata_2.h5;
 end
 
+
+%% Set ProjectID
+% ProjectID 
+
+% TODO:  decide how projectID and optionally ORCID will be implemented
+% ORCID placed on startup (alternative?) - projectID as persistent
+% otherwise dialogue?
 ProjectID = 'JL01';
+
+
+
 %% Data
-% some random data
+% Creating Random Data to use as data-file
 
 x = linspace(0,7);
 y = rand(1,100)+2;
@@ -30,30 +44,52 @@ h5create(fpath, "/y1", size(y1), "Datatype", class(y1))
 h5write(fpath, "/x1", x1)
 h5write(fpath, "/y1", y1)
 
-%% Plotting
 
-fig(1) =figure;
+%% Plotting
+% This is still part of a normal script to produce plots.
+% Make sure to save each figure in a variable to pass to PlotID-functions.
+fig(1) = figure;
 plot(x,y,'-k');
 box off
 set(gca, 'TickDir', 'out', 'YLim', [0,4]);
 
 hold on
-%fig(2) =figure;
+%fig(2) = figure;
 plot(x1,y1,'-r');
 set(gca, 'TickDir', 'out', 'YLim', [0,4]);
 
-% Tag the plot
+%% Tag the plot
+% PlotID Implementation starts here.
+ 
 [figs, ID] = TagPlot(fig, ProjectID);
 
-%% publishing
 
+
+%% call a dummy function
+% Place for post-processing of the plot, or additional related code.
+a=1;
+a = example_fcn(a);
+
+%% Publishing
+% Second part of plotID 
 % The functions needs the file location, the location of the data and the
-% figure
-path.script = mfilename('fullpath') % filename of the m.script
+% figure and can take several options.
+% TODO add explanations for Options
+path.script = mfilename('fullpath'); % filename of the m.script
 
-% file name of the data (should be extended to handle arrays)
+% file name of the dataset
 path.rdata =  {dataset1,dataset2} ; % don't forget the extension
 
+Publish(path, ID, figs, 'Location', 'local','Method','centraliced')
+
+%% Second Plot with identical data to test centralized method
+% 
+
+fig2 =figure;
+plot(x,y,'-k');
+hold on
+plot(x1,y1,'-r');
 
-Publish(path, ID, figs, 'Location', 'local','Method','individual')
+[fig2, ID] = TagPlot(fig2, ProjectID);
 
+Publish(path, ID, fig2, 'Location', 'local','Method','centraliced')
\ No newline at end of file

example_fcn.m

+function [outputArg1] = example_fcn(inputArg1)
+%TEST_2 just a dummy function
+outputArg1 = inputArg1;
+
+end
+

fcn_core/CreateID.m

@@ -11,7 +11,12 @@ switch method
         ID = posixtime(datetime('now')); %get current Unix time
         ID = dec2hex(int32(ID)); % get it as Hex
         pause(0.5); %Pausing for unique IDs
-    otherwise
+    case 2 % random UUID from Java 128 bit
+        %Static factory to retrieve a type 4 (pseudo randomly generated) UUID.
+        % The UUID is generated using a cryptographically strong pseudo random number generator.
+        temp =  java.util.UUID.randomUUID;
+        ID = temp.toString;           
+     otherwise
         error('The requested method is not defined yet');
 end
 

fcn_core/Publish.m

@@ -11,15 +11,18 @@ arguments
    DataPaths
    ID (1,:) {mustBeNonzeroLengthText} % ID must be provided
    figures (1,:) {mustBeFigure} % Checks if Figures are figures
-   options.Location {mustBeMember(options.Location ,['local','server'])} = 'local' % storage path
+   options.Location {mustBeMember(options.Location ,['local','server','CI-Test'])} = 'local' % storage path
    options.Method {mustBeMember(options.Method ,['individual','centraliced'])} = 'individual'
+   options.CopyUserFCN (1,1) {mustBeNumericOrLogical} = true 
 end
 
 switch options.Location 
     case 'local'
          storPath = fullfile(pwd,'export');
     case 'server'
-        storPath = '\\FST-220\Jans-50GB-SMB\Lemmer'; 
+        storPath = '\\FST-220\Jans-50GB-SMB\Lemmer';
+    case 'CI-Test'
+        storPath = fullfile(pwd,'CI_files','export');
 end
 folderName = char(ID);
 
@@ -32,40 +35,52 @@ end
 
 disp('Publishing started');
 
-%% Create a Copy of the script
+%% Create a Copy of the script and User functions(optional)
+createFileCopy({[DataPaths.script,'.m']},folderName,storPath,ID, 'script');
 
-try
-    %FileNameAndLocation=['C:\Users\Lemmer\Documents\GitLab\plot_identifier\MATLAB\test'];
-    FileNameAndLocation= DataPaths.script;
-    newbackup=sprintf([ID,'_script.m']);
-    currentfile=strcat(FileNameAndLocation, '.m');
-    % Write a Copy of the Plotting Script 
-    RemotePath = fullfile(storPath,folderName, newbackup);
-    copyfile(currentfile,RemotePath);
-    disp('Script sucessfully published');
-catch
-   warning('Script export was not sucessful')
+if options.CopyUserFCN
+   [fList,pList] = matlab.codetools.requiredFilesAndProducts(DataPaths.script);
+   % plist contains the required MATLAB Toolboxes, maybe usefull in future
+   fList = fList(~ismember(fList,[DataPaths.script,'.m'])); % rmv script from list
+   fList = removePltIdFiles(fList); % Do not copy files that are part of plot ID
+   if ~isempty(fList)
+       createFileCopy(fList,folderName,storPath,ID,'userFcn');
+   end
 end
+
 %% Research data handeling
 switch options.Method
     case 'centraliced' % Work in progresss!!!
-        %check if data folder exists
+        % check if data folder exists
         if ~isfolder(fullfile(storPath,'data'))
             mkdir(fullfile(storPath,'data'));
         end
+        %list all files
+        fList = dir(fullfile(storPath,'data', '**\*.*'));  %get list of files and folders in any subfolder
+        fList = fList(~[fList.isdir]);  %remove folders from list
+        fList = struct2table(fList);
+        
         % Check if the new plot is based on the original data-set    
             % copy the data(once)
-            createFileCopy(DataPaths.rdata,'data',storPath,ID);
-        % create a linked (soft) copy
-            
-        
-               
+        for i=1:numel(DataPaths.rdata)            
+            % check if identical file exists (status = 1)
+            [~, idx] = fileCompare(DataPaths.rdata{i},fList);
+            % create Linked HDF5 files for identical files
+            if any(idx)
+            sourcePath = fList.name{idx}; % If there are multiple copies already, this only picks the last entry
+            if contains(sourcePath,{'.h5','.hdf5'}) % Linking only for HDF5
+                createLinkedHDF5(sourcePath,storPath,ID);
+            end
+            else % no identical file exists
+               createFileCopy(DataPaths.rdata{i},'data',storPath,ID,'dataCentral');
+            end
+        end
     case 'individual'
         % Create a copy of the research data    
-        createFileCopy(DataPaths.rdata,folderName,storPath,ID);
+        createFileCopy(DataPaths.rdata,folderName,storPath,ID, 'data');
 end
 %% Export the Plots
-%try 
+try 
    for i=1:numel(figures)
        fig = figures(i);
        PlotName = [ID,'_plot',num2str(i)]; % Der Plotnamen
@@ -75,34 +90,14 @@ end
        exportgraphics(fig,[RemotePath,'.png'],'Resolution',300);
        disp([num2str(i),' of ',num2str(numel(figures)),' figures exported']);
    end
-%catch 
-    %warning('Plot export was not sucessful')
-%end
+catch 
+    warning('Plot export was not successful')
+end
 
-disp(['Publishing of ', ID , ' done']);
+disp(['publishing of ', ID , ' done']);
 
 end %function
 
-function [] = createFileCopy(filePaths,folderName,storPath,ID)
-% Creates a copy of the files (can used for multiple paths in a cell array)
-% folderName is the name of the exporting folder
-    disp('Start to copy research data ...');   
-    try        
-        for i = 1:numel(filePaths)
-            FileNameAndLocation = filePaths{i};
-            [~,~,ext] = fileparts(filePaths{i}); % get the extension
-            newbackup = sprintf([ID,'_data',ext]);
-
-            % Write the copied file
-            RemotePath = fullfile(storPath,folderName, newbackup);
-            copyfile(FileNameAndLocation,RemotePath);
-        end
-        disp('Research data sucessfully published');
-    catch
-        warning('Research data export was not sucessful')
-    end %try
-end
-
 function tf = mustBeFigure(h)
 %checks if input is a figure object
   tf = strcmp(get(h, 'type'), 'figure') & isa(h, 'matlab.ui.Figure');

fcn_core/TagPlot.m

-function [figs, ID] = TagPlot(figs, prefix)
+function [figs, ID] = TagPlot(figs, prefix, options)
 %TagPlot adds IDs to figures
 %   The ID is placed visual on the figure window and as Tag (Property of figure)
 %   TagPlot can tag multiple figures at once
+%   prefix is the project number (string or char)
+%   
+%   The ID is placed on the 'east' per default, if you want it somwhere
+%   else, use the 'Location' option. 'north','east','south','west' are
+%   predefined, otherwise use the 'custom' option and provide the desired
+%   'Position' (relative to your x- and y-axis limits)
 arguments
     figs (1,:) {mustBeFigure}
     prefix (1,:) {mustBeText}= ''
+    options.Fontsize (1,1) {mustBeInteger} = 8 
+    options.Location (1,:) {mustBeText} = 'east'
+    options.Position (1,2) {mustBeVector} = [1,0.4] % default for east
+    options.Rotation (1,1) {mustBeInteger} = 0  
 end
 
 if isempty(prefix)
     warning('no project prefix defined')
 end
 
+switch options.Location
+    case 'north'
+        options.Position = [0.4,0.95];
+        options.Rotation = 0;
+    case 'east'
+        options.Position = [1,0.4];
+        options.Rotation = 90;
+    case 'south'
+        options.Position = [0.4,.02];
+        options.Rotation = 0;
+    case 'west'
+        options.Position = [.05,0.4];
+        options.Rotation = 90;
+    case 'custom'
+        % Check if Position is valid
+        if ~all(0 <= options.Position & options.Position <= 1)
+           options.Position = [1,0.4];
+           warning('options.Position is not valid, TagPlot default values instead');
+        end
+    otherwise % set default position
+        warning([options.Location, ' is not a defined location, TagPlot uses location east instead']);
+        options.Location = 'east'; options.Position = [1,0.4];
+end
+    
 for n = 1:numel(figs)
     ID = CreateID; % Create ID
     ID = [prefix,'-',ID]; % add Prefix
@@ -19,8 +53,10 @@ for n = 1:numel(figs)
     ylim =get(axes,'YLim');
     xlim =get(axes,'XLim');
     %ID
-    text(axes,xlim(2),0.4*ylim(2), ID,'Fontsize',8,'Rotation',90,'VerticalAlignment','bottom',...
-    'Color', 0.65*[1 1 1],'BackgroundColor','w');
+    position = [options.Position(1)*xlim(2), options.Position(2)*ylim(2)];
+    text(axes,position(1),position(2), ID,'Fontsize',options.Fontsize,...
+    'Rotation',options.Rotation, 'VerticalAlignment','bottom','Color',...
+        0.65*[1 1 1],'BackgroundColor','w');
     set(figs(n),'Tag', ID);
     
 end

fcn_help/FileCompare.m

-function [status] = fileCompare(filename1,filename2)
-%UNTITLED2 Summary of this function goes here
-%   Detailed explanation goes here
-
-[~,~,ext1] = fileparts(filename1); 
-[~,~,ext2] = fileparts(filename2);
-
-if ~isequal(ext1,ext2)
-    warning('File extension are not identical');
-    status = false;
-    return
-end
-
-if ispc
-    filename2 = 'export\JL01-60DB3572\JL01-60DB3572_data.h5';
-    filename1 = 'export\JL01-60DB3576\JL01-60DB3576_data.h5';
-    [status,result] = system(['fc ' filename1 ' ' filename2]);
-
-elseif isunix %untested!
-    filename2 = 'export/JL01-60DB3572/JL01-60DB3572_data.h5';
-    filename1 = 'export/JL01-60DB3576/JL01-60DB3576_data.h5';
-    [status,result] = system(['diff ' filename1 ' ' filename2]);
-else
-    warning('Platform not supported')
-end
-
-
-end
-

fcn_help/createFileCopy.m

+function [] = createFileCopy(filePaths,folderName,storPath,ID,type)
+% Creates a copy of the files (can be used for multiple paths in a cell array)
+% folderName is the name of the exporting folder
+    disp(['start to copy ', type]);  
+    
+    if ~iscell(filePaths)
+        %fixes Issue if Filepath is a char and not a cell array
+        filePaths = {filePaths};
+    end
+        
+    try        
+        for i = 1:numel(filePaths)
+            FileNameAndLocation = filePaths{i};
+            [~,name,ext] = fileparts(filePaths{i}); % get the extension
+                          
+            switch type
+                case 'data'
+                    sufix = '_data';
+                    newfile = sprintf([ID, sufix, '_' , num2str(i) ,ext]);
+                case 'dataCentral'
+                    %keep original name
+                    newfile = sprintf([name,ext]);
+                case 'script'
+                    sufix = '_script';
+                    newfile = sprintf([ID, sufix ,ext]);
+                case 'userFcn'
+                    %keep original name
+                    newfile = sprintf([name,ext]);
+                otherwise 
+                    error([type,' is not a valid type for createFileCopy'])
+            end %switch 
+            
+            % Write the file
+            RemotePath = fullfile(storPath,folderName, newfile);
+            
+%           Check if remote file already exists
+            count = 0;
+            while isfile(RemotePath) && ismember(type,{'data','dataCentral'})
+                % Add a Sufix number to new file name
+                count = count + 1;
+                [~,name,ext] = fileparts(RemotePath);
+                RemotePath = fullfile(storPath,folderName,...
+                    [name,'_',num2str(count),ext]);
+            end
+            copyfile(FileNameAndLocation,RemotePath);
+        end
+        disp([type, ' sucessfully published']);
+    catch
+        warning([type,' export was not sucessful'])
+   end %try
+end
+

fcn_help/createLinkedHDF5.m

 function [status] = createLinkedHDF5(SourceFile,TargetPath,ID)
-%UNTITLED4 Summary of this function goes here
-%   Detailed explanation goes here
+%createLinkedHDF5 creates a HDF file that references the Sourcefile
+%   TargetPath is the storage location, ID the foldername
+%   Status returns true if the function was sucessfull
 
 plist_id = 'H5P_DEFAULT';
 
-% Work in Progress... Problem mit dem Pfad...
-
-try
-    fid = H5F.create(fullfile(TargetPath,[ID,'_data.h5']));
+% try
+    fid = H5F.create(fullfile(TargetPath,ID,[ID,'_data.h5']));
     %create External Link to Sourcefile in the Group linkToExternal
-    H5L.create_external(SourceFile,'/',fid,'linkToExternal',plist_id,plist_id);
+    H5L.create_external(['..\data\',SourceFile],'/',fid, SourceFile ,plist_id,plist_id);
     H5F.close(fid);
     status = 1;
-catch
-    warning('No linked HDF file was created');
-    status = 0;
-end
+% catch
+%     warning('No linked HDF file was created');
+%     status = 0;
+% end
+
 end
 

fcn_help/fileCompare.m

+function [status, id] = fileCompare(filename,fileList)
+%fileCompare checks if file1 is (binary) identical to a file in filelist
+% it returns a sttus and the id of the identical file
+% the functions uses the windows system function fc or the unix function
+% diff
+
+if isempty(fileList)
+   % no comparison necessary
+    status =false;
+    id = 0;
+    return
+end
+
+[~,~,ext1] = fileparts(filename); 
+id = zeros(height(fileList),1);
+
+for i=1:height(fileList)
+    [~,~,ext2] = fileparts(fileList.name{i});
+
+    if ~isequal(ext1,ext2)
+        %warning('File extension are not identical');
+        status = false;
+        continue
+    end
+
+    filepath = fullfile(fileList.folder{i},fileList.name{i});
+    if ispc
+        [status,~] = system(['fc ' filename ' ' filepath]);
+        % 0 -> identical, 1 -> not identical
+         status = ~status; % false (not identical), true(identical)
+                   
+    elseif isunix %untested!
+        [status,~] = system(['diff ' filename ' ' filepath]);
+    else
+        warning('Platform not supported')
+    end
+    
+    if status == 1        
+        id(i) = 1;
+    else 
+        % Status can also be any other number e.g. 2 
+        id(i) = 0;
+    end
+    
+    id =logical(id); %bugfix
+end
+
+end
+

fcn_help/removePltIdFiles.m

+function [fListClean] = removePltIdFiles(fList)
+%removePltIdFiles removes functions that are part of PlotID out of flist
+%   Detailed explanation goes here
+%addpath('..\fcn_core');
+
+[~,names,ext] = fileparts(fList);
+names = strcat(names, ext); % add ext for comparison
+
+% Get a list of all .m files that are part of Plot id
+PltID_flist = struct2table([dir('fcn_help'); dir('fcn_core')]); %get list of files 
+[~,~,PltID_flist.ext(:)] = fileparts(PltID_flist.name(:)); % add ext column
+
+PltID_flist = PltID_flist(strcmp(PltID_flist.ext,'.m'),:);
+
+% Comparison and filter
+fListClean = fList(~ismember(names,PltID_flist.name));
+
+end
+

fcn_help/replaceLinkedHDF5.m

+function replaceLinkedHDF5(linkedFilepath)
+% replaceLinkedHDF5 replaces the HDF5, that contains only a link to another
+% HDF5 File with a file containing the actual Data, thus turning it into an
+% independent data file. It essentially turns the plotID option
+% 'centralized' into 'individual'.
+%   Filepath is the location of the File that contains the link. The
+%   filepath to the linked data has to be present in the file itself.
+
+
+
+%% Check whether the only Objects present are links
+h5inf = h5info(linkedFilepath);
+
+% default behaviour dont overwrite
+overwrite = false;
+
+if isempty([h5inf.Groups, h5inf.Datasets, h5inf.Datatypes, h5inf.Attributes]) && ...
+      all(size(h5inf.Links) == [1 1])  
+    % There are no other known objects present and
+    % there is only one linked file
+       overwrite = true;
+end
+
+%% Create Copies of files
+for i = 1:size(h5inf.Links, 2)
+    % This is the file with full data  
+    linkTargetPath = h5inf.Links(i).Value{1,1}; 
+    % This is the empty file
+    % TODO filename decision
+    linkSourcePath = strcat(linkedFilepath,'_',num2str(i)); 
+
+    % Create an absolute path from the relative linking path
+    absoluteTP = regexp(linkSourcePath, filesep, 'split');
+    TargetPathSplit = regexp(linkTargetPath, filesep, 'split');
+    % How many levels do we need to go up in the relative path
+    % This is merely counting the occurances of '..' within the path, but
+    % should not cause issues if the realtive path was created by the
+    % linking process
+    levelsUp = sum(contains(TargetPathSplit,'..'));
+    absoluteTP = absoluteTP(1:(end-(1+levelsUp)) ); %Remove filename + levels
+    linkTargetPath = strjoin([absoluteTP, TargetPathSplit(~contains(TargetPathSplit,'..'))],filesep);
+    
+    if ~isfile(linkTargetPath)
+        warning(strcat("Link destination """,linkTargetPath,""" does not exist or relative path does not fit, no copy created"))
+    end
+    
+    if overwrite % can only be for i=1, and size = 1
+    linkSourcePath = linkedFilepath;
+    end
+    disp('Copying file from ', linkSourcePath, ' to ', linkTargetPath)
+    copyfile(linkTargetPath,linkSourcePath);
+    
+end
+        
+        
+    
+end
+
+
+
+