diff --git a/.gitignore b/.gitignore index b0afd4835bf581456c0cf7f6dc7bcd8139cddf34..3d5fc69ca1e50965038a6e833ed09a2d1a9d32a8 100644 --- a/.gitignore +++ b/.gitignore @@ -26,7 +26,13 @@ codegen/ # Personal test files test*.m +test123_data.h5 + +# files that are created in example.m +testdata_2.h5 +test_data.mat export/* # Octave session info octave-workspace +test_data.mat diff --git a/CI_files/default_test.m b/CI_files/default_test.m new file mode 100644 index 0000000000000000000000000000000000000000..515ad3718e2c3c0b06b6e037da2f6343796055eb --- /dev/null +++ b/CI_files/default_test.m @@ -0,0 +1,66 @@ +function [result] = default_test() +%UNTITLED2 This is a simple test if Plot ID works for the default settings +% Detailed explanation goes here + +clear; clc; close all; +addpath('./fcn_core','./fcn_help'); + +ProjectID = 'Test01'; +%% Data +% some random data +x = linspace(0,7); +y = rand(1,100)+2; +dataset1 = 'test_data.mat'; +save('CI_files/test_data.mat','x','y'); +% some data as .h5 +x1 = linspace(0,2*pi); +y1 = sin(x1)+2; + +% define file path & name +fpath = "CI_files/testdata_2.h5"; +dataset2 = 'testdata_2.h5'; + +% create hdf5 file and dataset > write data to hdf5 file / dataset +h5create(fpath, "/x1", size(x1), "Datatype", class(x1)) +h5create(fpath, "/y1", size(y1), "Datatype", class(y1)) +h5write(fpath, "/x1", x1) +h5write(fpath, "/y1", y1) + +%% Plotting + +fig(1) =figure('visible','off'); +plot(x,y,'-k'); +hold on +plot(x1,y1,'-r'); + +%% Tag the plot +try + [figs, ID] = TagPlot(fig, ProjectID); + + %% call a dummy function + a=1; + a = example_fcn(a); + + %% publishing + + % The functions needs the file location, the location of the data and the + % figure + path.script = mfilename('fullpath'); % filename of the m.script + + % file name of the data + path.rdata = {dataset1,dataset2} ; % don't forget the extension + + Publish(path, ID, figs, 'Location', 'CI-Test') + + % clean up + delete CI_files/export/* CI_files/*.mat CI_files/*.h5 + rmdir('CI_files/export','s'); + result = true; + clc; +catch + result = false; + warning('simple_test failed'); +end + +end + diff --git a/copiedFile.h5 b/copiedFile.h5 deleted file mode 100644 index 4ddca81c276e204d2476b044f5a2fda057105080..0000000000000000000000000000000000000000 Binary files a/copiedFile.h5 and /dev/null differ diff --git a/example.m b/example.m index e25950de4b3bf0e7bb0edb41c76791a8d3992e30..f208097f48304f8ca96fbc5dd1907c184233b44a 100644 --- a/example.m +++ b/example.m @@ -1,14 +1,28 @@ -% test skript +%% Example Script +% This Script is meant to demonstrate the capabilities of the PlotID tool. + +%% Clear Environment clear; clc; close all; addpath('fcn_core','fcn_help'); +addpath('CI_files'); % Test scripts try delete testdata_2.h5; end + +%% Set ProjectID +% ProjectID + +% TODO: decide how projectID and optionally ORCID will be implemented +% ORCID placed on startup (alternative?) - projectID as persistent +% otherwise dialogue? ProjectID = 'JL01'; + + + %% Data -% some random data +% Creating Random Data to use as data-file x = linspace(0,7); y = rand(1,100)+2; @@ -30,30 +44,52 @@ h5create(fpath, "/y1", size(y1), "Datatype", class(y1)) h5write(fpath, "/x1", x1) h5write(fpath, "/y1", y1) -%% Plotting -fig(1) =figure; +%% Plotting +% This is still part of a normal script to produce plots. +% Make sure to save each figure in a variable to pass to PlotID-functions. +fig(1) = figure; plot(x,y,'-k'); box off set(gca, 'TickDir', 'out', 'YLim', [0,4]); hold on -%fig(2) =figure; +%fig(2) = figure; plot(x1,y1,'-r'); set(gca, 'TickDir', 'out', 'YLim', [0,4]); -% Tag the plot +%% Tag the plot +% PlotID Implementation starts here. + [figs, ID] = TagPlot(fig, ProjectID); -%% publishing + +%% call a dummy function +% Place for post-processing of the plot, or additional related code. +a=1; +a = example_fcn(a); + +%% Publishing +% Second part of plotID % The functions needs the file location, the location of the data and the -% figure -path.script = mfilename('fullpath') % filename of the m.script +% figure and can take several options. +% TODO add explanations for Options +path.script = mfilename('fullpath'); % filename of the m.script -% file name of the data (should be extended to handle arrays) +% file name of the dataset path.rdata = {dataset1,dataset2} ; % don't forget the extension +Publish(path, ID, figs, 'Location', 'local','Method','centraliced') + +%% Second Plot with identical data to test centralized method +% + +fig2 =figure; +plot(x,y,'-k'); +hold on +plot(x1,y1,'-r'); -Publish(path, ID, figs, 'Location', 'local','Method','individual') +[fig2, ID] = TagPlot(fig2, ProjectID); +Publish(path, ID, fig2, 'Location', 'local','Method','centraliced') \ No newline at end of file diff --git a/example_fcn.m b/example_fcn.m new file mode 100644 index 0000000000000000000000000000000000000000..418f15ca734ac8423bdda5a6ead2c32af6820e22 --- /dev/null +++ b/example_fcn.m @@ -0,0 +1,6 @@ +function [outputArg1] = example_fcn(inputArg1) +%TEST_2 just a dummy function +outputArg1 = inputArg1; + +end + diff --git a/fcn_core/CreateID.m b/fcn_core/CreateID.m index 94d0df2216f9732540f21ae68b7eaccc9c8a2c8a..42ec2b29e9c2c041ecbe190175d07cf219deef02 100644 --- a/fcn_core/CreateID.m +++ b/fcn_core/CreateID.m @@ -11,7 +11,12 @@ switch method ID = posixtime(datetime('now')); %get current Unix time ID = dec2hex(int32(ID)); % get it as Hex pause(0.5); %Pausing for unique IDs - otherwise + case 2 % random UUID from Java 128 bit + %Static factory to retrieve a type 4 (pseudo randomly generated) UUID. + % The UUID is generated using a cryptographically strong pseudo random number generator. + temp = java.util.UUID.randomUUID; + ID = temp.toString; + otherwise error('The requested method is not defined yet'); end diff --git a/fcn_core/Publish.m b/fcn_core/Publish.m index eb0c428aebddba8a2311df6acb31d39c4af097ce..e6952d70589138df8eec1b9e2f3b7f74f5a3f1e2 100644 --- a/fcn_core/Publish.m +++ b/fcn_core/Publish.m @@ -11,15 +11,18 @@ arguments DataPaths ID (1,:) {mustBeNonzeroLengthText} % ID must be provided figures (1,:) {mustBeFigure} % Checks if Figures are figures - options.Location {mustBeMember(options.Location ,['local','server'])} = 'local' % storage path + options.Location {mustBeMember(options.Location ,['local','server','CI-Test'])} = 'local' % storage path options.Method {mustBeMember(options.Method ,['individual','centraliced'])} = 'individual' + options.CopyUserFCN (1,1) {mustBeNumericOrLogical} = true end switch options.Location case 'local' storPath = fullfile(pwd,'export'); case 'server' - storPath = '\\FST-220\Jans-50GB-SMB\Lemmer'; + storPath = '\\FST-220\Jans-50GB-SMB\Lemmer'; + case 'CI-Test' + storPath = fullfile(pwd,'CI_files','export'); end folderName = char(ID); @@ -32,40 +35,52 @@ end disp('Publishing started'); -%% Create a Copy of the script +%% Create a Copy of the script and User functions(optional) +createFileCopy({[DataPaths.script,'.m']},folderName,storPath,ID, 'script'); -try - %FileNameAndLocation=['C:\Users\Lemmer\Documents\GitLab\plot_identifier\MATLAB\test']; - FileNameAndLocation= DataPaths.script; - newbackup=sprintf([ID,'_script.m']); - currentfile=strcat(FileNameAndLocation, '.m'); - % Write a Copy of the Plotting Script - RemotePath = fullfile(storPath,folderName, newbackup); - copyfile(currentfile,RemotePath); - disp('Script sucessfully published'); -catch - warning('Script export was not sucessful') +if options.CopyUserFCN + [fList,pList] = matlab.codetools.requiredFilesAndProducts(DataPaths.script); + % plist contains the required MATLAB Toolboxes, maybe usefull in future + fList = fList(~ismember(fList,[DataPaths.script,'.m'])); % rmv script from list + fList = removePltIdFiles(fList); % Do not copy files that are part of plot ID + if ~isempty(fList) + createFileCopy(fList,folderName,storPath,ID,'userFcn'); + end end + %% Research data handeling switch options.Method case 'centraliced' % Work in progresss!!! - %check if data folder exists + % check if data folder exists if ~isfolder(fullfile(storPath,'data')) mkdir(fullfile(storPath,'data')); end + %list all files + fList = dir(fullfile(storPath,'data', '**\*.*')); %get list of files and folders in any subfolder + fList = fList(~[fList.isdir]); %remove folders from list + fList = struct2table(fList); + % Check if the new plot is based on the original data-set % copy the data(once) - createFileCopy(DataPaths.rdata,'data',storPath,ID); - % create a linked (soft) copy - - - + for i=1:numel(DataPaths.rdata) + % check if identical file exists (status = 1) + [~, idx] = fileCompare(DataPaths.rdata{i},fList); + % create Linked HDF5 files for identical files + if any(idx) + sourcePath = fList.name{idx}; % If there are multiple copies already, this only picks the last entry + if contains(sourcePath,{'.h5','.hdf5'}) % Linking only for HDF5 + createLinkedHDF5(sourcePath,storPath,ID); + end + else % no identical file exists + createFileCopy(DataPaths.rdata{i},'data',storPath,ID,'dataCentral'); + end + end case 'individual' % Create a copy of the research data - createFileCopy(DataPaths.rdata,folderName,storPath,ID); + createFileCopy(DataPaths.rdata,folderName,storPath,ID, 'data'); end %% Export the Plots -%try +try for i=1:numel(figures) fig = figures(i); PlotName = [ID,'_plot',num2str(i)]; % Der Plotnamen @@ -75,34 +90,14 @@ end exportgraphics(fig,[RemotePath,'.png'],'Resolution',300); disp([num2str(i),' of ',num2str(numel(figures)),' figures exported']); end -%catch - %warning('Plot export was not sucessful') -%end +catch + warning('Plot export was not successful') +end -disp(['Publishing of ', ID , ' done']); +disp(['publishing of ', ID , ' done']); end %function -function [] = createFileCopy(filePaths,folderName,storPath,ID) -% Creates a copy of the files (can used for multiple paths in a cell array) -% folderName is the name of the exporting folder - disp('Start to copy research data ...'); - try - for i = 1:numel(filePaths) - FileNameAndLocation = filePaths{i}; - [~,~,ext] = fileparts(filePaths{i}); % get the extension - newbackup = sprintf([ID,'_data',ext]); - - % Write the copied file - RemotePath = fullfile(storPath,folderName, newbackup); - copyfile(FileNameAndLocation,RemotePath); - end - disp('Research data sucessfully published'); - catch - warning('Research data export was not sucessful') - end %try -end - function tf = mustBeFigure(h) %checks if input is a figure object tf = strcmp(get(h, 'type'), 'figure') & isa(h, 'matlab.ui.Figure'); diff --git a/fcn_core/TagPlot.m b/fcn_core/TagPlot.m index 3c426266624a4a9dd64e93d970428c2fe08837a0..4fac128393edd75b1b7877cee7ce29eb517f54c1 100644 --- a/fcn_core/TagPlot.m +++ b/fcn_core/TagPlot.m @@ -1,16 +1,50 @@ -function [figs, ID] = TagPlot(figs, prefix) +function [figs, ID] = TagPlot(figs, prefix, options) %TagPlot adds IDs to figures % The ID is placed visual on the figure window and as Tag (Property of figure) % TagPlot can tag multiple figures at once +% prefix is the project number (string or char) +% +% The ID is placed on the 'east' per default, if you want it somwhere +% else, use the 'Location' option. 'north','east','south','west' are +% predefined, otherwise use the 'custom' option and provide the desired +% 'Position' (relative to your x- and y-axis limits) arguments figs (1,:) {mustBeFigure} prefix (1,:) {mustBeText}= '' + options.Fontsize (1,1) {mustBeInteger} = 8 + options.Location (1,:) {mustBeText} = 'east' + options.Position (1,2) {mustBeVector} = [1,0.4] % default for east + options.Rotation (1,1) {mustBeInteger} = 0 end if isempty(prefix) warning('no project prefix defined') end +switch options.Location + case 'north' + options.Position = [0.4,0.95]; + options.Rotation = 0; + case 'east' + options.Position = [1,0.4]; + options.Rotation = 90; + case 'south' + options.Position = [0.4,.02]; + options.Rotation = 0; + case 'west' + options.Position = [.05,0.4]; + options.Rotation = 90; + case 'custom' + % Check if Position is valid + if ~all(0 <= options.Position & options.Position <= 1) + options.Position = [1,0.4]; + warning('options.Position is not valid, TagPlot default values instead'); + end + otherwise % set default position + warning([options.Location, ' is not a defined location, TagPlot uses location east instead']); + options.Location = 'east'; options.Position = [1,0.4]; +end + for n = 1:numel(figs) ID = CreateID; % Create ID ID = [prefix,'-',ID]; % add Prefix @@ -19,8 +53,10 @@ for n = 1:numel(figs) ylim =get(axes,'YLim'); xlim =get(axes,'XLim'); %ID - text(axes,xlim(2),0.4*ylim(2), ID,'Fontsize',8,'Rotation',90,'VerticalAlignment','bottom',... - 'Color', 0.65*[1 1 1],'BackgroundColor','w'); + position = [options.Position(1)*xlim(2), options.Position(2)*ylim(2)]; + text(axes,position(1),position(2), ID,'Fontsize',options.Fontsize,... + 'Rotation',options.Rotation, 'VerticalAlignment','bottom','Color',... + 0.65*[1 1 1],'BackgroundColor','w'); set(figs(n),'Tag', ID); end diff --git a/fcn_help/FileCompare.m b/fcn_help/FileCompare.m deleted file mode 100644 index aa5adfe3cbce82b1627760697908143b11894e53..0000000000000000000000000000000000000000 --- a/fcn_help/FileCompare.m +++ /dev/null @@ -1,29 +0,0 @@ -function [status] = fileCompare(filename1,filename2) -%UNTITLED2 Summary of this function goes here -% Detailed explanation goes here - -[~,~,ext1] = fileparts(filename1); -[~,~,ext2] = fileparts(filename2); - -if ~isequal(ext1,ext2) - warning('File extension are not identical'); - status = false; - return -end - -if ispc - filename2 = 'export\JL01-60DB3572\JL01-60DB3572_data.h5'; - filename1 = 'export\JL01-60DB3576\JL01-60DB3576_data.h5'; - [status,result] = system(['fc ' filename1 ' ' filename2]); - -elseif isunix %untested! - filename2 = 'export/JL01-60DB3572/JL01-60DB3572_data.h5'; - filename1 = 'export/JL01-60DB3576/JL01-60DB3576_data.h5'; - [status,result] = system(['diff ' filename1 ' ' filename2]); -else - warning('Platform not supported') -end - - -end - diff --git a/fcn_help/createFileCopy.m b/fcn_help/createFileCopy.m new file mode 100644 index 0000000000000000000000000000000000000000..4bf198e429c269dbdc75f42e846f4e416b3a0130 --- /dev/null +++ b/fcn_help/createFileCopy.m @@ -0,0 +1,52 @@ +function [] = createFileCopy(filePaths,folderName,storPath,ID,type) +% Creates a copy of the files (can be used for multiple paths in a cell array) +% folderName is the name of the exporting folder + disp(['start to copy ', type]); + + if ~iscell(filePaths) + %fixes Issue if Filepath is a char and not a cell array + filePaths = {filePaths}; + end + + try + for i = 1:numel(filePaths) + FileNameAndLocation = filePaths{i}; + [~,name,ext] = fileparts(filePaths{i}); % get the extension + + switch type + case 'data' + sufix = '_data'; + newfile = sprintf([ID, sufix, '_' , num2str(i) ,ext]); + case 'dataCentral' + %keep original name + newfile = sprintf([name,ext]); + case 'script' + sufix = '_script'; + newfile = sprintf([ID, sufix ,ext]); + case 'userFcn' + %keep original name + newfile = sprintf([name,ext]); + otherwise + error([type,' is not a valid type for createFileCopy']) + end %switch + + % Write the file + RemotePath = fullfile(storPath,folderName, newfile); + +% Check if remote file already exists + count = 0; + while isfile(RemotePath) && ismember(type,{'data','dataCentral'}) + % Add a Sufix number to new file name + count = count + 1; + [~,name,ext] = fileparts(RemotePath); + RemotePath = fullfile(storPath,folderName,... + [name,'_',num2str(count),ext]); + end + copyfile(FileNameAndLocation,RemotePath); + end + disp([type, ' sucessfully published']); + catch + warning([type,' export was not sucessful']) + end %try +end + diff --git a/fcn_help/createLinkedHDF5.m b/fcn_help/createLinkedHDF5.m index 5b1e9716ccd5219e8756ae0af60b23c0d6ac282c..2738fd173b7e00ed36cadad45912c79246c6a872 100644 --- a/fcn_help/createLinkedHDF5.m +++ b/fcn_help/createLinkedHDF5.m @@ -1,20 +1,20 @@ function [status] = createLinkedHDF5(SourceFile,TargetPath,ID) -%UNTITLED4 Summary of this function goes here -% Detailed explanation goes here +%createLinkedHDF5 creates a HDF file that references the Sourcefile +% TargetPath is the storage location, ID the foldername +% Status returns true if the function was sucessfull plist_id = 'H5P_DEFAULT'; -% Work in Progress... Problem mit dem Pfad... - -try - fid = H5F.create(fullfile(TargetPath,[ID,'_data.h5'])); +% try + fid = H5F.create(fullfile(TargetPath,ID,[ID,'_data.h5'])); %create External Link to Sourcefile in the Group linkToExternal - H5L.create_external(SourceFile,'/',fid,'linkToExternal',plist_id,plist_id); + H5L.create_external(['..\data\',SourceFile],'/',fid, SourceFile ,plist_id,plist_id); H5F.close(fid); status = 1; -catch - warning('No linked HDF file was created'); - status = 0; -end +% catch +% warning('No linked HDF file was created'); +% status = 0; +% end + end diff --git a/fcn_help/fileCompare.m b/fcn_help/fileCompare.m new file mode 100644 index 0000000000000000000000000000000000000000..c9007e704eb7a8ff8870830bc2d6610dce185107 --- /dev/null +++ b/fcn_help/fileCompare.m @@ -0,0 +1,49 @@ +function [status, id] = fileCompare(filename,fileList) +%fileCompare checks if file1 is (binary) identical to a file in filelist +% it returns a sttus and the id of the identical file +% the functions uses the windows system function fc or the unix function +% diff + +if isempty(fileList) + % no comparison necessary + status =false; + id = 0; + return +end + +[~,~,ext1] = fileparts(filename); +id = zeros(height(fileList),1); + +for i=1:height(fileList) + [~,~,ext2] = fileparts(fileList.name{i}); + + if ~isequal(ext1,ext2) + %warning('File extension are not identical'); + status = false; + continue + end + + filepath = fullfile(fileList.folder{i},fileList.name{i}); + if ispc + [status,~] = system(['fc ' filename ' ' filepath]); + % 0 -> identical, 1 -> not identical + status = ~status; % false (not identical), true(identical) + + elseif isunix %untested! + [status,~] = system(['diff ' filename ' ' filepath]); + else + warning('Platform not supported') + end + + if status == 1 + id(i) = 1; + else + % Status can also be any other number e.g. 2 + id(i) = 0; + end + + id =logical(id); %bugfix +end + +end + diff --git a/fcn_help/removePltIdFiles.m b/fcn_help/removePltIdFiles.m new file mode 100644 index 0000000000000000000000000000000000000000..3080c22985474f8d2227e2bd62c17ecbd6f3163f --- /dev/null +++ b/fcn_help/removePltIdFiles.m @@ -0,0 +1,19 @@ +function [fListClean] = removePltIdFiles(fList) +%removePltIdFiles removes functions that are part of PlotID out of flist +% Detailed explanation goes here +%addpath('..\fcn_core'); + +[~,names,ext] = fileparts(fList); +names = strcat(names, ext); % add ext for comparison + +% Get a list of all .m files that are part of Plot id +PltID_flist = struct2table([dir('fcn_help'); dir('fcn_core')]); %get list of files +[~,~,PltID_flist.ext(:)] = fileparts(PltID_flist.name(:)); % add ext column + +PltID_flist = PltID_flist(strcmp(PltID_flist.ext,'.m'),:); + +% Comparison and filter +fListClean = fList(~ismember(names,PltID_flist.name)); + +end + diff --git a/fcn_help/replaceLinkedHDF5.m b/fcn_help/replaceLinkedHDF5.m new file mode 100644 index 0000000000000000000000000000000000000000..e84e49f6e3aeed36d24f59b50288984fbe4d3998 --- /dev/null +++ b/fcn_help/replaceLinkedHDF5.m @@ -0,0 +1,61 @@ +function replaceLinkedHDF5(linkedFilepath) +% replaceLinkedHDF5 replaces the HDF5, that contains only a link to another +% HDF5 File with a file containing the actual Data, thus turning it into an +% independent data file. It essentially turns the plotID option +% 'centralized' into 'individual'. +% Filepath is the location of the File that contains the link. The +% filepath to the linked data has to be present in the file itself. + + + +%% Check whether the only Objects present are links +h5inf = h5info(linkedFilepath); + +% default behaviour dont overwrite +overwrite = false; + +if isempty([h5inf.Groups, h5inf.Datasets, h5inf.Datatypes, h5inf.Attributes]) && ... + all(size(h5inf.Links) == [1 1]) + % There are no other known objects present and + % there is only one linked file + overwrite = true; +end + +%% Create Copies of files +for i = 1:size(h5inf.Links, 2) + % This is the file with full data + linkTargetPath = h5inf.Links(i).Value{1,1}; + % This is the empty file + % TODO filename decision + linkSourcePath = strcat(linkedFilepath,'_',num2str(i)); + + % Create an absolute path from the relative linking path + absoluteTP = regexp(linkSourcePath, filesep, 'split'); + TargetPathSplit = regexp(linkTargetPath, filesep, 'split'); + % How many levels do we need to go up in the relative path + % This is merely counting the occurances of '..' within the path, but + % should not cause issues if the realtive path was created by the + % linking process + levelsUp = sum(contains(TargetPathSplit,'..')); + absoluteTP = absoluteTP(1:(end-(1+levelsUp)) ); %Remove filename + levels + linkTargetPath = strjoin([absoluteTP, TargetPathSplit(~contains(TargetPathSplit,'..'))],filesep); + + if ~isfile(linkTargetPath) + warning(strcat("Link destination """,linkTargetPath,""" does not exist or relative path does not fit, no copy created")) + end + + if overwrite % can only be for i=1, and size = 1 + linkSourcePath = linkedFilepath; + end + disp('Copying file from ', linkSourcePath, ' to ', linkTargetPath) + copyfile(linkTargetPath,linkSourcePath); + +end + + + +end + + + + diff --git a/test_data.mat b/test_data.mat deleted file mode 100644 index 06ca1639f8c3febb925fb5bbdc67b1953a894ab6..0000000000000000000000000000000000000000 Binary files a/test_data.mat and /dev/null differ diff --git a/testdata_2.h5 b/testdata_2.h5 deleted file mode 100644 index efc071f84eaf0f39c147f2e74f80c4666ff22467..0000000000000000000000000000000000000000 Binary files a/testdata_2.h5 and /dev/null differ