From 009ef3da93c8b8432fc99e10c36a2688142c719e Mon Sep 17 00:00:00 2001
From: David Schimmel <David.schimmel@gmx.de>
Date: Thu, 28 Jan 2021 11:06:34 +0100
Subject: [PATCH 1/2] Update: extend analytics logs (coscine/issues#1221)
---
src/Tree/App.config | 8 +++--
src/Tree/Controllers/TreeController.cs | 44 +++++++++++++++++++++++---
src/Tree/Tree.csproj | 12 +++----
src/Tree/packages.config | 4 +--
4 files changed, 54 insertions(+), 14 deletions(-)
diff --git a/src/Tree/App.config b/src/Tree/App.config
index 20153e8..f2d19a6 100644
--- a/src/Tree/App.config
+++ b/src/Tree/App.config
@@ -149,7 +149,7 @@
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Coscine.Database" publicKeyToken="767d77427707b70a" culture="neutral" />
- <bindingRedirect oldVersion="0.0.0.0-1.27.0.0" newVersion="1.27.0.0" />
+ <bindingRedirect oldVersion="0.0.0.0-1.28.0.0" newVersion="1.28.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Extensions.Options" publicKeyToken="adb9793829ddae60" culture="neutral" />
@@ -205,7 +205,11 @@
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Coscine.Database.T4" publicKeyToken="84b4c404a0696261" culture="neutral" />
- <bindingRedirect oldVersion="0.0.0.0-1.27.0.0" newVersion="1.27.0.0" />
+ <bindingRedirect oldVersion="0.0.0.0-1.28.0.0" newVersion="1.28.0.0" />
+ </dependentAssembly>
+ <dependentAssembly>
+ <assemblyIdentity name="Coscine.Logging" publicKeyToken="e1ed402bc3f6525e" culture="neutral" />
+ <bindingRedirect oldVersion="0.0.0.0-1.3.0.0" newVersion="1.3.0.0" />
</dependentAssembly>
</assemblyBinding>
</runtime>
diff --git a/src/Tree/Controllers/TreeController.cs b/src/Tree/Controllers/TreeController.cs
index 71f7488..eeaee87 100644
--- a/src/Tree/Controllers/TreeController.cs
+++ b/src/Tree/Controllers/TreeController.cs
@@ -145,7 +145,23 @@ namespace Coscine.Api.Tree.Controllers
))
);
- LogAnalytics("View MD", resourceId, path, user);
+ if (resource.ApplicationProfile[resource.ApplicationProfile.Length - 1] != '/')
+ {
+ resource.ApplicationProfile += '/';
+ }
+ string metadataCompleteness = null;
+ try
+ {
+ var metadata = jObject.SelectToken("data").SelectToken("metadataStorage").First.First.First;
+ var shapesGraph = (Graph)_rdfStoreConnector.GetGraph(resource.ApplicationProfile);
+ metadataCompleteness = GetMetadataCompleteness(metadata, shapesGraph);
+ }
+ catch (Exception e)
+ {
+ metadataCompleteness = "Error";
+ }
+
+ LogAnalytics("Update MD", resourceId, path, user, metadataCompleteness);
return Json(jObject);
}
catch
@@ -258,12 +274,12 @@ namespace Coscine.Api.Tree.Controllers
if (_rdfStoreConnector.HasGraph(graphNameUri))
{
_rdfStoreConnector.ClearGraph(graphNameUri);
- LogAnalytics("Update MD", resourceId, path, user);
+ LogAnalytics("Update MD", resourceId, path, user, GetMetadataCompleteness(innerBlock, shapesGraph));
}
else
{
_rdfStoreConnector.CreateNamedGraph(graphNameUri);
- LogAnalytics("Upload MD", resourceId, path, user);
+ LogAnalytics("Upload MD", resourceId, path, user, GetMetadataCompleteness(innerBlock, shapesGraph));
}
// BaseUri must be set for the sparql query
@@ -330,7 +346,8 @@ namespace Coscine.Api.Tree.Controllers
/// <param name="resourceId">Resource of the id</param>
/// <param name="filename">Name of the file</param>
/// <param name="user">User object</param>
- private void LogAnalytics(string operation, string resourceId, string filename, User user)
+ /// <param name="metadataCompleteness">Metadata Completeness</param>
+ private void LogAnalytics(string operation, string resourceId, string filename, User user, string metadataCompleteness = null)
{
if (CoscineLoggerConfiguration.IsLogLevelActivated(LogType.Analytics))
{
@@ -340,8 +357,27 @@ namespace Coscine.Api.Tree.Controllers
_analyticsLogObject.ResourceId = resourceId;
_analyticsLogObject.ProjectId = _projectResourceModel.GetProjectForResource(new Guid(resourceId)).ToString();
_analyticsLogObject.RoleId = _projectRoleModel.GetGetUserRoleForProject(new Guid(_analyticsLogObject.ProjectId), user.Id).ToString();
+ _analyticsLogObject.ApplicationsProfile = _resourceModel.CreateReturnObjectFromDatabaseObject(_resourceModel.GetById(new Guid(resourceId))).ApplicationProfile;
+ _analyticsLogObject.MetadataCompleteness = metadataCompleteness;
_coscineLogger.AnalyticsLog(_analyticsLogObject);
}
}
+
+ /// <summary>
+ /// Gets metadata completeness by comparing submitted values to the application profile specific shape graph
+ /// </summary>
+ /// <param name="valueTokens"></param>
+ /// <param name="shapesGraph"></param>
+ /// <returns></returns>
+ private string GetMetadataCompleteness (JToken valueTokens, IGraph shapesGraph)
+ {
+ NodeFactory nodeFactory = new NodeFactory();
+ UriNode uriNode = (UriNode)nodeFactory.CreateUriNode(new Uri("http://www.w3.org/ns/shacl#property"));
+
+ int total = shapesGraph.GetTriplesWithPredicate(uriNode).Count();
+ int present = valueTokens.Count();
+
+ return $"{present}/{total}";
+ }
}
}
diff --git a/src/Tree/Tree.csproj b/src/Tree/Tree.csproj
index 5d3b7b8..fdd1ec3 100644
--- a/src/Tree/Tree.csproj
+++ b/src/Tree/Tree.csproj
@@ -56,17 +56,17 @@
<Reference Include="Coscine.Configuration, Version=1.5.0.0, Culture=neutral, PublicKeyToken=ce3d7a32d7dc1e5a, processorArchitecture=MSIL">
<HintPath>..\packages\Coscine.Configuration.1.5.0\lib\net461\Coscine.Configuration.dll</HintPath>
</Reference>
- <Reference Include="Coscine.Database, Version=1.27.0.0, Culture=neutral, PublicKeyToken=767d77427707b70a, processorArchitecture=MSIL">
- <HintPath>..\packages\Coscine.Database.1.27.0\lib\net461\Coscine.Database.dll</HintPath>
+ <Reference Include="Coscine.Database, Version=1.28.0.0, Culture=neutral, PublicKeyToken=767d77427707b70a, processorArchitecture=MSIL">
+ <HintPath>..\packages\Coscine.Database.1.28.0-topic-1221-loggi0005\lib\net461\Coscine.Database.dll</HintPath>
</Reference>
- <Reference Include="Coscine.Database.T4, Version=1.27.0.0, Culture=neutral, PublicKeyToken=84b4c404a0696261, processorArchitecture=MSIL">
- <HintPath>..\packages\Coscine.Database.1.27.0\lib\net461\Coscine.Database.T4.dll</HintPath>
+ <Reference Include="Coscine.Database.T4, Version=1.28.0.0, Culture=neutral, PublicKeyToken=84b4c404a0696261, processorArchitecture=MSIL">
+ <HintPath>..\packages\Coscine.Database.1.28.0-topic-1221-loggi0005\lib\net461\Coscine.Database.T4.dll</HintPath>
</Reference>
<Reference Include="Coscine.JwtHandler, Version=1.2.0.0, Culture=neutral, PublicKeyToken=aaacf41df3a6253c, processorArchitecture=MSIL">
<HintPath>..\packages\Coscine.JwtHandler.1.2.0\lib\net461\Coscine.JwtHandler.dll</HintPath>
</Reference>
- <Reference Include="Coscine.Logging, Version=1.2.0.0, Culture=neutral, PublicKeyToken=e1ed402bc3f6525e, processorArchitecture=MSIL">
- <HintPath>..\packages\Coscine.Logging.1.2.0\lib\net461\Coscine.Logging.dll</HintPath>
+ <Reference Include="Coscine.Logging, Version=1.3.0.0, Culture=neutral, PublicKeyToken=e1ed402bc3f6525e, processorArchitecture=MSIL">
+ <HintPath>..\packages\Coscine.Logging.1.3.0-topic-1221-loggi0001\lib\net461\Coscine.Logging.dll</HintPath>
</Reference>
<Reference Include="Coscine.Metadata, Version=1.5.0.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\Coscine.Metadata.1.5.0\lib\net461\Coscine.Metadata.dll</HintPath>
diff --git a/src/Tree/packages.config b/src/Tree/packages.config
index 5cb7fdd..34afa10 100644
--- a/src/Tree/packages.config
+++ b/src/Tree/packages.config
@@ -3,9 +3,9 @@
<package id="Consul" version="0.7.2.6" targetFramework="net461" />
<package id="Coscine.ApiCommons" version="1.11.0" targetFramework="net461" />
<package id="Coscine.Configuration" version="1.5.0" targetFramework="net461" />
- <package id="Coscine.Database" version="1.27.0" targetFramework="net461" />
+ <package id="Coscine.Database" version="1.28.0-topic-1221-loggi0005" targetFramework="net461" />
<package id="Coscine.JwtHandler" version="1.2.0" targetFramework="net461" />
- <package id="Coscine.Logging" version="1.2.0" targetFramework="net461" />
+ <package id="Coscine.Logging" version="1.3.0-topic-1221-loggi0001" targetFramework="net461" />
<package id="Coscine.Metadata" version="1.5.0" targetFramework="net461" />
<package id="Coscine.ResourceConfiguration" version="1.3.0" targetFramework="net461" />
<package id="Coscine.ResourceLoader" version="1.2.0" targetFramework="net461" />
--
GitLab
From ef3fe5d4fe2ba6cc3bc6a8752d946159463d6bdb Mon Sep 17 00:00:00 2001
From: David Schimmel <David.schimmel@gmx.de>
Date: Mon, 1 Feb 2021 14:25:00 +0100
Subject: [PATCH 2/2] add check for empty metadata
---
src/Tree/Controllers/TreeController.cs | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/src/Tree/Controllers/TreeController.cs b/src/Tree/Controllers/TreeController.cs
index eeaee87..2d0d7a0 100644
--- a/src/Tree/Controllers/TreeController.cs
+++ b/src/Tree/Controllers/TreeController.cs
@@ -152,7 +152,16 @@ namespace Coscine.Api.Tree.Controllers
string metadataCompleteness = null;
try
{
- var metadata = jObject.SelectToken("data").SelectToken("metadataStorage").First.First.First;
+ JToken metadata;
+ if (jObject.SelectToken("data").SelectToken("metadataStorage").Count() > 0)
+ {
+ metadata = jObject.SelectToken("data").SelectToken("metadataStorage").First.First.First;
+ }
+ else
+ {
+ metadata = jObject.SelectToken("data").SelectToken("metadataStorage");
+ }
+
var shapesGraph = (Graph)_rdfStoreConnector.GetGraph(resource.ApplicationProfile);
metadataCompleteness = GetMetadataCompleteness(metadata, shapesGraph);
}
--
GitLab