Skip to content
Snippets Groups Projects
Commit 43e70333 authored by Petar Hristov's avatar Petar Hristov :speech_balloon:
Browse files

Merge branch 'dev' into 'main' 

Release: Sprint/2023 04 :robot:

See merge request !2
parents dd009898 56acc99a
Branches
No related tags found
1 merge request!2Release: Sprint/2023 04 :robot:
Pipeline #924605 passed
Stage: build
Stage: publish
Stage: commands
Show whitespace changes
Inline Side-by-side
src/MetadataMigrator/Implementations/ResourceStructuralData.cs
+ 35
− 262
View file @ 43e70333
...@@ -2,43 +2,18 @@ using Coscine.Database.DataModel; ...@@ -2,43 +2,18 @@ using Coscine.Database.DataModel;
using Coscine.Database.Models; using Coscine.Database.Models;
using Coscine.Database.Util; using Coscine.Database.Util;
using Coscine.Metadata; using Coscine.Metadata;
using Coscine.Metadata.Util;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using System.Globalization;
using VDS.RDF; using VDS.RDF;
using VDS.RDF.Parsing;
using VDS.RDF.Query; using VDS.RDF.Query;
namespace MetadataMigrator.Implementations namespace MetadataMigrator.Implementations
{ {
public class ResourceStructuralData : StructuralData<Resource, ResourceModel> public class ResourceStructuralData : StructuralData<Resource, ResourceModel>
{ {
private readonly string partOfUri = "http://purl.org/dc/terms/isPartOf";
private readonly string aUri = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type";
private readonly string basicContainerUri = "http://www.w3.org/ns/ldp#BasicContainer";
private readonly string rdfSourceUri = "http://www.w3.org/ns/ldp#RDFSource";
private readonly string dcatcatalogUri = "http://www.w3.org/ns/dcat#catalog";
private readonly string dcatCatalogClassUri = "http://www.w3.org/ns/dcat#Catalog";
private readonly string dctermsIdentifierUri = "http://purl.org/dc/terms/identifier";
private readonly string dctermsModifiedUri = "http://purl.org/dc/terms/modified";
private readonly string fdpMetadataServiceUri = "http://purl.org/fdp/fdp-o#MetadataService";
private readonly string fdphasMetadataUri = "http://purl.org/fdp/fdp-o#hasMetadata";
private readonly string fdpisMetadataOfUri = "http://purl.org/fdp/fdp-o#isMetadataOf";
private readonly string provEntityUri = "http://www.w3.org/ns/prov#Entity";
private readonly string provGeneratedAtTimeUri = "http://www.w3.org/ns/prov#generatedAtTime";
private readonly string provWasRevisionOfUri = "http://www.w3.org/ns/prov#wasRevisionOfNode";
private readonly string ldpDescribedByUri = "http://www.w3.org/ns/ldp#describedBy";
private readonly string coscineLinkedBodyUri = "https://purl.org/coscine/terms/linked#body";
private readonly Dictionary<string, string> _targetClassMap = new Dictionary<string, string>(); private readonly Dictionary<string, string> _targetClassMap = new Dictionary<string, string>();
private readonly int QUERY_LIMIT = 1000; private readonly int QUERY_LIMIT = 1000;
// Override to also receive deleted resources // Override to also receive deleted resources
public override IEnumerable<Resource> GetAll() public override IEnumerable<Resource> GetAll()
{ {
...@@ -67,13 +42,19 @@ namespace MetadataMigrator.Implementations ...@@ -67,13 +42,19 @@ namespace MetadataMigrator.Implementations
/// <returns></returns> /// <returns></returns>
public override IEnumerable<IGraph> ConvertToLinkedData(IEnumerable<Resource> entries) public override IEnumerable<IGraph> ConvertToLinkedData(IEnumerable<Resource> entries)
{ {
var trellisGraph = WrapRequest(() => GetNewGraph("http://www.trellisldp.org/ns/trellis#PreferServerManaged")); var trellisGraph = RdfStoreConnector.GetEmptySmallUpdateGraph(Uris.TrellisGraph);
var trellisIds = GetTrellisIds(); var trellisIds = GetTrellisIds();
var leftGraphs = new List<IGraph> { trellisGraph }; var leftGraphs = new List<IGraph> { trellisGraph };
var resourceUrlPrefix = "https://purl.org/coscine/resources"; var resourceUrlPrefix = "https://purl.org/coscine/resources";
var resourceUrlHandlePrefix = $"https://hdl.handle.net/{Prefix}"; var resourceUrlHandlePrefix = $"https://hdl.handle.net/{Prefix}";
var metadataGraphCreator = new MetadataGraphsCreator(RdfStoreConnector);
var maxVersionScanning = long.Parse(
Configuration.GetStringAndWait("coscine/global/metadata/migration/maxVersion", long.MaxValue + "")
);
var graphs = new List<IGraph>(); var graphs = new List<IGraph>();
foreach (var entry in entries) foreach (var entry in entries)
...@@ -95,8 +76,8 @@ namespace MetadataMigrator.Implementations ...@@ -95,8 +76,8 @@ namespace MetadataMigrator.Implementations
{ {
Logger.LogInformation($"Migrating {fileGraph.AbsoluteUri}"); Logger.LogInformation($"Migrating {fileGraph.AbsoluteUri}");
var fileGraphObject = WrapRequest(() => GetGraph(fileGraph)); var fileGraphObject = RdfStoreConnector.GetGraph(fileGraph);
IGraph? linkedGraph = WrapRequest(() => GetGraph(fileGraph.AbsoluteUri + "&data")); IGraph? linkedGraph = RdfStoreConnector.GetGraph(fileGraph.AbsoluteUri + "&data");
if (linkedGraph.IsEmpty) if (linkedGraph.IsEmpty)
{ {
linkedGraph = null; linkedGraph = null;
...@@ -127,6 +108,7 @@ namespace MetadataMigrator.Implementations ...@@ -127,6 +108,7 @@ namespace MetadataMigrator.Implementations
} }
var recentMetadataVersion = VersionUtil.GetRecentMetadataVersion(existingGraphs); var recentMetadataVersion = VersionUtil.GetRecentMetadataVersion(existingGraphs);
var recentMetadataVersionLong = VersionUtil.GetVersion(recentMetadataVersion);
var currentMetadataVersionGraph = GetRecentVersionGraph(newMetadataVersionFileGraphName, recentMetadataVersion); var currentMetadataVersionGraph = GetRecentVersionGraph(newMetadataVersionFileGraphName, recentMetadataVersion);
if (fileGraphObject != null) if (fileGraphObject != null)
...@@ -136,25 +118,33 @@ namespace MetadataMigrator.Implementations ...@@ -136,25 +118,33 @@ namespace MetadataMigrator.Implementations
(triple.Object.Equals(otherTriple.Object) || (triple.Object.NodeType == NodeType.Blank && otherTriple.Object.NodeType == NodeType.Blank)))); (triple.Object.Equals(otherTriple.Object) || (triple.Object.NodeType == NodeType.Blank && otherTriple.Object.NodeType == NodeType.Blank))));
} }
if (recentMetadataVersionLong != null && recentMetadataVersionLong > maxVersionScanning)
{
Logger.LogInformation($"New Version too recent (after migration)!");
continue;
}
if (notChanged) if (notChanged)
{ {
Logger.LogInformation($"No change for {fileGraph.AbsoluteUri}, skipping!"); Logger.LogInformation($"No change for {fileGraph.AbsoluteUri}, skipping!");
continue; continue;
} }
var newFileGraph = GetNewGraph(newFileGraphName); graphs.AddRange(
var fileNode = newFileGraph.CreateUriNode(new Uri(newFileGraphName)); metadataGraphCreator.CreateGraphs(entry.Id.ToString(), path, true, true, new ProvidedGraphData {
graphs.Add(newFileGraph); CurrentDataVersionGraph = currentDataVersionGraph,
CurrentMetadataVersionGraph = currentMetadataVersionGraph,
newFileGraph.Assert(new Triple(fileNode, newFileGraph.CreateUriNode(new Uri(aUri)), newFileGraph.CreateUriNode(new Uri(dcatCatalogClassUri)))); OldDataGraph = linkedGraph,
newFileGraph.Assert(new Triple(fileNode, newFileGraph.CreateUriNode(new Uri(aUri)), newFileGraph.CreateUriNode(new Uri(fdpMetadataServiceUri)))); OldMetadataGraph = fileGraphObject,
AddFilesToAFolder(trellisGraph, new Uri(newFileGraphName), resourceGraphName, graphs);
var metadataFileGraph = SetMetadataGraph(trellisGraph, graphs, newMetadataFileGraphName, newFileGraph, newFileGraphName); RecentDataVersion = recentDataVersion,
var dataFileGraph = SetDataGraph(trellisGraph, graphs, newDataFileGraphName, newFileGraph, newFileGraphName, metadataFileGraph.BaseUri.AbsoluteUri); RecentMetadataVersion = recentMetadataVersion,
SetDataVersionGraph(graphs, recentDataVersion, currentDataVersionGraph, linkedGraph, newDataVersionFileGraphName, dataFileGraph, trellisGraph, newFileGraphName); TrellisGraph = trellisGraph
SetMetadataVersionGraph(graphs, recentMetadataVersion, currentMetadataVersionGraph, fileGraphObject, newMetadataVersionFileGraphName, metadataFileGraph, trellisGraph, newFileGraphName, currentDataVersionGraph); })
);
graphs.RemoveAll((graph) => graph.BaseUri == Uris.TrellisGraph);
} }
// Intermediate update to save memory // Intermediate update to save memory
...@@ -174,185 +164,19 @@ namespace MetadataMigrator.Implementations ...@@ -174,185 +164,19 @@ namespace MetadataMigrator.Implementations
return leftGraphs; return leftGraphs;
} }
private void AddFilesToAFolder(IGraph trellisGraph, Uri fileGraph, string resourceGraphName, List<IGraph> graphs)
{
string rootUri;
var fileGraphUri = fileGraph.AbsoluteUri;
do
{
rootUri = fileGraphUri[..fileGraphUri.LastIndexOf("/")];
var rootGraph = GetGraph(rootUri);
rootGraph.Assert(new Triple(rootGraph.CreateUriNode(new Uri(rootUri)), rootGraph.CreateUriNode(new Uri(dcatcatalogUri)), rootGraph.CreateUriNode(fileGraph)));
AddToTrellis(trellisGraph, basicContainerUri, rootUri, fileGraphUri);
graphs.Add(rootGraph);
fileGraphUri = rootUri;
} while (rootUri != resourceGraphName && rootUri.Contains("/"));
}
private IGraph SetMetadataGraph(IGraph trellisGraph, List<IGraph> graphs, string newMetadataFileGraphName, IGraph newFileGraph, string fileUri)
{
var metadataFileNode = newFileGraph.CreateUriNode(new Uri(newMetadataFileGraphName));
var metadataFileGraph = GetNewGraph(newMetadataFileGraphName);
graphs.Add(metadataFileGraph);
AddToTrellis(trellisGraph, rdfSourceUri, fileUri, newMetadataFileGraphName);
var fileNode = newFileGraph.CreateUriNode(new Uri(fileUri));
newFileGraph.Assert(new Triple(fileNode, newFileGraph.CreateUriNode(new Uri(dcatcatalogUri)), metadataFileNode));
newFileGraph.Assert(new Triple(fileNode, newFileGraph.CreateUriNode(new Uri(fdphasMetadataUri)), metadataFileNode));
metadataFileGraph.Assert(new Triple(
Tools.CopyNode(metadataFileNode, metadataFileGraph),
metadataFileGraph.CreateUriNode(new Uri(aUri)),
metadataFileGraph.CreateUriNode(new Uri(dcatCatalogClassUri))
));
return metadataFileGraph;
}
private IGraph SetDataGraph(IGraph trellisGraph, List<IGraph> graphs, string newDataFileGraphName, IGraph newFileGraph, string fileUri, string metadataFileUri)
{
var dataFileNode = newFileGraph.CreateUriNode(new Uri(newDataFileGraphName));
var dataFileGraph = GetNewGraph(newDataFileGraphName);
graphs.Add(dataFileGraph);
AddToTrellis(trellisGraph, rdfSourceUri, fileUri, newDataFileGraphName);
var fileNode = newFileGraph.CreateUriNode(new Uri(fileUri));
newFileGraph.Assert(new Triple(fileNode, newFileGraph.CreateUriNode(new Uri(dcatcatalogUri)), dataFileNode));
dataFileGraph.Assert(new Triple(Tools.CopyNode(dataFileNode, dataFileGraph), dataFileGraph.CreateUriNode(new Uri(aUri)), dataFileGraph.CreateUriNode(new Uri(dcatCatalogClassUri))));
dataFileGraph.Assert(new Triple(Tools.CopyNode(dataFileNode, dataFileGraph), dataFileGraph.CreateUriNode(new Uri(ldpDescribedByUri)), dataFileGraph.CreateUriNode(new Uri(metadataFileUri))));
return dataFileGraph;
}
private void SetDataVersionGraph(List<IGraph> graphs, Uri? recentDataVersion, IGraph currentDataVersionGraph, IGraph? linkedGraph, string newDataVersionFileGraphName, IGraph dataFileGraph, IGraph trellisGraph, string fileUri)
{
SetLinkedData(linkedGraph, currentDataVersionGraph, new Uri(newDataVersionFileGraphName));
var currentDataVersionNode = currentDataVersionGraph.CreateUriNode(currentDataVersionGraph.BaseUri);
currentDataVersionGraph.Assert(new Triple(currentDataVersionNode, currentDataVersionGraph.CreateUriNode(new Uri(dctermsIdentifierUri)), currentDataVersionGraph.CreateLiteralNode(
currentDataVersionGraph.BaseUri.AbsoluteUri,
new Uri(XmlSpecsHelper.XmlSchemaDataTypeString)
)));
currentDataVersionGraph.Assert(new Triple(
currentDataVersionNode,
currentDataVersionGraph.CreateUriNode(new Uri(aUri)),
currentDataVersionGraph.CreateUriNode(new Uri("http://www.w3.org/ns/dcat#Dataset"))
));
// PROV Info
var provTriple = new Triple(Tools.CopyNode(currentDataVersionNode, dataFileGraph), dataFileGraph.CreateUriNode(new Uri(aUri)), dataFileGraph.CreateUriNode(new Uri(provEntityUri)));
dataFileGraph.Assert(provTriple);
dataFileGraph.Assert(new Triple(
dataFileGraph.CreateUriNode(dataFileGraph.BaseUri),
dataFileGraph.CreateUriNode(new Uri("http://www.w3.org/ns/dcat#dataset")),
Tools.CopyNode(currentDataVersionNode, dataFileGraph)
));
dataFileGraph.Assert(new Triple(Tools.CopyNode(currentDataVersionNode, dataFileGraph), dataFileGraph.CreateUriNode(new Uri(provGeneratedAtTimeUri)), dataFileGraph.CreateLiteralNode(
DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture),
new Uri(XmlSpecsHelper.XmlSchemaDataTypeDateTime)
)));
AddToTrellis(trellisGraph, rdfSourceUri, fileUri, currentDataVersionGraph.BaseUri.AbsoluteUri);
if (recentDataVersion != null && recentDataVersion.AbsoluteUri != currentDataVersionGraph.BaseUri.AbsoluteUri)
{
var recentDataVersionNode = dataFileGraph.CreateUriNode(recentDataVersion);
dataFileGraph.Assert(new Triple(Tools.CopyNode(currentDataVersionNode, dataFileGraph), dataFileGraph.CreateUriNode(new Uri(provWasRevisionOfUri)), recentDataVersionNode));
}
graphs.Add(currentDataVersionGraph);
}
private void SetMetadataVersionGraph(List<IGraph> graphs, Uri? recentMetadataVersion, IGraph currentMetadataVersionGraph, IGraph? metadataGraph, string newMetadataVersionFileGraphName, IGraph metadataFileGraph, IGraph trellisGraph, string fileUri, IGraph currentDataVersionGraph)
{
SetMetadata(metadataGraph, currentMetadataVersionGraph, new Uri(newMetadataVersionFileGraphName));
var currentMetadataVersionNode = currentMetadataVersionGraph.CreateUriNode(currentMetadataVersionGraph.BaseUri);
// PROV Info
var provTriple = new Triple(Tools.CopyNode(currentMetadataVersionNode, metadataFileGraph), metadataFileGraph.CreateUriNode(new Uri(aUri)), metadataFileGraph.CreateUriNode(new Uri(provEntityUri)));
metadataFileGraph.Assert(provTriple);
metadataFileGraph.Assert(new Triple(
metadataFileGraph.CreateUriNode(metadataFileGraph.BaseUri),
metadataFileGraph.CreateUriNode(new Uri(fdphasMetadataUri)),
Tools.CopyNode(currentMetadataVersionNode, metadataFileGraph)
));
metadataFileGraph.Assert(new Triple(Tools.CopyNode(currentMetadataVersionNode, metadataFileGraph), metadataFileGraph.CreateUriNode(new Uri(provGeneratedAtTimeUri)), metadataFileGraph.CreateLiteralNode(
DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture),
new Uri(XmlSpecsHelper.XmlSchemaDataTypeDateTime)
)));
metadataFileGraph.Assert(new Triple(
metadataFileGraph.CreateUriNode(currentMetadataVersionGraph.BaseUri),
metadataFileGraph.CreateUriNode(new Uri(fdpisMetadataOfUri)),
metadataFileGraph.CreateUriNode(currentDataVersionGraph.BaseUri)
));
AddToTrellis(trellisGraph, rdfSourceUri, fileUri, currentMetadataVersionGraph.BaseUri.AbsoluteUri);
if (recentMetadataVersion != null && recentMetadataVersion.AbsoluteUri != currentMetadataVersionGraph.BaseUri.AbsoluteUri)
{
var recentMetadataVersionNode = metadataFileGraph.CreateUriNode(recentMetadataVersion);
metadataFileGraph.Assert(new Triple(Tools.CopyNode(currentMetadataVersionNode, metadataFileGraph), metadataFileGraph.CreateUriNode(new Uri(provWasRevisionOfUri)), recentMetadataVersionNode));
}
graphs.Add(currentMetadataVersionGraph);
}
private IGraph GetRecentVersionGraph(string newVersionFileGraphName, Uri? recentVersion) private IGraph GetRecentVersionGraph(string newVersionFileGraphName, Uri? recentVersion)
{ {
if (recentVersion == null) if (recentVersion == null)
{ {
return new WrapperGraph() return new Graph()
{ {
BaseUri = new Uri(newVersionFileGraphName), BaseUri = new Uri(newVersionFileGraphName),
}; };
} }
else else
{ {
return WrapRequest(() => GetGraph(recentVersion)); return RdfStoreConnector.GetGraph(recentVersion);
}
}
private void SetLinkedData(IGraph? linkedGraph, IGraph currentDataVersionGraph, Uri otherURICandidate)
{
var linkedBody = currentDataVersionGraph.GetTriplesWithPredicate(new Uri(coscineLinkedBodyUri));
if (linkedGraph != null)
{
var setLinkedBody = linkedGraph.GetTriplesWithPredicate(new Uri(coscineLinkedBodyUri));
if (!linkedBody.Any()
|| linkedBody.First().Object.ToString()
!= setLinkedBody.First().Object.ToString())
{
var oldDataVersionNode = currentDataVersionGraph.CreateUriNode(currentDataVersionGraph.BaseUri);
var currentDataVersionNode = currentDataVersionGraph.CreateUriNode(otherURICandidate);
currentDataVersionGraph.BaseUri = otherURICandidate;
foreach (var currentLinkedBody in linkedBody)
{
currentDataVersionGraph.Retract(currentLinkedBody);
}
foreach (var oldTriple in currentDataVersionGraph.GetTriplesWithSubject(oldDataVersionNode))
{
currentDataVersionGraph.Retract(oldTriple);
currentDataVersionGraph.Assert(new Triple(currentDataVersionNode, Tools.CopyNode(oldTriple.Predicate, currentDataVersionGraph), Tools.CopyNode(oldTriple.Object, currentDataVersionGraph)));
}
foreach (var currentSetLinkedBody in setLinkedBody)
{
currentDataVersionGraph.Assert(new Triple(currentDataVersionNode, currentDataVersionGraph.CreateUriNode(new Uri(coscineLinkedBodyUri)), Tools.CopyNode(currentSetLinkedBody.Object, currentDataVersionGraph)));
}
}
}
}
private static void SetMetadata(IGraph? metadataGraph, IGraph currentMetadataVersionGraph, Uri otherURICandidate)
{
if (metadataGraph != null)
{
// If the predicate & object pairs are different
if (!
(metadataGraph.Triples.All((triple) => currentMetadataVersionGraph.GetTriplesWithPredicateObject(triple.Predicate, triple.Object).Any())
&& currentMetadataVersionGraph.Triples.All((triple) => metadataGraph.GetTriplesWithPredicateObject(triple.Predicate, triple.Object).Any())))
{
currentMetadataVersionGraph.Retract(currentMetadataVersionGraph.Triples);
currentMetadataVersionGraph.BaseUri = otherURICandidate;
var currentMetadataVersionNode = currentMetadataVersionGraph.CreateUriNode(otherURICandidate);
foreach (var triple in metadataGraph.Triples)
{
currentMetadataVersionGraph.Assert(currentMetadataVersionNode, Tools.CopyNode(triple.Predicate, currentMetadataVersionGraph), Tools.CopyNode(triple.Object, currentMetadataVersionGraph));
}
}
} }
} }
...@@ -365,40 +189,6 @@ namespace MetadataMigrator.Implementations ...@@ -365,40 +189,6 @@ namespace MetadataMigrator.Implementations
return path; return path;
} }
private void AddToTrellis(IGraph trellisGraph, string ldpAssignment, string thePartUri, string graphUri)
{
var setGraphNode = trellisGraph.CreateUriNode(new Uri(graphUri));
var setThePartNode = trellisGraph.CreateUriNode(new Uri(thePartUri));
var triple = new Triple(
setGraphNode,
trellisGraph.CreateUriNode(new Uri(partOfUri)),
setThePartNode
);
if (!trellisGraph.ContainsTriple(triple))
{
trellisGraph.Assert(triple);
trellisGraph.Assert(new Triple(
setGraphNode,
trellisGraph.CreateUriNode(new Uri(aUri)),
trellisGraph.CreateUriNode(new Uri(ldpAssignment))
));
AddModifiedDate(trellisGraph, graphUri);
}
}
/// <summary>
/// Get a WrapperGraph Instance for Assert & Retract
/// </summary>
/// <param name="graphUrl"></param>
/// <returns></returns>
private IGraph GetNewGraph(string graphUrl)
{
return new WrapperGraph()
{
BaseUri = new Uri(graphUrl)
};
}
/// <summary> /// <summary>
/// Gets all trellis subject ids that are RDFSources. /// Gets all trellis subject ids that are RDFSources.
/// </summary> /// </summary>
...@@ -417,7 +207,7 @@ namespace MetadataMigrator.Implementations ...@@ -417,7 +207,7 @@ namespace MetadataMigrator.Implementations
{ {
CommandText = "SELECT COUNT(?s) AS ?count " + query CommandText = "SELECT COUNT(?s) AS ?count " + query
}; };
var result = WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(cmdString.ToString())); var result = BaseRdfStoreConnector.WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(cmdString.ToString()));
var numberOfResult = Convert.ToInt32(((ILiteralNode)result.First().Value("count")).Value); var numberOfResult = Convert.ToInt32(((ILiteralNode)result.First().Value("count")).Value);
Logger.LogInformation($"{numberOfResult} Trellis Ids"); Logger.LogInformation($"{numberOfResult} Trellis Ids");
...@@ -432,7 +222,7 @@ namespace MetadataMigrator.Implementations ...@@ -432,7 +222,7 @@ namespace MetadataMigrator.Implementations
{ {
CommandText = "SELECT ?s " + query + $" LIMIT {QUERY_LIMIT} OFFSET {offset}" CommandText = "SELECT ?s " + query + $" LIMIT {QUERY_LIMIT} OFFSET {offset}"
}; };
using var results = WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(cmdString.ToString())); using var results = BaseRdfStoreConnector.WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(cmdString.ToString()));
trellisIds.AddRange(results.Select(x => (x.Value("s") as IUriNode).Uri).ToList()); trellisIds.AddRange(results.Select(x => (x.Value("s") as IUriNode).Uri).ToList());
} }
...@@ -457,7 +247,7 @@ namespace MetadataMigrator.Implementations ...@@ -457,7 +247,7 @@ namespace MetadataMigrator.Implementations
cmdString.SetUri("targetClass", new Uri(targetClass)); cmdString.SetUri("targetClass", new Uri(targetClass));
cmdString.SetLiteral("graph", id); cmdString.SetLiteral("graph", id);
var resultSet = WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(cmdString.ToString())); var resultSet = BaseRdfStoreConnector.WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(cmdString.ToString()));
var graphs = new List<Uri>(); var graphs = new List<Uri>();
foreach (SparqlResult r in resultSet) foreach (SparqlResult r in resultSet)
...@@ -471,23 +261,6 @@ namespace MetadataMigrator.Implementations ...@@ -471,23 +261,6 @@ namespace MetadataMigrator.Implementations
return graphs; return graphs;
} }
private void AddModifiedDate(IGraph graph, string root)
{
var dcTermsModifiedNode = graph.CreateUriNode(new Uri(dctermsModifiedUri));
var rootNode = graph.CreateUriNode(new Uri(root));
if (!graph.GetTriplesWithSubjectPredicate(rootNode, dcTermsModifiedNode).Any())
{
graph.Assert(new Triple(
rootNode,
dcTermsModifiedNode,
graph.CreateLiteralNode(
DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture),
new Uri(XmlSpecsHelper.XmlSchemaDataTypeDateTime)
)
));
}
}
/// <summary> /// <summary>
/// Since Application Profile URL does not have to be the targetClass, resolve it first /// Since Application Profile URL does not have to be the targetClass, resolve it first
/// </summary> /// </summary>
...@@ -506,7 +279,7 @@ namespace MetadataMigrator.Implementations ...@@ -506,7 +279,7 @@ namespace MetadataMigrator.Implementations
"WHERE { @applicationProfile <http://www.w3.org/ns/shacl#targetClass> ?targetClass }" "WHERE { @applicationProfile <http://www.w3.org/ns/shacl#targetClass> ?targetClass }"
}; };
targetClassCmdString.SetUri("applicationProfile", new Uri(entry.ApplicationProfile)); targetClassCmdString.SetUri("applicationProfile", new Uri(entry.ApplicationProfile));
var targetClassResultSet = WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(targetClassCmdString.ToString())); var targetClassResultSet = BaseRdfStoreConnector.WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(targetClassCmdString.ToString()));
var targetClass = entry.ApplicationProfile; var targetClass = entry.ApplicationProfile;
foreach (var result in targetClassResultSet) foreach (var result in targetClassResultSet)
......
src/MetadataMigrator/MetadataMigrator.csproj
+ 0
− 1
View file @ 43e70333
...@@ -12,7 +12,6 @@ ...@@ -12,7 +12,6 @@
<PackageReference Include="Coscine.Metadata" Version="2.*-*" /> <PackageReference Include="Coscine.Metadata" Version="2.*-*" />
<PackageReference Include="NLog" Version="5.1.1" /> <PackageReference Include="NLog" Version="5.1.1" />
<PackageReference Include="NLog.Extensions.Logging" Version="5.2.1" /> <PackageReference Include="NLog.Extensions.Logging" Version="5.2.1" />
<PackageReference Include="Polly" Version="7.2.3" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
......
src/MetadataMigrator/StructuralData.cs
+ 2
− 124
View file @ 43e70333
using Polly; using Coscine.Configuration;
using Coscine.Configuration;
using Coscine.Database.Models; using Coscine.Database.Models;
using Coscine.Metadata; using Coscine.Metadata;
using VDS.RDF; using VDS.RDF;
...@@ -60,132 +59,11 @@ namespace MetadataMigrator ...@@ -60,132 +59,11 @@ namespace MetadataMigrator
{ {
Logger.LogInformation($" ({graph.BaseUri})"); Logger.LogInformation($" ({graph.BaseUri})");
if (graph is WrapperGraph wrapperGraph) RdfStoreConnector.AddGraph(graph);
{
// Chunking since the size otherwise can be too large
foreach (var triples in wrapperGraph.AssertList.Chunk(100))
{
WrapRequest(() => RdfStoreConnector.ReadWriteSparqlConnector.UpdateGraph(graph.BaseUri, triples, new List<Triple>()));
}
// Chunking since the size otherwise can be too large
foreach (var triples in wrapperGraph.RetractList.Chunk(100))
{
WrapRequest(() => RdfStoreConnector.ReadWriteSparqlConnector.UpdateGraph(graph.BaseUri, new List<Triple>(), triples));
}
}
else
{
var exists = WrapRequest(() => RdfStoreConnector.HasGraph(graph.BaseUri));
if (exists)
{
Logger.LogInformation($" - Graph {graph.BaseUri} exists");
// Clear the existing graph from the store
WrapRequest(() => RdfStoreConnector.ClearGraph(graph.BaseUri));
Logger.LogInformation($" - Cleared Graph {graph.BaseUri}");
}
// Chunking since the size otherwise can be too large
// Don't change to only addition of triples, otherwise this could break things
foreach (var triples in graph.Triples.Chunk(100))
{
WrapRequest(() => RdfStoreConnector.ReadWriteSparqlConnector.UpdateGraph(graph.BaseUri, triples, new List<Triple>()));
}
}
Logger.LogInformation($" - Graph {graph.BaseUri} added successfully"); Logger.LogInformation($" - Graph {graph.BaseUri} added successfully");
Logger.LogInformation(""); Logger.LogInformation("");
} }
} }
public void AssertToGraphUriNode(IGraph graph, string graphSubject, string graphPredicate, string? graphObject)
{
if (graphObject != null)
{
graph.Assert(
new Triple(
graph.CreateUriNode(new Uri(graphSubject)),
graph.CreateUriNode(new Uri(graphPredicate)),
graph.CreateUriNode(new Uri(graphObject))
)
);
}
}
public void AssertToGraphLiteralNode(IGraph graph, string graphSubject, string graphPredicate, string? graphObject, Uri? objectType = null)
{
if (graphObject != null)
{
graph.Assert(
new Triple(
graph.CreateUriNode(new Uri(graphSubject)),
graph.CreateUriNode(new Uri(graphPredicate)),
graph.CreateLiteralNode(graphObject, objectType)
)
);
}
}
public void AssertToGraphBlankAndUriNode(IGraph graph, IBlankNode graphSubject, string graphPredicate, string? graphObject)
{
if (graphObject != null)
{
graph.Assert(
new Triple(
graphSubject,
graph.CreateUriNode(new Uri(graphPredicate)),
graph.CreateUriNode(new Uri(graphObject))
)
);
}
}
// Returns an empty graph, if the graph does not exists
public IGraph GetGraph(string graphName)
{
return GetGraph(new Uri(graphName));
}
// Returns an empty graph, if the graph does not exists
public IGraph GetGraph(Uri graphUri)
{
var graph = new WrapperGraph();
RdfStoreConnector.ReadWriteSparqlConnector.LoadGraph(graph, graphUri.AbsoluteUri);
graph.AssertList.Clear();
graph.RetractList.Clear();
return graph;
}
/// <summary>
/// Retry Virtuoso Requests since they sometimes just fail
/// </summary>
/// <typeparam name="W"></typeparam>
/// <param name="function"></param>
/// <returns></returns>
public void WrapRequest(Action action)
{
Policy
.Handle<Exception>()
.WaitAndRetry(5, retryNumber => TimeSpan.FromMilliseconds(200))
.Execute(() => action.Invoke());
}
/// <summary>
/// Retry Virtuoso Requests since they sometimes just fail
/// </summary>
/// <typeparam name="W"></typeparam>
/// <param name="function"></param>
/// <returns></returns>
public W WrapRequest<W>(Func<W> function)
{
return Policy
.Handle<Exception>()
.WaitAndRetry(5, retryNumber => TimeSpan.FromMilliseconds(200))
.ExecuteAndCapture(() => function.Invoke()).Result;
}
} }
} }
src/MetadataMigrator/VersionUtil.cs deleted 100644 → 0
+ 0
− 43
View file @ dd009898
namespace MetadataMigrator
{
public class VersionUtil
{
public static Uri? GetRecentVersion(IEnumerable<Uri> graphs, string? filter = null)
{
var currentBest = graphs.FirstOrDefault();
var currentBestVersion = 0L;
foreach (var graph in graphs)
{
var queryDictionary = System.Web.HttpUtility.ParseQueryString(
new Uri(graph.ToString().Replace("@", "?")).Query);
var version = queryDictionary["version"];
if (version == null || !long.TryParse(version, out long longVersion))
{
continue;
}
if (longVersion > currentBestVersion && queryDictionary["extracted"] == null && (filter == null || queryDictionary["type"] == filter))
{
currentBestVersion = longVersion;
currentBest = graph;
}
}
return currentBest;
}
public static Uri? GetRecentDataVersion(IEnumerable<Uri> graphs)
{
return GetRecentVersion(graphs, "data");
}
public static Uri? GetRecentMetadataVersion(IEnumerable<Uri> graphs)
{
return GetRecentVersion(graphs, "metadata");
}
public static long GetNewVersion()
{
// UTC Timestamp
return long.Parse(Convert.ToString((int)DateTime.UtcNow.Subtract(new DateTime(1970, 1, 1)).TotalSeconds));
}
}
}
src/MetadataMigrator/WrapperGraph.cs deleted 100644 → 0
+ 0
− 40
View file @ dd009898
using VDS.RDF;
namespace MetadataMigrator
{
public class WrapperGraph : Graph
{
public List<Triple> AssertList { get; set; }
public List<Triple> RetractList { get; set; }
public WrapperGraph() : base()
{
AssertList = new List<Triple>();
RetractList = new List<Triple>();
}
public override bool Assert(Triple t)
{
AssertList.Add(t);
return base.Assert(t);
}
public override bool Assert(IEnumerable<Triple> triples)
{
AssertList.AddRange(triples);
return base.Assert(triples);
}
public override bool Retract(Triple t)
{
RetractList.Add(t);
return base.Retract(t);
}
public override bool Retract(IEnumerable<Triple> triples)
{
RetractList.AddRange(triples);
return base.Retract(triples);
}
}
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment