Skip to content
Snippets Groups Projects
Commit a7c9ae38 authored by Benedikt Heinrichs's avatar Benedikt Heinrichs
Browse files

Simplify Metadata Migrator

parent dd009898
Branches
No related tags found
1 merge request!1Update: More usage of metadata library
Pipeline #890656 skipped
Stage: commands
Show whitespace changes
Inline Side-by-side
src/MetadataMigrator/Implementations/ResourceStructuralData.cs
+ 24
− 262
View file @ a7c9ae38
......@@ -2,43 +2,18 @@ using Coscine.Database.DataModel;
using Coscine.Database.Models;
using Coscine.Database.Util;
using Coscine.Metadata;
using Coscine.Metadata.Util;
using Microsoft.Extensions.Logging;
using System.Globalization;
using VDS.RDF;
using VDS.RDF.Parsing;
using VDS.RDF.Query;
namespace MetadataMigrator.Implementations
{
public class ResourceStructuralData : StructuralData<Resource, ResourceModel>
{
private readonly string partOfUri = "http://purl.org/dc/terms/isPartOf";
private readonly string aUri = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type";
private readonly string basicContainerUri = "http://www.w3.org/ns/ldp#BasicContainer";
private readonly string rdfSourceUri = "http://www.w3.org/ns/ldp#RDFSource";
private readonly string dcatcatalogUri = "http://www.w3.org/ns/dcat#catalog";
private readonly string dcatCatalogClassUri = "http://www.w3.org/ns/dcat#Catalog";
private readonly string dctermsIdentifierUri = "http://purl.org/dc/terms/identifier";
private readonly string dctermsModifiedUri = "http://purl.org/dc/terms/modified";
private readonly string fdpMetadataServiceUri = "http://purl.org/fdp/fdp-o#MetadataService";
private readonly string fdphasMetadataUri = "http://purl.org/fdp/fdp-o#hasMetadata";
private readonly string fdpisMetadataOfUri = "http://purl.org/fdp/fdp-o#isMetadataOf";
private readonly string provEntityUri = "http://www.w3.org/ns/prov#Entity";
private readonly string provGeneratedAtTimeUri = "http://www.w3.org/ns/prov#generatedAtTime";
private readonly string provWasRevisionOfUri = "http://www.w3.org/ns/prov#wasRevisionOfNode";
private readonly string ldpDescribedByUri = "http://www.w3.org/ns/ldp#describedBy";
private readonly string coscineLinkedBodyUri = "https://purl.org/coscine/terms/linked#body";
private readonly Dictionary<string, string> _targetClassMap = new Dictionary<string, string>();
private readonly int QUERY_LIMIT = 1000;
// Override to also receive deleted resources
public override IEnumerable<Resource> GetAll()
{
......@@ -67,13 +42,15 @@ namespace MetadataMigrator.Implementations
/// <returns></returns>
public override IEnumerable<IGraph> ConvertToLinkedData(IEnumerable<Resource> entries)
{
var trellisGraph = WrapRequest(() => GetNewGraph("http://www.trellisldp.org/ns/trellis#PreferServerManaged"));
var trellisGraph = RdfStoreConnector.GetEmptySmallUpdateGraph(Uris.TrellisGraph);
var trellisIds = GetTrellisIds();
var leftGraphs = new List<IGraph> { trellisGraph };
var resourceUrlPrefix = "https://purl.org/coscine/resources";
var resourceUrlHandlePrefix = $"https://hdl.handle.net/{Prefix}";
var metadataGraphCreator = new MetadataGraphsCreator(RdfStoreConnector);
var graphs = new List<IGraph>();
foreach (var entry in entries)
......@@ -95,8 +72,8 @@ namespace MetadataMigrator.Implementations
{
Logger.LogInformation($"Migrating {fileGraph.AbsoluteUri}");
var fileGraphObject = WrapRequest(() => GetGraph(fileGraph));
IGraph? linkedGraph = WrapRequest(() => GetGraph(fileGraph.AbsoluteUri + "&data"));
var fileGraphObject = RdfStoreConnector.GetGraph(fileGraph);
IGraph? linkedGraph = RdfStoreConnector.GetGraph(fileGraph.AbsoluteUri + "&data");
if (linkedGraph.IsEmpty)
{
linkedGraph = null;
......@@ -142,19 +119,21 @@ namespace MetadataMigrator.Implementations
continue;
}
var newFileGraph = GetNewGraph(newFileGraphName);
var fileNode = newFileGraph.CreateUriNode(new Uri(newFileGraphName));
graphs.Add(newFileGraph);
graphs.AddRange(
metadataGraphCreator.CreateGraphs(entry.Id.ToString(), path, true, true, new ProvidedGraphData {
CurrentDataVersionGraph = currentDataVersionGraph,
CurrentMetadataVersionGraph = currentMetadataVersionGraph,
newFileGraph.Assert(new Triple(fileNode, newFileGraph.CreateUriNode(new Uri(aUri)), newFileGraph.CreateUriNode(new Uri(dcatCatalogClassUri))));
newFileGraph.Assert(new Triple(fileNode, newFileGraph.CreateUriNode(new Uri(aUri)), newFileGraph.CreateUriNode(new Uri(fdpMetadataServiceUri))));
AddFilesToAFolder(trellisGraph, new Uri(newFileGraphName), resourceGraphName, graphs);
OldDataGraph = linkedGraph,
OldMetadataGraph = fileGraphObject,
var metadataFileGraph = SetMetadataGraph(trellisGraph, graphs, newMetadataFileGraphName, newFileGraph, newFileGraphName);
var dataFileGraph = SetDataGraph(trellisGraph, graphs, newDataFileGraphName, newFileGraph, newFileGraphName, metadataFileGraph.BaseUri.AbsoluteUri);
RecentDataVersion = recentDataVersion,
RecentMetadataVersion = recentMetadataVersion,
SetDataVersionGraph(graphs, recentDataVersion, currentDataVersionGraph, linkedGraph, newDataVersionFileGraphName, dataFileGraph, trellisGraph, newFileGraphName);
SetMetadataVersionGraph(graphs, recentMetadataVersion, currentMetadataVersionGraph, fileGraphObject, newMetadataVersionFileGraphName, metadataFileGraph, trellisGraph, newFileGraphName, currentDataVersionGraph);
TrellisGraph = trellisGraph
})
);
graphs.RemoveAll((graph) => graph.BaseUri == Uris.TrellisGraph);
}
// Intermediate update to save memory
......@@ -174,185 +153,19 @@ namespace MetadataMigrator.Implementations
return leftGraphs;
}
private void AddFilesToAFolder(IGraph trellisGraph, Uri fileGraph, string resourceGraphName, List<IGraph> graphs)
{
string rootUri;
var fileGraphUri = fileGraph.AbsoluteUri;
do
{
rootUri = fileGraphUri[..fileGraphUri.LastIndexOf("/")];
var rootGraph = GetGraph(rootUri);
rootGraph.Assert(new Triple(rootGraph.CreateUriNode(new Uri(rootUri)), rootGraph.CreateUriNode(new Uri(dcatcatalogUri)), rootGraph.CreateUriNode(fileGraph)));
AddToTrellis(trellisGraph, basicContainerUri, rootUri, fileGraphUri);
graphs.Add(rootGraph);
fileGraphUri = rootUri;
} while (rootUri != resourceGraphName && rootUri.Contains("/"));
}
private IGraph SetMetadataGraph(IGraph trellisGraph, List<IGraph> graphs, string newMetadataFileGraphName, IGraph newFileGraph, string fileUri)
{
var metadataFileNode = newFileGraph.CreateUriNode(new Uri(newMetadataFileGraphName));
var metadataFileGraph = GetNewGraph(newMetadataFileGraphName);
graphs.Add(metadataFileGraph);
AddToTrellis(trellisGraph, rdfSourceUri, fileUri, newMetadataFileGraphName);
var fileNode = newFileGraph.CreateUriNode(new Uri(fileUri));
newFileGraph.Assert(new Triple(fileNode, newFileGraph.CreateUriNode(new Uri(dcatcatalogUri)), metadataFileNode));
newFileGraph.Assert(new Triple(fileNode, newFileGraph.CreateUriNode(new Uri(fdphasMetadataUri)), metadataFileNode));
metadataFileGraph.Assert(new Triple(
Tools.CopyNode(metadataFileNode, metadataFileGraph),
metadataFileGraph.CreateUriNode(new Uri(aUri)),
metadataFileGraph.CreateUriNode(new Uri(dcatCatalogClassUri))
));
return metadataFileGraph;
}
private IGraph SetDataGraph(IGraph trellisGraph, List<IGraph> graphs, string newDataFileGraphName, IGraph newFileGraph, string fileUri, string metadataFileUri)
{
var dataFileNode = newFileGraph.CreateUriNode(new Uri(newDataFileGraphName));
var dataFileGraph = GetNewGraph(newDataFileGraphName);
graphs.Add(dataFileGraph);
AddToTrellis(trellisGraph, rdfSourceUri, fileUri, newDataFileGraphName);
var fileNode = newFileGraph.CreateUriNode(new Uri(fileUri));
newFileGraph.Assert(new Triple(fileNode, newFileGraph.CreateUriNode(new Uri(dcatcatalogUri)), dataFileNode));
dataFileGraph.Assert(new Triple(Tools.CopyNode(dataFileNode, dataFileGraph), dataFileGraph.CreateUriNode(new Uri(aUri)), dataFileGraph.CreateUriNode(new Uri(dcatCatalogClassUri))));
dataFileGraph.Assert(new Triple(Tools.CopyNode(dataFileNode, dataFileGraph), dataFileGraph.CreateUriNode(new Uri(ldpDescribedByUri)), dataFileGraph.CreateUriNode(new Uri(metadataFileUri))));
return dataFileGraph;
}
private void SetDataVersionGraph(List<IGraph> graphs, Uri? recentDataVersion, IGraph currentDataVersionGraph, IGraph? linkedGraph, string newDataVersionFileGraphName, IGraph dataFileGraph, IGraph trellisGraph, string fileUri)
{
SetLinkedData(linkedGraph, currentDataVersionGraph, new Uri(newDataVersionFileGraphName));
var currentDataVersionNode = currentDataVersionGraph.CreateUriNode(currentDataVersionGraph.BaseUri);
currentDataVersionGraph.Assert(new Triple(currentDataVersionNode, currentDataVersionGraph.CreateUriNode(new Uri(dctermsIdentifierUri)), currentDataVersionGraph.CreateLiteralNode(
currentDataVersionGraph.BaseUri.AbsoluteUri,
new Uri(XmlSpecsHelper.XmlSchemaDataTypeString)
)));
currentDataVersionGraph.Assert(new Triple(
currentDataVersionNode,
currentDataVersionGraph.CreateUriNode(new Uri(aUri)),
currentDataVersionGraph.CreateUriNode(new Uri("http://www.w3.org/ns/dcat#Dataset"))
));
// PROV Info
var provTriple = new Triple(Tools.CopyNode(currentDataVersionNode, dataFileGraph), dataFileGraph.CreateUriNode(new Uri(aUri)), dataFileGraph.CreateUriNode(new Uri(provEntityUri)));
dataFileGraph.Assert(provTriple);
dataFileGraph.Assert(new Triple(
dataFileGraph.CreateUriNode(dataFileGraph.BaseUri),
dataFileGraph.CreateUriNode(new Uri("http://www.w3.org/ns/dcat#dataset")),
Tools.CopyNode(currentDataVersionNode, dataFileGraph)
));
dataFileGraph.Assert(new Triple(Tools.CopyNode(currentDataVersionNode, dataFileGraph), dataFileGraph.CreateUriNode(new Uri(provGeneratedAtTimeUri)), dataFileGraph.CreateLiteralNode(
DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture),
new Uri(XmlSpecsHelper.XmlSchemaDataTypeDateTime)
)));
AddToTrellis(trellisGraph, rdfSourceUri, fileUri, currentDataVersionGraph.BaseUri.AbsoluteUri);
if (recentDataVersion != null && recentDataVersion.AbsoluteUri != currentDataVersionGraph.BaseUri.AbsoluteUri)
{
var recentDataVersionNode = dataFileGraph.CreateUriNode(recentDataVersion);
dataFileGraph.Assert(new Triple(Tools.CopyNode(currentDataVersionNode, dataFileGraph), dataFileGraph.CreateUriNode(new Uri(provWasRevisionOfUri)), recentDataVersionNode));
}
graphs.Add(currentDataVersionGraph);
}
private void SetMetadataVersionGraph(List<IGraph> graphs, Uri? recentMetadataVersion, IGraph currentMetadataVersionGraph, IGraph? metadataGraph, string newMetadataVersionFileGraphName, IGraph metadataFileGraph, IGraph trellisGraph, string fileUri, IGraph currentDataVersionGraph)
{
SetMetadata(metadataGraph, currentMetadataVersionGraph, new Uri(newMetadataVersionFileGraphName));
var currentMetadataVersionNode = currentMetadataVersionGraph.CreateUriNode(currentMetadataVersionGraph.BaseUri);
// PROV Info
var provTriple = new Triple(Tools.CopyNode(currentMetadataVersionNode, metadataFileGraph), metadataFileGraph.CreateUriNode(new Uri(aUri)), metadataFileGraph.CreateUriNode(new Uri(provEntityUri)));
metadataFileGraph.Assert(provTriple);
metadataFileGraph.Assert(new Triple(
metadataFileGraph.CreateUriNode(metadataFileGraph.BaseUri),
metadataFileGraph.CreateUriNode(new Uri(fdphasMetadataUri)),
Tools.CopyNode(currentMetadataVersionNode, metadataFileGraph)
));
metadataFileGraph.Assert(new Triple(Tools.CopyNode(currentMetadataVersionNode, metadataFileGraph), metadataFileGraph.CreateUriNode(new Uri(provGeneratedAtTimeUri)), metadataFileGraph.CreateLiteralNode(
DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture),
new Uri(XmlSpecsHelper.XmlSchemaDataTypeDateTime)
)));
metadataFileGraph.Assert(new Triple(
metadataFileGraph.CreateUriNode(currentMetadataVersionGraph.BaseUri),
metadataFileGraph.CreateUriNode(new Uri(fdpisMetadataOfUri)),
metadataFileGraph.CreateUriNode(currentDataVersionGraph.BaseUri)
));
AddToTrellis(trellisGraph, rdfSourceUri, fileUri, currentMetadataVersionGraph.BaseUri.AbsoluteUri);
if (recentMetadataVersion != null && recentMetadataVersion.AbsoluteUri != currentMetadataVersionGraph.BaseUri.AbsoluteUri)
{
var recentMetadataVersionNode = metadataFileGraph.CreateUriNode(recentMetadataVersion);
metadataFileGraph.Assert(new Triple(Tools.CopyNode(currentMetadataVersionNode, metadataFileGraph), metadataFileGraph.CreateUriNode(new Uri(provWasRevisionOfUri)), recentMetadataVersionNode));
}
graphs.Add(currentMetadataVersionGraph);
}
private IGraph GetRecentVersionGraph(string newVersionFileGraphName, Uri? recentVersion)
{
if (recentVersion == null)
{
return new WrapperGraph()
return new Graph()
{
BaseUri = new Uri(newVersionFileGraphName),
};
}
else
{
return WrapRequest(() => GetGraph(recentVersion));
}
}
private void SetLinkedData(IGraph? linkedGraph, IGraph currentDataVersionGraph, Uri otherURICandidate)
{
var linkedBody = currentDataVersionGraph.GetTriplesWithPredicate(new Uri(coscineLinkedBodyUri));
if (linkedGraph != null)
{
var setLinkedBody = linkedGraph.GetTriplesWithPredicate(new Uri(coscineLinkedBodyUri));
if (!linkedBody.Any()
|| linkedBody.First().Object.ToString()
!= setLinkedBody.First().Object.ToString())
{
var oldDataVersionNode = currentDataVersionGraph.CreateUriNode(currentDataVersionGraph.BaseUri);
var currentDataVersionNode = currentDataVersionGraph.CreateUriNode(otherURICandidate);
currentDataVersionGraph.BaseUri = otherURICandidate;
foreach (var currentLinkedBody in linkedBody)
{
currentDataVersionGraph.Retract(currentLinkedBody);
}
foreach (var oldTriple in currentDataVersionGraph.GetTriplesWithSubject(oldDataVersionNode))
{
currentDataVersionGraph.Retract(oldTriple);
currentDataVersionGraph.Assert(new Triple(currentDataVersionNode, Tools.CopyNode(oldTriple.Predicate, currentDataVersionGraph), Tools.CopyNode(oldTriple.Object, currentDataVersionGraph)));
}
foreach (var currentSetLinkedBody in setLinkedBody)
{
currentDataVersionGraph.Assert(new Triple(currentDataVersionNode, currentDataVersionGraph.CreateUriNode(new Uri(coscineLinkedBodyUri)), Tools.CopyNode(currentSetLinkedBody.Object, currentDataVersionGraph)));
}
}
}
}
private static void SetMetadata(IGraph? metadataGraph, IGraph currentMetadataVersionGraph, Uri otherURICandidate)
{
if (metadataGraph != null)
{
// If the predicate & object pairs are different
if (!
(metadataGraph.Triples.All((triple) => currentMetadataVersionGraph.GetTriplesWithPredicateObject(triple.Predicate, triple.Object).Any())
&& currentMetadataVersionGraph.Triples.All((triple) => metadataGraph.GetTriplesWithPredicateObject(triple.Predicate, triple.Object).Any())))
{
currentMetadataVersionGraph.Retract(currentMetadataVersionGraph.Triples);
currentMetadataVersionGraph.BaseUri = otherURICandidate;
var currentMetadataVersionNode = currentMetadataVersionGraph.CreateUriNode(otherURICandidate);
foreach (var triple in metadataGraph.Triples)
{
currentMetadataVersionGraph.Assert(currentMetadataVersionNode, Tools.CopyNode(triple.Predicate, currentMetadataVersionGraph), Tools.CopyNode(triple.Object, currentMetadataVersionGraph));
}
}
return RdfStoreConnector.GetGraph(recentVersion);
}
}
......@@ -365,40 +178,6 @@ namespace MetadataMigrator.Implementations
return path;
}
private void AddToTrellis(IGraph trellisGraph, string ldpAssignment, string thePartUri, string graphUri)
{
var setGraphNode = trellisGraph.CreateUriNode(new Uri(graphUri));
var setThePartNode = trellisGraph.CreateUriNode(new Uri(thePartUri));
var triple = new Triple(
setGraphNode,
trellisGraph.CreateUriNode(new Uri(partOfUri)),
setThePartNode
);
if (!trellisGraph.ContainsTriple(triple))
{
trellisGraph.Assert(triple);
trellisGraph.Assert(new Triple(
setGraphNode,
trellisGraph.CreateUriNode(new Uri(aUri)),
trellisGraph.CreateUriNode(new Uri(ldpAssignment))
));
AddModifiedDate(trellisGraph, graphUri);
}
}
/// <summary>
/// Get a WrapperGraph Instance for Assert & Retract
/// </summary>
/// <param name="graphUrl"></param>
/// <returns></returns>
private IGraph GetNewGraph(string graphUrl)
{
return new WrapperGraph()
{
BaseUri = new Uri(graphUrl)
};
}
/// <summary>
/// Gets all trellis subject ids that are RDFSources.
/// </summary>
......@@ -417,7 +196,7 @@ namespace MetadataMigrator.Implementations
{
CommandText = "SELECT COUNT(?s) AS ?count " + query
};
var result = WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(cmdString.ToString()));
var result = RdfStoreConnector.WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(cmdString.ToString()));
var numberOfResult = Convert.ToInt32(((ILiteralNode)result.First().Value("count")).Value);
Logger.LogInformation($"{numberOfResult} Trellis Ids");
......@@ -432,7 +211,7 @@ namespace MetadataMigrator.Implementations
{
CommandText = "SELECT ?s " + query + $" LIMIT {QUERY_LIMIT} OFFSET {offset}"
};
using var results = WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(cmdString.ToString()));
using var results = RdfStoreConnector.WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(cmdString.ToString()));
trellisIds.AddRange(results.Select(x => (x.Value("s") as IUriNode).Uri).ToList());
}
......@@ -457,7 +236,7 @@ namespace MetadataMigrator.Implementations
cmdString.SetUri("targetClass", new Uri(targetClass));
cmdString.SetLiteral("graph", id);
var resultSet = WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(cmdString.ToString()));
var resultSet = RdfStoreConnector.WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(cmdString.ToString()));
var graphs = new List<Uri>();
foreach (SparqlResult r in resultSet)
......@@ -471,23 +250,6 @@ namespace MetadataMigrator.Implementations
return graphs;
}
private void AddModifiedDate(IGraph graph, string root)
{
var dcTermsModifiedNode = graph.CreateUriNode(new Uri(dctermsModifiedUri));
var rootNode = graph.CreateUriNode(new Uri(root));
if (!graph.GetTriplesWithSubjectPredicate(rootNode, dcTermsModifiedNode).Any())
{
graph.Assert(new Triple(
rootNode,
dcTermsModifiedNode,
graph.CreateLiteralNode(
DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture),
new Uri(XmlSpecsHelper.XmlSchemaDataTypeDateTime)
)
));
}
}
/// <summary>
/// Since Application Profile URL does not have to be the targetClass, resolve it first
/// </summary>
......@@ -506,7 +268,7 @@ namespace MetadataMigrator.Implementations
"WHERE { @applicationProfile <http://www.w3.org/ns/shacl#targetClass> ?targetClass }"
};
targetClassCmdString.SetUri("applicationProfile", new Uri(entry.ApplicationProfile));
var targetClassResultSet = WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(targetClassCmdString.ToString()));
var targetClassResultSet = RdfStoreConnector.WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(targetClassCmdString.ToString()));
var targetClass = entry.ApplicationProfile;
foreach (var result in targetClassResultSet)
......
src/MetadataMigrator/MetadataMigrator.csproj
+ 0
− 1
View file @ a7c9ae38
......@@ -12,7 +12,6 @@
<PackageReference Include="Coscine.Metadata" Version="2.*-*" />
<PackageReference Include="NLog" Version="5.1.1" />
<PackageReference Include="NLog.Extensions.Logging" Version="5.2.1" />
<PackageReference Include="Polly" Version="7.2.3" />
</ItemGroup>
<ItemGroup>
......
src/MetadataMigrator/StructuralData.cs
+ 2
− 124
View file @ a7c9ae38
using Polly;
using Coscine.Configuration;
using Coscine.Configuration;
using Coscine.Database.Models;
using Coscine.Metadata;
using VDS.RDF;
......@@ -60,132 +59,11 @@ namespace MetadataMigrator
{
Logger.LogInformation($" ({graph.BaseUri})");
if (graph is WrapperGraph wrapperGraph)
{
// Chunking since the size otherwise can be too large
foreach (var triples in wrapperGraph.AssertList.Chunk(100))
{
WrapRequest(() => RdfStoreConnector.ReadWriteSparqlConnector.UpdateGraph(graph.BaseUri, triples, new List<Triple>()));
}
// Chunking since the size otherwise can be too large
foreach (var triples in wrapperGraph.RetractList.Chunk(100))
{
WrapRequest(() => RdfStoreConnector.ReadWriteSparqlConnector.UpdateGraph(graph.BaseUri, new List<Triple>(), triples));
}
}
else
{
var exists = WrapRequest(() => RdfStoreConnector.HasGraph(graph.BaseUri));
if (exists)
{
Logger.LogInformation($" - Graph {graph.BaseUri} exists");
// Clear the existing graph from the store
WrapRequest(() => RdfStoreConnector.ClearGraph(graph.BaseUri));
Logger.LogInformation($" - Cleared Graph {graph.BaseUri}");
}
// Chunking since the size otherwise can be too large
// Don't change to only addition of triples, otherwise this could break things
foreach (var triples in graph.Triples.Chunk(100))
{
WrapRequest(() => RdfStoreConnector.ReadWriteSparqlConnector.UpdateGraph(graph.BaseUri, triples, new List<Triple>()));
}
}
RdfStoreConnector.AddGraph(graph);
Logger.LogInformation($" - Graph {graph.BaseUri} added successfully");
Logger.LogInformation("");
}
}
public void AssertToGraphUriNode(IGraph graph, string graphSubject, string graphPredicate, string? graphObject)
{
if (graphObject != null)
{
graph.Assert(
new Triple(
graph.CreateUriNode(new Uri(graphSubject)),
graph.CreateUriNode(new Uri(graphPredicate)),
graph.CreateUriNode(new Uri(graphObject))
)
);
}
}
public void AssertToGraphLiteralNode(IGraph graph, string graphSubject, string graphPredicate, string? graphObject, Uri? objectType = null)
{
if (graphObject != null)
{
graph.Assert(
new Triple(
graph.CreateUriNode(new Uri(graphSubject)),
graph.CreateUriNode(new Uri(graphPredicate)),
graph.CreateLiteralNode(graphObject, objectType)
)
);
}
}
public void AssertToGraphBlankAndUriNode(IGraph graph, IBlankNode graphSubject, string graphPredicate, string? graphObject)
{
if (graphObject != null)
{
graph.Assert(
new Triple(
graphSubject,
graph.CreateUriNode(new Uri(graphPredicate)),
graph.CreateUriNode(new Uri(graphObject))
)
);
}
}
// Returns an empty graph, if the graph does not exists
public IGraph GetGraph(string graphName)
{
return GetGraph(new Uri(graphName));
}
// Returns an empty graph, if the graph does not exists
public IGraph GetGraph(Uri graphUri)
{
var graph = new WrapperGraph();
RdfStoreConnector.ReadWriteSparqlConnector.LoadGraph(graph, graphUri.AbsoluteUri);
graph.AssertList.Clear();
graph.RetractList.Clear();
return graph;
}
/// <summary>
/// Retry Virtuoso Requests since they sometimes just fail
/// </summary>
/// <typeparam name="W"></typeparam>
/// <param name="function"></param>
/// <returns></returns>
public void WrapRequest(Action action)
{
Policy
.Handle<Exception>()
.WaitAndRetry(5, retryNumber => TimeSpan.FromMilliseconds(200))
.Execute(() => action.Invoke());
}
/// <summary>
/// Retry Virtuoso Requests since they sometimes just fail
/// </summary>
/// <typeparam name="W"></typeparam>
/// <param name="function"></param>
/// <returns></returns>
public W WrapRequest<W>(Func<W> function)
{
return Policy
.Handle<Exception>()
.WaitAndRetry(5, retryNumber => TimeSpan.FromMilliseconds(200))
.ExecuteAndCapture(() => function.Invoke()).Result;
}
}
}
src/MetadataMigrator/VersionUtil.cs deleted 100644 → 0
+ 0
− 43
View file @ dd009898
namespace MetadataMigrator
{
public class VersionUtil
{
public static Uri? GetRecentVersion(IEnumerable<Uri> graphs, string? filter = null)
{
var currentBest = graphs.FirstOrDefault();
var currentBestVersion = 0L;
foreach (var graph in graphs)
{
var queryDictionary = System.Web.HttpUtility.ParseQueryString(
new Uri(graph.ToString().Replace("@", "?")).Query);
var version = queryDictionary["version"];
if (version == null || !long.TryParse(version, out long longVersion))
{
continue;
}
if (longVersion > currentBestVersion && queryDictionary["extracted"] == null && (filter == null || queryDictionary["type"] == filter))
{
currentBestVersion = longVersion;
currentBest = graph;
}
}
return currentBest;
}
public static Uri? GetRecentDataVersion(IEnumerable<Uri> graphs)
{
return GetRecentVersion(graphs, "data");
}
public static Uri? GetRecentMetadataVersion(IEnumerable<Uri> graphs)
{
return GetRecentVersion(graphs, "metadata");
}
public static long GetNewVersion()
{
// UTC Timestamp
return long.Parse(Convert.ToString((int)DateTime.UtcNow.Subtract(new DateTime(1970, 1, 1)).TotalSeconds));
}
}
}
src/MetadataMigrator/WrapperGraph.cs deleted 100644 → 0
+ 0
− 40
View file @ dd009898
using VDS.RDF;
namespace MetadataMigrator
{
public class WrapperGraph : Graph
{
public List<Triple> AssertList { get; set; }
public List<Triple> RetractList { get; set; }
public WrapperGraph() : base()
{
AssertList = new List<Triple>();
RetractList = new List<Triple>();
}
public override bool Assert(Triple t)
{
AssertList.Add(t);
return base.Assert(t);
}
public override bool Assert(IEnumerable<Triple> triples)
{
AssertList.AddRange(triples);
return base.Assert(triples);
}
public override bool Retract(Triple t)
{
RetractList.Add(t);
return base.Retract(t);
}
public override bool Retract(IEnumerable<Triple> triples)
{
RetractList.AddRange(triples);
return base.Retract(triples);
}
}
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment