Skip to content
Snippets Groups Projects

Draft: Issue/2668 remaining cron

Open Draft: Issue/2668 remaining cron
Issue/2668-remainingCron into dev
Open
Petar Hristovrequested to merge
Issue/2668-remainingCron into dev
Compare
and
2 files
+ 234
− 233
Compare changes
  • Side-by-side
  • Inline

Files

src/TrellisMigrator/StructuralData.cs
+ 168
− 199
using Coscine.Configuration;

using Coscine.Database.Models;
using Coscine.ApiClient;
using Polly;
using Coscine.ApiClient.Core.Api;
using System.Globalization;
using Coscine.ApiClient.Core.Client;
 
using Coscine.ApiClient.Core.Model;
 
using Microsoft.Extensions.Configuration;
using VDS.RDF;
using VDS.RDF;
using VDS.RDF.Parsing;
using VDS.RDF.Query;
using VDS.RDF.Storage;
using VDS.RDF.Update;
namespace TrellisMigrator
namespace TrellisMigrator;
{
public abstract class StructuralData<S, T> where S : class where T : DatabaseModel<S>, new()
{
public T Model { get; init; }
public ConsulConfiguration Configuration { get; init; }
public static string Prefix { get; set; }
public readonly SparqlRemoteUpdateEndpoint UpdateEndpoint;
/// <summary>
public readonly SparqlRemoteEndpoint QueryEndpoint;
/// Provides an abstract base class for handling structural data transformations and migrations.
public readonly ReadWriteSparqlConnector ReadWriteSparqlConnector;
/// </summary>
 
/// <typeparam name="S">The type of data to be transformed or migrated.</typeparam>
 
/// <remarks>
 
/// This class is partially shared between this script and the SQL2Linked script.
 
/// </remarks>
 
public abstract class StructuralData<S>
 
{
 
private readonly string _adminToken;
public StructuralData()
protected readonly Configuration _apiConfiguration;
{
protected readonly AdminApi _adminApi;
Configuration = new ConsulConfiguration();
protected readonly PidConfiguration _pidConfiguration; // Comes from the API Client, not from the application's own configuration
Model = new T();
Prefix = Configuration.GetStringAndWait("coscine/global/epic/prefix");
var sparqlEndpoint = Configuration.GetStringAndWait("coscine/local/virtuoso/additional/url");
UpdateEndpoint = new SparqlRemoteUpdateEndpoint(new Uri(string.Format(sparqlEndpoint)));
QueryEndpoint = new SparqlRemoteEndpoint(new Uri(string.Format(sparqlEndpoint)));
ReadWriteSparqlConnector = new ReadWriteSparqlConnector(QueryEndpoint, UpdateEndpoint);
// 100 second timeout
var timeout = 100000;
QueryEndpoint.Timeout = timeout;
UpdateEndpoint.Timeout = timeout;
ReadWriteSparqlConnector.Timeout = timeout;
}
public abstract IEnumerable<IGraph> ConvertToLinkedData(IEnumerable<S> entries);
/// <summary>
 
/// Initializes a new instance of the <see cref="StructuralData{S}"/> class.
 
/// </summary>
 
public StructuralData()
 
{
 
// Retrieve the configuration settings for the PID ePIC API from the API Client
 
var apiConfiguration = ApiConfigurationUtil.RetrieveApiConfiguration();
 
_pidConfiguration = apiConfiguration.GetSection(PidConfiguration.Section).Get<PidConfiguration>() ?? new();
 
 
// Ensiure that the prefix is not null or empty.
 
ArgumentException.ThrowIfNullOrWhiteSpace(_pidConfiguration.Prefix, nameof(_pidConfiguration.Prefix));
 
 
// Generate an admin token for the API Client
 
var jwtConfiguration = ApiConfigurationUtil.RetrieveJwtConfiguration();
 
_adminToken = ApiConfigurationUtil.GenerateAdminToken(jwtConfiguration);
 
_adminApi = new AdminApi(new Configuration()
 
_apiConfiguration = new Configuration()
 
{
 
BasePath = "http://localhost:7206/coscine",
 
ApiKeyPrefix = { { "Authorization", "Bearer" } },
 
ApiKey = { { "Authorization", _adminToken } },
 
});
 
};
 
_adminApi = new AdminApi(_apiConfiguration);
 
}
public virtual IEnumerable<S> GetAll()
/// <summary>
 
/// Converts the given entries to linked data graphs.
 
/// </summary>
 
/// <param name="entries">The entries to convert.</param>
 
/// <returns>An enumerable collection of graphs representing the linked data.</returns>
 
public abstract IEnumerable<IGraph> ConvertToLinkedData(IEnumerable<S> entries);
 
 
/// <summary>
 
/// Retrieves all entries of type <typeparamref name="S"/>.
 
/// </summary>
 
/// <returns>A task that represents the asynchronous operation. The task result contains an enumerable of all entries.</returns>
 
public abstract Task<IEnumerable<S>> GetAll();
 
 
/// <summary>
 
/// Migrates the data, optionally in a dummy mode where changes are not persisted.
 
/// </summary>
 
/// <param name="dummyMode">If set to <c>true</c>, the migration is simulated but not executed.</param>
 
/// <returns>A task representing the asynchronous operation.</returns>
 
public async Task Migrate(bool dummyMode)
 
{
 
var spacer = new string('-', 35);
 
Console.WriteLine($"\n{spacer}\n{typeof(S).Name}\n{spacer}");
 
var graphs = ConvertToLinkedData(await GetAll());
 
if (!dummyMode)
{
{
return Model.GetAll();
await StoreGraphs(graphs);
}
}
 
}
public void Migrate(bool dummyMode)
/// <summary>
{
/// Stores the given graphs in the underlying data store.
var spacer = new string('-', 35);
/// </summary>
Console.WriteLine($"\n{spacer}\n{typeof(T).Name}\n{spacer}");
/// <param name="graphs">The graphs to be stored.</param>
var graphs = ConvertToLinkedData(GetAll());
/// <returns>A task representing the asynchronous operation.</returns>
if (!dummyMode)
public async Task StoreGraphs(IEnumerable<IGraph> graphs)
{
{
StoreGraphs(graphs);
var formatEnum = RdfFormat.TextTurtle;
}
var format = "text/turtle";
}
private void StoreGraphs(IEnumerable<IGraph> graphs)
foreach (var graph in graphs)
{
{
foreach (var graph in graphs)
Console.WriteLine($" ({graph.BaseUri})");
{
Console.WriteLine($" ({graph.BaseUri})");
 
try
 
{
if (graph is WrapperGraph)
if (graph is WrapperGraph)
{
{
var wrapperGraph = (WrapperGraph)graph;
var rdfWriter = MimeTypesHelper.GetWriter(format);
// Chunking since the size otherwise can be too large
var assert = VDS.RDF.Writing.StringWriter.Write((graph as WrapperGraph)?.AssertGraph, rdfWriter);
foreach (var triples in wrapperGraph.AssertList.Chunk(100))
var retract = VDS.RDF.Writing.StringWriter.Write((graph as WrapperGraph)?.RetractGraph, rdfWriter);
{
WrapRequest(() => ReadWriteSparqlConnector.UpdateGraph(graph.BaseUri, triples, new List<Triple>()));
throw new NotImplementedException();
}
//await _adminApi.PatchMetadataGraphAsync(
// Chunking since the size otherwise can be too large
// new MetadataPatchAdminParameters
foreach (var triples in wrapperGraph.RetractList.Chunk(100))
// (
{
// graph.BaseUri.AbsoluteUri,
WrapRequest(() => ReadWriteSparqlConnector.UpdateGraph(graph.BaseUri, new List<Triple>(), triples));
// new RdfDefinitionForManipulationDto(assert, formatEnum),
}
// new RdfDefinitionForManipulationDto(retract, formatEnum)
 
// )
 
//);
}
}
else
else
{
{
var exists = WrapRequest(() => HasGraph(graph.BaseUri));
var rdfWriter = MimeTypesHelper.GetWriter(format);
if (exists)
var content = VDS.RDF.Writing.StringWriter.Write(graph, rdfWriter);
{
Console.WriteLine($" - Graph {graph.BaseUri} exists");
await _adminApi.UpdateMetadataGraphAsync(
new MetadataUpdateAdminParameters
// Clear the existing graph from the store
(
WrapRequest(() => ClearGraph(graph.BaseUri));
graph.BaseUri.AbsoluteUri,
Console.WriteLine($" - Cleared Graph {graph.BaseUri}");
new RdfDefinitionForManipulationDto(content, formatEnum)
}
)
);
// Chunking since the size otherwise can be too large
// Don't change to only addition of triples, otherwise this could break things
foreach (var triples in graph.Triples.Chunk(100))
{
WrapRequest(() => ReadWriteSparqlConnector.UpdateGraph(graph.BaseUri, triples, new List<Triple>()));
}
}
}
 
Console.WriteLine($" - Graph {graph.BaseUri} added successfully");
Console.WriteLine($" - Graph {graph.BaseUri} added successfully");
Console.WriteLine();
Console.WriteLine();
}
}
}
catch (Exception e)
public void AssertToGraphUriNode(IGraph graph, string graphSubject, string graphPredicate, string? graphObject)
{
if (graphObject != null)
{
graph.Assert(
new Triple(
graph.CreateUriNode(new Uri(graphSubject)),
graph.CreateUriNode(new Uri(graphPredicate)),
graph.CreateUriNode(new Uri(graphObject))
)
);
}
}
public void AssertToGraphLiteralNode(IGraph graph, string graphSubject, string graphPredicate, string? graphObject, Uri? objectType = null)
{
if (graphObject != null)
{
graph.Assert(
new Triple(
graph.CreateUriNode(new Uri(graphSubject)),
graph.CreateUriNode(new Uri(graphPredicate)),
graph.CreateLiteralNode(graphObject, objectType)
)
);
}
}
public void AssertToGraphBlankAndUriNode(IGraph graph, IBlankNode graphSubject, string graphPredicate, string? graphObject)
{
if (graphObject != null)
{
graph.Assert(
new Triple(
graphSubject,
graph.CreateUriNode(new Uri(graphPredicate)),
graph.CreateUriNode(new Uri(graphObject))
)
);
}
}
public static void AddModifiedDate(IGraph graph, IUriNode dctermsModifiedNode, IUriNode rootNode)
{
if (!graph.GetTriplesWithSubjectPredicate(rootNode, dctermsModifiedNode).Any())
{
{
graph.Assert(new Triple(
Console.Error.WriteLine($"Error on ({graph.BaseUri}):");
rootNode,
Console.Error.WriteLine(e);
dctermsModifiedNode,
Console.Error.WriteLine(e.InnerException);
graph.CreateLiteralNode(
Console.Error.WriteLine();
DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture),
new Uri(XmlSpecsHelper.XmlSchemaDataTypeDateTime)
)
));
}
}
}
}
 
}
public bool HasGraph(string graphName)
/// <summary>
{
/// Asserts a triple to the graph with a URI node object.
return HasGraph(new Uri(graphName));
/// </summary>
}
/// <param name="graph">The graph to assert to.</param>
/// <param name="graphSubject">The subject URI of the triple.</param>
public bool HasGraph(Uri graphUri)
/// <param name="graphPredicate">The predicate URI of the triple.</param>
{
/// <param name="graphObject">The object URI of the triple.</param>
return !GetGraph(graphUri).IsEmpty;
public void AssertToGraphUriNode(IGraph graph, Uri graphSubject, Uri graphPredicate, Uri? graphObject)
}
{
if (graphObject != null)
// Returns an empty graph, if the graph does not exists
public IGraph GetGraph(string graphName)
{
return GetGraph(new Uri(graphName));
}
// Returns an empty graph, if the graph does not exists
public IGraph GetGraph(Uri graphUri)
{
var graph = new WrapperGraph();
ReadWriteSparqlConnector.LoadGraph(graph, graphUri.AbsoluteUri);
graph.AssertList.Clear();
graph.RetractList.Clear();
return graph;
}
public void ClearGraph(string graphName)
{
ClearGraph(new Uri(graphName));
}
public void ClearGraph(Uri graphUri)
{
{
SparqlParameterizedString queryString = new SparqlParameterizedString
graph.Assert(
{
new Triple(
CommandText = "CLEAR GRAPH @graph"
graph.CreateUriNode(graphSubject),
};
graph.CreateUriNode(graphPredicate),
queryString.SetUri("graph", graphUri);
graph.CreateUriNode(graphObject)
QueryEndpoint.QueryRaw(queryString.ToString());
)
 
);
}
}
 
}
/// <summary>
/// <summary>
/// Retry Virtuoso Requests since they sometimes just fail
/// Asserts a triple to the graph with a literal node object.
/// </summary>
/// </summary>
/// <typeparam name="W"></typeparam>
/// <param name="graph">The graph to assert to.</param>
/// <param name="function"></param>
/// <param name="graphSubject">The subject URI of the triple.</param>
/// <returns></returns>
/// <param name="graphPredicate">The predicate URI of the triple.</param>
public void WrapRequest(Action action)
/// <param name="graphObject">The literal object of the triple.</param>
 
/// <param name="objectType">The data type URI of the literal object, if any.</param>
 
public void AssertToGraphLiteralNode(IGraph graph, Uri graphSubject, Uri graphPredicate, string? graphObject, Uri? objectType = null)
 
{
 
if (graphObject != null)
{
{
Policy
graph.Assert(
.Handle<Exception>()
new Triple(
.WaitAndRetry(5, retryNumber => TimeSpan.FromMilliseconds(200))
graph.CreateUriNode(graphSubject),
.Execute(() => action.Invoke());
graph.CreateUriNode(graphPredicate),
 
objectType is not null ? graph.CreateLiteralNode(graphObject, objectType) : graph.CreateLiteralNode(graphObject)
 
)
 
);
}
}
 
}
/// <summary>
/// <summary>
/// Retry Virtuoso Requests since they sometimes just fail
/// Asserts a triple to the graph with a blank node subject and a URI node object.
/// </summary>
/// </summary>
/// <typeparam name="W"></typeparam>
/// <param name="graph">The graph to assert to.</param>
/// <param name="function"></param>
/// <param name="graphSubject">The blank node subject of the triple.</param>
/// <returns></returns>
/// <param name="graphPredicate">The predicate URI of the triple.</param>
public W WrapRequest<W>(Func<W> function)
/// <param name="graphObject">The object URI of the triple.</param>
 
public void AssertToGraphBlankAndUriNode(IGraph graph, IBlankNode graphSubject, Uri graphPredicate, Uri? graphObject)
 
{
 
if (graphObject != null)
{
{
return Policy
graph.Assert(
.Handle<Exception>()
new Triple(
.WaitAndRetry(5, retryNumber => TimeSpan.FromMilliseconds(200))
graphSubject,
.ExecuteAndCapture(() => function.Invoke()).Result;
graph.CreateUriNode(graphPredicate),
 
graph.CreateUriNode(graphObject)
 
)
 
);
}
}
}
}
}
}