StructuralData.cs

using Coscine.Configuration;
using Coscine.Database.Models;
using Coscine.Metadata;
using Polly;
using VDS.RDF;

namespace SQL2Linked
{
    public abstract class StructuralData<S, T> where S : class where T : DatabaseModel<S>, new()
    {
        public T Model { get; init; }
        public ConsulConfiguration Configuration { get; init; }
        public RdfStoreConnector RdfStoreConnector { get; init; }
        public static string Prefix { get; set; }

        public StructuralData()
        {
            Configuration = new ConsulConfiguration();
            RdfStoreConnector = new RdfStoreConnector(Configuration.GetStringAndWait("coscine/local/virtuoso/additional/url"));
            Model = new T();
            Prefix = Configuration.GetStringAndWait("coscine/global/epic/prefix");
            // 100 second timeout
            var timeout = 100000;
            RdfStoreConnector.QueryEndpoint.Timeout = timeout;
            RdfStoreConnector.UpdateEndpoint.Timeout = timeout;
            RdfStoreConnector.ReadWriteSparqlConnector.Timeout = timeout;
        }

        public abstract IEnumerable<IGraph> ConvertToLinkedData(IEnumerable<S> entries);

        public virtual IEnumerable<S> GetAll()
        {
            return Model.GetAll();
        }

        public void Migrate(bool dummyMode)
        {
            var spacer = new string('-', 35);
            Console.WriteLine($"\n{spacer}\n{typeof(T).Name}\n{spacer}");
            var graphs = ConvertToLinkedData(GetAll());
            if (!dummyMode)
            {
                StoreGraphs(graphs);
            }
        }

        public void StoreGraphs(IEnumerable<IGraph> graphs)
        {
            foreach (var graph in graphs)
            {
                try
                {
                    Console.WriteLine($" ({graph.BaseUri})");

                    if (graph is WrapperGraph)
                    {
                        var wrapperGraph = (WrapperGraph)graph;
                        // Chunking since the size otherwise can be too large
                        foreach (var triples in wrapperGraph.AssertList.Chunk(100))
                        {
                            WrapRequest(() => RdfStoreConnector.ReadWriteSparqlConnector.UpdateGraph(graph.BaseUri, triples, new List<Triple>()));
                        }
                        // Chunking since the size otherwise can be too large
                        foreach (var triples in wrapperGraph.RetractList.Chunk(100))
                        {
                            WrapRequest(() => RdfStoreConnector.ReadWriteSparqlConnector.UpdateGraph(graph.BaseUri, new List<Triple>(), triples));
                        }
                    }
                    else
                    {
                        var exists = WrapRequest(() => RdfStoreConnector.HasGraph(graph.BaseUri));
                        if (exists)
                        {
                            Console.WriteLine($" - Graph {graph.BaseUri} exists");

                            // Clear the existing graph from the store
                            WrapRequest(() => RdfStoreConnector.ClearGraph(graph.BaseUri));
                            Console.WriteLine($" - Cleared Graph {graph.BaseUri}");
                        }

                        // Chunking since the size otherwise can be too large
                        // Don't change to only addition of triples, otherwise this could break things
                        foreach (var triples in graph.Triples.Chunk(100))
                        {
                            WrapRequest(() => RdfStoreConnector.ReadWriteSparqlConnector.UpdateGraph(graph.BaseUri, triples, new List<Triple>()));
                        }
                    }

                    Console.WriteLine($" - Graph {graph.BaseUri} added successfully");
                    Console.WriteLine();
                }
                catch (Exception e)
                {
                    Console.Error.WriteLine($"Error on ({graph.BaseUri}):");
                    Console.Error.WriteLine(e);
                    Console.Error.WriteLine(e.InnerException);
                    Console.Error.WriteLine();
                }
            }
        }

        public void AssertToGraphUriNode(IGraph graph, string graphSubject, string graphPredicate, string? graphObject)
        {
            if (graphObject != null)
            {
                graph.Assert(
                    new Triple(
                        graph.CreateUriNode(new Uri(graphSubject)),
                        graph.CreateUriNode(new Uri(graphPredicate)),
                        graph.CreateUriNode(new Uri(graphObject))
                    )
                );
            }
        }

        public void AssertToGraphLiteralNode(IGraph graph, string graphSubject, string graphPredicate, string? graphObject, Uri? objectType = null)
        {
            if (graphObject != null)
            {
                graph.Assert(
                    new Triple(
                        graph.CreateUriNode(new Uri(graphSubject)),
                        graph.CreateUriNode(new Uri(graphPredicate)),
                        graph.CreateLiteralNode(graphObject, objectType)
                    )
                );
            }
        }

        public void AssertToGraphBlankAndUriNode(IGraph graph, IBlankNode graphSubject, string graphPredicate, string? graphObject)
        {
            if (graphObject != null)
            {
                graph.Assert(
                    new Triple(
                        graphSubject,
                        graph.CreateUriNode(new Uri(graphPredicate)),
                        graph.CreateUriNode(new Uri(graphObject))
                    )
                );
            }
        }

        /// <summary>
        /// Retry Virtuoso Requests since they sometimes just fail
        /// </summary>
        /// <typeparam name="W"></typeparam>
        /// <param name="function"></param>
        /// <returns></returns>
        public void WrapRequest(Action action)
        {
            Policy
                .Handle<Exception>()
                .WaitAndRetry(5, retryNumber => TimeSpan.FromMilliseconds(200))
                .Execute(() => action.Invoke());
        }

        /// <summary>
        /// Retry Virtuoso Requests since they sometimes just fail
        /// </summary>
        /// <typeparam name="W"></typeparam>
        /// <param name="function"></param>
        /// <returns></returns>
        public W WrapRequest<W>(Func<W> function)
        {
            return Policy
                .Handle<Exception>()
                .WaitAndRetry(5, retryNumber => TimeSpan.FromMilliseconds(200))
                .ExecuteAndCapture(() => function.Invoke()).Result;
        }
    }
}