diff --git a/src/GraphDeployer/GraphDeployer.csproj b/src/GraphDeployer/GraphDeployer.csproj index 986062122079385d7a6cd52f257664627e151c8b..fa9711d1764bf7f9abf0aaf8683a7fc0f1fb7a9e 100644 --- a/src/GraphDeployer/GraphDeployer.csproj +++ b/src/GraphDeployer/GraphDeployer.csproj @@ -27,6 +27,7 @@ <PackageReference Include="LibGit2Sharp" Version="0.26.2" /> <PackageReference Include="NLog" Version="5.1.0" /> <PackageReference Include="NLog.Extensions.Logging" Version="5.2.0" /> + <PackageReference Include="Polly.Extensions.Http" Version="3.0.0" /> </ItemGroup> <ItemGroup> diff --git a/src/GraphDeployer/Helpers.cs b/src/GraphDeployer/Helpers.cs new file mode 100644 index 0000000000000000000000000000000000000000..bc5e66bfa81ba85ae74a446256c1a81df4df9030 --- /dev/null +++ b/src/GraphDeployer/Helpers.cs @@ -0,0 +1,35 @@ +using Polly; + +namespace Coscine.GraphDeployer; + +public static class Helpers +{ + /// <summary> + /// Retry Virtuoso Requests since they sometimes just fail + /// </summary> + /// <typeparam name="W"></typeparam> + /// <param name="function"></param> + /// <returns></returns> + public static void WrapRequest(Action action) + { + Policy + .Handle<Exception>() + .WaitAndRetry(5, retryNumber => TimeSpan.FromMilliseconds(200)) + .Execute(() => action.Invoke()); + } + + /// <summary> + /// Retry Virtuoso Requests since they sometimes just fail + /// </summary> + /// <typeparam name="W"></typeparam> + /// <param name="function"></param> + /// <returns></returns> + public static W WrapRequest<W>(Func<W> function) + { + return Policy + .Handle<Exception>() + .WaitAndRetry(5, retryNumber => TimeSpan.FromMilliseconds(200)) + .ExecuteAndCapture(() => function.Invoke()).Result; + } + +} diff --git a/src/GraphDeployer/Program.cs b/src/GraphDeployer/Program.cs index bfaf7d4841140bbc4dd569bd72a3111d1d81090c..3ff8947d6e5432f66fdc73f72d6da99dbfc32a91 100644 --- a/src/GraphDeployer/Program.cs +++ b/src/GraphDeployer/Program.cs @@ -90,24 +90,59 @@ public class Program // Graph Insertion var queries = new List<string>(); var turtleFiles = Directory.GetFiles(WorkingFolder, "*.ttl", SearchOption.AllDirectories); - foreach (var file in turtleFiles) + + var graphAccumulation = new Dictionary<Uri, (Graph, List<string>)>(); + Array.ForEach(turtleFiles, (file) => { - var fileInfo = new FileInfo(file); var graph = new Graph(); graph.LoadFromFile(file); - var graphName = graph.BaseUri.ToString(); - - if (_rdfStoreConnector.HasGraph(graphName)) + if (graphAccumulation.ContainsKey(graph.BaseUri)) { - _rdfStoreConnector.ClearGraph(graphName); - _logger.LogInformation("Cleared Graph {graphName}", graphName); + graphAccumulation[graph.BaseUri].Item1.Merge(graph); + graphAccumulation[graph.BaseUri].Item2.Add(file); } else { - _logger.LogInformation("No Graph {graphName}", graphName); + graphAccumulation.Add(graph.BaseUri, (graph, new List<string>() { file })); + } + }); + + foreach (var kv in graphAccumulation) + { + var graph = kv.Value.Item1; + var graphName = kv.Key.ToString(); + + var currentGraph = Helpers.WrapRequest(() => _rdfStoreConnector.GetGraph(graphName)); + + var graphWasChanged = graph.Triples.Count != currentGraph.Triples.Count + || graph.Triples.Any((triple) => !currentGraph.Triples.Any((currentTriple) => + (triple.Subject.Equals(currentTriple.Subject) || (triple.Subject.NodeType == NodeType.Blank && currentTriple.Subject.NodeType == NodeType.Blank) + && triple.Predicate.Equals(currentTriple.Predicate) + && triple.Object.Equals(currentTriple.Object) || (triple.Object.NodeType == NodeType.Blank && currentTriple.Object.NodeType == NodeType.Blank)))); + + if (graphWasChanged) + { + if (!currentGraph.IsEmpty) + { + Helpers.WrapRequest(() => _rdfStoreConnector.ClearGraph(graphName)); + _logger.LogInformation("Cleared Graph {graphName}", graphName); + } + else + { + _logger.LogInformation("No Graph {graphName}", graphName); + } + + foreach (var file in kv.Value.Item2) + { + var fileInfo = new FileInfo(file); + queries.Add($"ld_dir('{fileInfo.DirectoryName[2..].Replace("\\", "/")}', '{fileInfo.Name}', '{graphName}');"); + } } + else + { + _logger.LogInformation("Skipping {graphName}", graphName); - queries.Add($"ld_dir('{fileInfo.DirectoryName[2..].Replace("\\", "/")}', '{fileInfo.Name}', '{graphName}');"); + } } queries.Add($"rdf_loader_run ();"); queries.Add($"DELETE from DB.DBA.load_list where 1=1;");