Program.cs

using Coscine.Configuration;
using Coscine.GraphDeployer.Logging;
using Coscine.Metadata;
using LibGit2Sharp;
using Microsoft.Extensions.Logging;
using NLog.Config;
using NLog.Extensions.Logging;
using System.Diagnostics;
using VDS.RDF;
using VDS.RDF.Storage;

namespace Coscine.GraphDeployer;

public class Program
{
    public static ConsulConfiguration Configuration { get; } = new ConsulConfiguration();
    public static string WorkingFolder { get; } = Path.GetFullPath("C:/voc");
    public static string GraphsConsulLocation { get; } = "coscine/local/graphs";
    private static string Token { get; set; } = string.Empty;

    private static ILogger _logger = null!;

    public static void Main(string[] args)
    {
        ConfigurationItemFactory.Default.LayoutRenderers.RegisterDefinition("assembly-name", typeof(AssemblyNameLayoutRenderer));
        ConfigurationItemFactory.Default.LayoutRenderers.RegisterDefinition("assembly-version", typeof(AssemblyVersionLayoutRenderer));
        _logger = LoggerFactory.Create(builder => builder.AddNLog()).CreateLogger<Program>();
        try
        {
            Token = Configuration.GetStringAndWait("coscine/global/gitlabtoken");
            Run();
        }
        catch (Exception ex)
        {
            // Handle exceptions by logging them in
            _logger.LogError(ex, ex.Message);
            LogInnerException(ex);
            throw;
        }
    }

    private static void Run()
    {
        var virtuosoServer = Configuration.GetString("coscine/local/virtuoso/additional/url");
        var virtuosoHost = new Uri(virtuosoServer).Host;
        var virtuosoUser = Configuration.GetString("coscine/global/virtuoso_db_user");
        var virtuosoPassword = Configuration.GetString("coscine/global/virtuoso_db_password");

        var virtuosoManager = new VirtuosoManager($"Server={virtuosoHost};Uid={virtuosoUser};pwd={virtuosoPassword}");
        var _rdfStoreConnector = new RdfStoreConnector(virtuosoServer);

        _logger.LogInformation("Connecting to Virtuoso Server - {serverUrl}", virtuosoServer);

        var virtuosoISQLLocation = Configuration.GetString(
            "coscine/local/virtuoso/isql",
            "C:/Programs/Virtuoso/bin/isql.exe"
        );

        // Collect Graphs Types to deploy from Consul
        var graphRepos = new List<GraphRepo>();
        var graphRepoKeys = Configuration.Keys(GraphsConsulLocation).Select(x =>
        {
            x = x.Replace($"{GraphsConsulLocation}/", "");
            return x[..x.IndexOf('/')];
        }).Distinct().ToList();
        graphRepoKeys.ForEach(e =>
        {
            var graphRepo = new GraphRepo()
            {
                Name = e,
                Branch = Configuration.GetStringAndWait(Path.Combine(GraphsConsulLocation, e, "branch")),
                Url = Configuration.GetStringAndWait(Path.Combine(GraphsConsulLocation, e, "repositoryurl"))
            };
            _logger.LogInformation("Found Graph Repository Url - {@graphRepo}", graphRepo);
            graphRepos.Add(graphRepo);
        });

        foreach (var graphRepo in graphRepos)
        {
            _logger.LogInformation("Working with {repoName}", graphRepo.Name.ToUpper());

            // Clear contents inside Working Folder
            EmptyWorkingFolder();

            // Clone the repository inside the Working Folder
            CloneGraphRepo(graphRepo, out var success);

            if (success)
            {
                // Graph Insertion
                var queries = new List<string>();
                var turtleFiles = Directory.GetFiles(WorkingFolder, "*.ttl", SearchOption.AllDirectories);

                var graphAccumulation = new Dictionary<Uri, (Graph, List<string>)>();
                Array.ForEach(turtleFiles, (file) =>
                {
                    var graph = new Graph();
                    graph.LoadFromFile(file);
                    if (graphAccumulation.ContainsKey(graph.BaseUri))
                    {
                        graphAccumulation[graph.BaseUri].Item1.Merge(graph);
                        graphAccumulation[graph.BaseUri].Item2.Add(file);
                    }
                    else
                    {
                        graphAccumulation.Add(graph.BaseUri, (graph, new List<string>() { file }));
                    }
                });

                foreach (var kv in graphAccumulation)
                {
                    var graph = kv.Value.Item1;
                    var graphName = kv.Key.ToString();

                    var currentGraph = Helpers.WrapRequest(() => _rdfStoreConnector.GetGraph(graphName));

                    var graphWasChanged = graph.Triples.Count != currentGraph.Triples.Count
                        || graph.Triples.Any((triple) => !currentGraph.Triples.Any((currentTriple) =>
                            (triple.Subject.Equals(currentTriple.Subject) || (triple.Subject.NodeType == NodeType.Blank && currentTriple.Subject.NodeType == NodeType.Blank)
                            && triple.Predicate.Equals(currentTriple.Predicate)
                            && triple.Object.Equals(currentTriple.Object) || (triple.Object.NodeType == NodeType.Blank && currentTriple.Object.NodeType == NodeType.Blank))));

                    if (graphWasChanged)
                    {
                        if (!currentGraph.IsEmpty)
                        {
                            Helpers.WrapRequest(() => _rdfStoreConnector.ClearGraph(graphName));
                            _logger.LogInformation("Cleared Graph {graphName}", graphName);
                        }
                        else
                        {
                            _logger.LogInformation("No Graph {graphName}", graphName);
                        }

                        foreach (var file in kv.Value.Item2)
                        {
                            var fileInfo = new FileInfo(file);
                            queries.Add($"ld_dir('{fileInfo.DirectoryName[2..].Replace("\\", "/")}', '{fileInfo.Name}', '{graphName}');");
                        }
                    }
                    else
                    {
                        _logger.LogInformation("Skipping {graphName}", graphName);

                    }
                }
                queries.Add($"rdf_loader_run ();");
                queries.Add($"DELETE from DB.DBA.load_list where 1=1;");

                foreach (var query in queries)
                {
                    ExecuteCommand(
                        "powershell.exe",
                        $"\"\\\"{query}\\\" | {virtuosoISQLLocation}\""
                    );
                    _logger.LogInformation("Query executed {@query}", query);
                }
            }
        }

        _logger.LogInformation("Done");
    }

    private static void ExecuteCommand(string fileName, string arguments)
    {
        var startInfo = new ProcessStartInfo
        {
            FileName = fileName,
            Arguments = arguments,
            RedirectStandardOutput = true,
            RedirectStandardError = true,
            UseShellExecute = false,
            CreateNoWindow = true,
        };
        using var process = new Process
        {
            StartInfo = startInfo
        };
        process.Start();

        string output = process.StandardOutput.ReadToEnd();
        if (!string.IsNullOrWhiteSpace(output))
            _logger.LogDebug("Executed queries {filename} {arguments} >>> {output}", fileName, arguments, output);

        string errors = process.StandardError.ReadToEnd();
        if (!string.IsNullOrWhiteSpace(errors))
            _logger.LogError("Errors during execution of command {filename} {arguments} >>> {errors}", fileName, arguments, errors);
    }

    public static void CloneGraphRepo(GraphRepo graphRepo, out bool success)
    {
        string url = $"https://gitlab-ci-token:{Token}@{graphRepo.Url}";
        var cloneOptions = new CloneOptions()
        {
            BranchName = graphRepo.Branch
        };
        try
        {
            var cloneResult = Repository.Clone(url, WorkingFolder, cloneOptions);
            using var repo = new Repository(WorkingFolder);
            _logger.LogInformation("Repository successfully cloned and switched to branch {branchName}", repo.Head.FriendlyName);
            success = true;
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Cloning Repository {repoUrl} on branch {branchName}", graphRepo.Url, graphRepo.Branch);
            success = false;
        }
    }

    public static void EmptyWorkingFolder()
    {
        var directory = new DirectoryInfo(WorkingFolder);
        var filesToDelete = directory.EnumerateFiles("*.*", SearchOption.AllDirectories);
        var dirsToDelete = directory.EnumerateDirectories();
        _logger.LogInformation("Deleting {fileCount} files and {directoryCount} directories from {workingFolder}.", filesToDelete.Count(), dirsToDelete.Count(), WorkingFolder);
        foreach (var file in filesToDelete)
        {
            file.Attributes = FileAttributes.Normal;
            file.Delete();
        }
        foreach (var dir in dirsToDelete)
        {
            dir.Delete(true);
        }
    }

    private static void LogInnerException(Exception ex)
    {
        if (ex.InnerException is not null)
        {
            _logger.LogError(ex.InnerException, "InnerException: {innerException}", ex.InnerException.Message);
            LogInnerException(ex.InnerException);
        }
    }
}