Deployer.cs

using Coscine.ApiClient;
using Coscine.ApiClient.Core.Api;
using Coscine.ApiClient.Core.Model;
using Coscine.GraphDeployer.Models.ConfigurationModels;
using Coscine.GraphDeployer.Utils;
using LibGit2Sharp;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Polly;
using VDS.RDF;
using static Coscine.GraphDeployer.Utils.CommandLineOptions;
using Configuration = Coscine.ApiClient.Core.Client.Configuration;

namespace Coscine.GraphDeployer;

public class Deployer
{
    private readonly ILogger<Deployer> _logger;
    private readonly GraphDeployerConfiguration _graphDeployerConfiguration;
    private readonly AdminApi _adminApi;

    public Deployer(ILogger<Deployer> logger, IOptionsMonitor<GraphDeployerConfiguration> graphDeployerConfiguration)
    {
        _logger = logger;
        _graphDeployerConfiguration = graphDeployerConfiguration.CurrentValue;
        // Build the configuration for the API client based on the configuration settings
        var apiClientConfig = new Configuration
        {
            BasePath = $"{_graphDeployerConfiguration.Endpoint.TrimEnd('/')}/coscine",
            ApiKeyPrefix = { { "Authorization", "Bearer" } },
            ApiKey = { { "Authorization", _graphDeployerConfiguration.ApiKey } },
            Timeout = 300000 // 5 minutes
        };

        // Check if the graph deployer has to skip SSL checks when connecting to the API
        if (_graphDeployerConfiguration.SkipSslCheck)
        {
            _logger.LogInformation("Skipping SSL certificate validation...");
            // Skip SSL certificate validation
            apiClientConfig.RemoteCertificateValidationCallback = (_, _, _, _) => true;
        }
        _adminApi = new(apiClientConfig);
    }

    public static string WorkingFolder { get; set; } = "./output/";
    public static List<string> DeployedGraphs { get; set; } = [];
    public static List<string> SkippedGraphs { get; set; } = [];

    // RDF URIs
    public static Uri CoscineEntitiesGraphDeployed { get; } = new("https://purl.org/coscine/entities/graph#deployed");
    public static Uri CoscineTermsDeployedVersion { get; } = new("https://purl.org/coscine/terms/deployed#version");

    public async Task<bool> RunAsync(GraphDeployerOptions opts)
    {
        // Check if the graph deployer is enabled 
        if (!_graphDeployerConfiguration.IsEnabled)
        {
            _logger.LogInformation("Graph Deployer is disabled in the configuration. Exiting...");
            return false;
        }

        // Log the current application execution mode
        if (opts.DummyMode)
        {
            _logger.LogInformation("Running in Dummy Mode. No changes will be made.");
        }
        _logger.LogDebug("Redeploy: {redeploy}", opts.Redeploy);

        // Override the working folder if specified in the configuration
        if (!string.IsNullOrWhiteSpace(_graphDeployerConfiguration.WorkingFolder))
        {
            WorkingFolder = _graphDeployerConfiguration.WorkingFolder;
        }

        var graphs = PaginationHelper.GetAllAsync<DeployedGraphDtoPagedResponse, DeployedGraphDto>(
            (currentPage) => _adminApi.GetDeployedGraphsAsync(pageNumber: currentPage, pageSize: 50));
        var deployedGraphsList = await graphs.ToListAsync();

        // Iterate over the repositories and deploy the graphs
        foreach (var graphRepo in _graphDeployerConfiguration.GitLab.Repositories)
        {
            _logger.LogInformation("Working with {repoName}", graphRepo.Name);

            // Clone the repository inside the Working Folder
            var success = CloneRepo(graphRepo.Url, WorkingFolder, _graphDeployerConfiguration.GitLab.Token, graphRepo.Ref);

            if (success)
            {
                // Graph deployment logic
                var queries = new List<string>();
                var turtleFiles = Directory.GetFiles(WorkingFolder, "*.ttl", SearchOption.AllDirectories);

                // Accumulate the graphs to be deployed, as they may be split across multiple files
                _logger.LogDebug("Computing accumulated graphs for deployment from {count} Turtle files...", turtleFiles.Length);
                var graphAccumulation = new Dictionary<Uri, (Graph, List<string>)>();
                Array.ForEach(turtleFiles, (file) =>
                {
                    try
                    {
                        var graph = new Graph();
                        graph.LoadFromFile(file);
                        if (graphAccumulation.TryGetValue(graph.BaseUri, out (Graph, List<string>) value))
                        {
                            value.Item1.Merge(graph);
                            value.Item2.Add(file);
                        }
                        else
                        {
                            graphAccumulation.Add(graph.BaseUri, (graph, new List<string>() { file }));
                        }
                    }
                    catch (Exception e)
                    {
                        _logger.LogError("Failed to load and process Turtle file: \"{file}\". Error: {errorMessage}", file, e.Message);
                    }
                });
                _logger.LogDebug("Accumulated {count} graphs for possible deployment.", graphAccumulation.Count);

                // Iterate over the accumulated graphs and deploy them
                foreach (var kv in graphAccumulation)
                {
                    var graph = kv.Value.Item1;
                    var files = kv.Value.Item2;
                    var graphId = kv.Key.ToString();
                    var currentRun = new Dictionary<string, string>();

                    _logger.LogDebug("Deploying graph: {graphName}", graphId);

                    // Get the hash of the currently deployed graph and compare it with the hash of the graph to be deployed
                    files.ForEach((path) => currentRun.TryAdd(graphId, HashUtil.GetFileHash(path)));

                    var deployedGraph = deployedGraphsList.FirstOrDefault((g) => g.Uri == graphId);
                    var hasChanged = deployedGraph is null || !deployedGraph.FileHashes.Contains(currentRun[graphId]) || opts.Redeploy;

                    if (hasChanged) 
                    {
                        _logger.LogDebug("The graph has changed");
                    } else 
                    {
                        _logger.LogDebug("The graph has not changed");
                    }

                    if(deployedGraph is null)
                    {
                        _logger.LogDebug("Deployed graph is null");
                    } else {
                        _logger.LogDebug("Deployed hash: {hash}", string.Join(',', deployedGraph.FileHashes));
                    }

                    _logger.LogDebug("Incoming hash: {hash}", currentRun[graphId]);

                    // Deploy the graph if it has changed or if the redeploy flag is set
                    if (hasChanged)
                    {
                        // Insert the information about the deployed version
                        var deployedGraphSubject = graph.CreateUriNode(CoscineEntitiesGraphDeployed);
                        var deployedVersionPredicate = graph.CreateUriNode(CoscineTermsDeployedVersion);
                        foreach (var fileHash in currentRun.Values)
                        {
                            graph.Assert(deployedGraphSubject, deployedVersionPredicate, graph.CreateLiteralNode(fileHash));
                        }

                        // Update the graph
                        var formatEnum = RdfFormat.TextTurtle;
                        var format = "text/turtle";
                        var rdfWriter = MimeTypesHelper.GetWriter(format);
                        var content = VDS.RDF.Writing.StringWriter.Write(graph, rdfWriter);
                        if (!opts.DummyMode)
                        {
                            // Retry the operation in case of failure up to 3 times using exponential backoff (2^retryAttempt seconds as the delay between retries)
                            await Policy.Handle<Exception>().WaitAndRetry(3, (retryAttempt) => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt))).Execute(async () =>
                            {
                                await _adminApi.UpdateMetadataGraphAsync(
                                    graph.BaseUri.AbsoluteUri,
                                    new MetadataUpdateAdminParameters(new RdfDefinitionForManipulationDto(content, formatEnum))
                                );
                            });
                        }

                        _logger.LogInformation("Deployed {graphName} successfully.", graphId);
                        DeployedGraphs.Add(graphId);
                    }
                    else
                    {
                        _logger.LogDebug("Skipped {graphName} as it has not changed.", graphId);
                        SkippedGraphs.Add(graphId);
                        continue;
                    }
                }
            }

            // Clean up the working folder
            EmptyWorkingFolder();
        }
        return true;
    }

    /// <summary>
    /// Clones a repository from a given URL into a specified local directory.
    /// </summary>
    /// <param name="projectUrl">The URL of the project repository to clone.</param>
    /// <param name="projectPath">The local path where the repository should be cloned to.</param>
    /// <param name="token">The token used for authentication with the repository host.</param>
    /// <param name="branchName">Optional. The specific branch name to clone. If <c>null</c>, the default branch is cloned.</param>
    /// <returns>Returns <c>true</c> if the repository was successfully cloned, otherwise <c>false</c>.</returns>
    private bool CloneRepo(Uri projectUrl, string projectPath, string token, string? branchName)
    {
        // Ensure the target directory is empty, if it exists
        EmptyWorkingFolder();
        // Ensure the target directory exists
        Directory.CreateDirectory(projectPath);

        // Prepare the repository clone URL with authentication token
        var gitLink = projectUrl.AbsoluteUri.Replace("https://", "").Replace("http://", "");
        string url = $"https://gitlab-ci-token:{token}@{gitLink}";
        try
        {
            var cloneOptions = new CloneOptions();

            // Perform the clone operation
            _logger.LogDebug("Starting clone of repository: \"{projectUrl}\" into \"{projectPath}\"", projectUrl, projectPath);
            var repo = Repository.Clone(url, projectPath);
            var localRepo = new Repository(repo);

            // First clone on, then checkout, as direct cloning of commit SHA is not supported
            if (!string.IsNullOrWhiteSpace(branchName))
            {
                Commands.Checkout(localRepo, branchName);
            }

            // Retrieve the reference of the repository, either the branch name or the commit hash
            var repoRef = localRepo.Head.IsTracking ? localRepo.Head.FriendlyName : localRepo.Head.Tip.Sha;
            _logger.LogInformation("Repository successfully cloned and switched on ref \"{ref}\".", repoRef);

            return true;
        }
        catch (LibGit2SharpException e)
        {
            _logger.LogError(e, "LibGit2Sharp-specific error while cloning the repository.");
            return false;
        }
        catch (Exception e)
        {
            _logger.LogError(e, "General error while cloning the repository.");
            return false;
        }
    }

    /// <summary>
    /// Empties all files and directories from the working folder.
    /// </summary>
    public void EmptyWorkingFolder()
    {
        try
        {
            var directory = new DirectoryInfo(WorkingFolder);
            if (!directory.Exists)
            {
                _logger.LogDebug("The specified directory does not exist: \"{workingFolder}\"", WorkingFolder);
                return;
            }

            // Retrieve all files and directories
            var files = directory.EnumerateFiles("*", SearchOption.AllDirectories);
            var directories = directory.EnumerateDirectories("*", SearchOption.AllDirectories).OrderByDescending(d => d.FullName.Length);

            _logger.LogDebug("Deleting files and directories from \"{workingFolder}\".", WorkingFolder);

            // First delete all files
            foreach (var file in files)
            {
                file.Attributes = FileAttributes.Normal;
                file.Delete();
            }

            // Then delete all directories from deepest to shallowest
            foreach (var dir in directories)
            {
                dir.Delete(recursive: true); // Remove directories and their contents
            }

            _logger.LogDebug("Successfully cleared the working folder: \"{workingFolder}\"", WorkingFolder);
        }
        catch (Exception e)
        {
            _logger.LogError(e, "Failed to delete contents of the working folder: \"{workingFolder}\"", WorkingFolder);
        }
    }
}