New: Containerization (coscine/issues#2906)

.devcontainer/Dockerfile

+# Development container for dotnet
+
+FROM mcr.microsoft.com/devcontainers/dotnet:8.0 as develop
+
+USER vscode
\ No newline at end of file

.devcontainer/compose.yml

+version: "3.9"
+
+services:
+
+  app:
+    build:
+      target: develop
+    user: vscode
+    userns_mode: keep-id:uid=1000,gid=1000
+    environment:
+      SSH_AUTH_SOCK: /.ssh/ssh-agent.sock
+    command: /bin/sh -c "while sleep 2s; do :; done"
+    volumes:
+      - ..:/workspace/graph-deployer:cached
+      - ./.vscode-server:/home/vscode/.vscode-server/:cached
+      - ${SSH_AUTH_SOCK}:/.ssh/ssh-agent.sock
+    networks:
+      - default
+
+networks:
+  default:
+    driver: bridge

.devcontainer/devcontainer.json

+{
+    "name": "Graph Deployer",
+    "dockerComposeFile": [
+        "compose.yml"
+    ],
+    "service": "app",
+    "workspaceFolder": "/workspace/graph-deployer",
+    "updateRemoteUserUID": false,
+    "remoteUser": "vscode",
+    "containerUser": "vscode",
+    "customizations": {
+        "vscode": {
+            "settings": {},
+            "extensions": [
+                "ms-dotnettools.csdevkit",
+                "ms-azuretools.vscode-docker",
+                "ms-dotnettools.csharp",
+                "mhutchie.git-graph",
+                "mutantdino.resourcemonitor"
+            ]
+        }
+    },
+    "forwardPorts": []
+}
\ No newline at end of file

.gitignore

+/.devcontainer/.vscode-server/*
+!/.devcontainer/.vscode-server/.gitkeep
+
+/.devcontainer/.data/*
+!/.devcontainer/.data/.gitkeep
+
+.logs/
+Output/*
+internallog.txt
+
+## Ignore the secrets settings
+appsettings.Development.json
+appsettings.Staging.json
+appsettings.Production.json
+
 ## Ignore Visual Studio temporary files, build results, and
 ## files generated by popular Visual Studio add-ons.
 

.gitlab-ci.yml

@@ -2,6 +2,7 @@ include:
   - project: coscine/tools/gitlab-ci-templates
     file: 
       - /dotnet.yml
+      - /container.yml
 
 stages:
   - build
@@ -21,3 +22,8 @@ publish-gitlab-release:
   
 publish:
   extends: .publish-artifact-release
+
+publish-container:
+  variables:
+    CONTAINER_CONTEXT_FOLDER_PATH: $CI_PROJECT_DIR/src/$DOTNET_MAIN_PROJECT_FOLDER
+  extends: .publish-container-gitlab

.vscode/launch.json

+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": 
+    [
+        {
+            "name": "C#: GraphDeployer Debug",
+            "type": "coreclr",
+            "request": "launch",
+            "preLaunchTask": "dotnet: build",
+            "program": "${workspaceFolder}/src/GraphDeployer/bin/Debug/net8.0/Coscine.GraphDeployer.dll",
+            "args": [],
+            "cwd": "${workspaceFolder}/src/GraphDeployer",
+            "stopAtEntry": false,
+            "console": "internalConsole",
+            "env": {
+                "DOTNET_ENVIRONMENT": "Development"
+            }
+        }
+    ]
+}
\ No newline at end of file

.vscode/settings.json

+{
+    "resmon.disk.format": "UsedOutOfTotal",
+    "resmon.show.battery": false,
+    "resmon.show.cpufreq": false,
+    "resmon.show.disk": true,
+    "resmon.disk.drives": null
+}
\ No newline at end of file

.vscode/tasks.json

+{
+	"version": "2.0.0",
+	"tasks": [
+		{
+			"type": "dotnet",
+			"task": "build",
+			"group": {
+				"kind": "build",
+				"isDefault": true
+			},
+			"problemMatcher": [],
+			"label": "dotnet: build"
+		}
+	]
+}
\ No newline at end of file

README.md

@@ -6,6 +6,25 @@
 
 The Graph Deployer, intended to be used as a CRON job, is a .NET application designed to manage and deploy RDF graphs. It interacts with configured GitLab repositories to pull RDF graph definitions and updates a centralized repository if changes are detected or redeployment is triggered. The job uses a series of configurations to determine the operational parameters including the execution environment, repository access, and deployment specifications.
 
+## Getting Started
+
+Clone the repository:
+
+```bash
+$ mkdir -p ~/code/git.rwth-aachen.de/coscine/backend/scripts && cd "$_"
+$ git clone git@git.rwth-aachen.de:coscine/backend/scripts/graphdeployer.git
+```
+
+Open the new folder in code and in a [devcontainer](https://code.visualstudio.com/docs/devcontainers/containers#_open-a-folder-on-a-remote-ssh-host-in-a-container).
+
+Insider the devcontainer, create a development settings file:
+
+```bash
+$ cp /workspace/graph-deployer/src/GraphDeployer/appsettings.Example.json /workspace/graph-deployer/src/GraphDeployer/appsettings.Development.json
+```
+
+In the `appsettings.Development.json` Replace any `null` value with a fitting value.
+
 ### Features
 - **Configuration-driven**: Behavior driven by settings defined in configuration files and environment variables (see `appsettings.json`).
 - **Supports Dummy Mode**: Can run in a non-operative mode for testing configurations and process flow without affecting live data (CLI arg `--dummy`).

src/GraphDeployer/Deployer.cs

@@ -13,19 +13,24 @@ using Configuration = Coscine.ApiClient.Core.Client.Configuration;
 
 namespace Coscine.GraphDeployer;
 
-public class Deployer(ILogger<Deployer> logger, IOptionsMonitor<GraphDeployerConfiguration> graphDeployerConfiguration)
+public class Deployer
 {
-    private readonly ILogger<Deployer> _logger = logger;
-    private readonly GraphDeployerConfiguration _graphDeployerConfiguration = graphDeployerConfiguration.CurrentValue;
-    private readonly AdminApi _adminApi = new(_configuration);
+    private readonly ILogger<Deployer> _logger;
+    private readonly GraphDeployerConfiguration _graphDeployerConfiguration;
+    private readonly AdminApi _adminApi;
 
-    private static readonly Configuration _configuration = new()
+    public Deployer(ILogger<Deployer> logger, IOptionsMonitor<GraphDeployerConfiguration> graphDeployerConfiguration)
     {
-        BasePath = "http://localhost:7206/coscine",
+        _logger = logger;
+        _graphDeployerConfiguration = graphDeployerConfiguration.CurrentValue;
+        _adminApi = new(new Configuration
+        {
+            BasePath = $"{_graphDeployerConfiguration.Endpoint.TrimEnd('/')}/coscine",
             ApiKeyPrefix = { { "Authorization", "Bearer" } },
-        ApiKey = { { "Authorization", ApiConfigurationUtil.GenerateAdminToken(ApiConfigurationUtil.RetrieveJwtConfiguration()) } },
+            ApiKey = { { "Authorization", _graphDeployerConfiguration.ApiKey } },
             Timeout = 300000 // 5 minutes
-    };
+        });
+    }
 
     public static string WorkingFolder { get; set; } = "./output/";
     public static List<string> DeployedGraphs { get; set; } = [];

src/GraphDeployer/Dockerfile

+FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build-env
+WORKDIR /App
+
+# Copy everything
+COPY . ./
+
+# Restore as distinct layers
+RUN dotnet restore
+
+# Build and publish a release
+RUN dotnet publish -c Release -o out
+
+# Build runtime image
+FROM mcr.microsoft.com/dotnet/aspnet:8.0
+
+# Install cron and other necessary packages
+RUN apt-get update && \
+    apt-get install -y cron && \
+    apt-get clean
+
+WORKDIR /App
+COPY --from=build-env /App/out .
+
+# Copy the cron job file to /etc/cron.d/
+COPY graph-deployer-cron /etc/cron.d/
+
+# Give execution rights on the cron job file
+RUN chmod 0644 /etc/cron.d/graph-deployer-cron
+
+# Apply the cron job
+RUN crontab /etc/cron.d/graph-deployer-cron
+
+# Set the build-time argument and default environment variable
+ENV DOTNET_ENVIRONMENT=Development
+
+# Create the log file
+RUN touch /var/log/cron.log
+
+ENTRYPOINT /bin/sh -c "cron;/App/Coscine.GraphDeployer;tail -f /var/log/cron.log"

src/GraphDeployer/GraphDeployer.csproj

@@ -22,7 +22,7 @@
 
 	<ItemGroup>
 		<PackageReference Include="CommandLineParser" Version="2.9.1" />
-		<PackageReference Include="Coscine.ApiClient" Version="1.7.0-issue-2668-graph0001" />
+		<PackageReference Include="Coscine.ApiClient" Version="1.8.0" />
 		<PackageReference Include="dotNetRdf" Version="3.1.1" />
 		<PackageReference Include="GitLabApiClient" Version="1.8.1-beta.5" />
 		<PackageReference Include="LibGit2Sharp" Version="0.30.0" />
@@ -40,6 +40,9 @@
 		<None Update="appsettings.json">
 		  <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
 		</None>
+		<None Update="appsettings.development.json">
+		  <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+		</None>
 		<None Update="nlog.config">
 			<CopyToOutputDirectory>Always</CopyToOutputDirectory>
 		</None>

src/GraphDeployer/Models/ConfigurationModels/GraphDeployerConfiguration.cs

@@ -17,6 +17,17 @@ public class GraphDeployerConfiguration
     /// </summary>
     public string? WorkingFolder { get; init; }
 
+    /// <summary>
+    /// API token for the coscine API.
+    /// Requires the administrator role.
+    /// </summary>
+    public string ApiKey { get; set; } = null!;
+
+    /// <summary>
+    /// Address of the Coscine API.
+    /// </summary>
+    public string Endpoint { get; set; } = null!;
+
     /// <summary>
     /// Logger configuration settings.
     /// </summary>

src/GraphDeployer/Program.cs

@@ -15,8 +15,14 @@ public class Program
 {
     private static IServiceProvider _serviceProvider = null!;
 
+    private static string? _environmentName;
+
     public static int Main(string[] args)
     {
+        _environmentName = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT");
+
+        Console.WriteLine($"Running in environment: {_environmentName ?? "environment is null"}");
+
         InitializeServices();
 
         var logger = _serviceProvider.GetRequiredService<ILogger<Program>>();
@@ -65,7 +71,8 @@ public class Program
 
         // Add Consul as a configuration source
         var configuration = configBuilder
-            .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true)
+            .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
+            .AddJsonFile($"appsettings.{_environmentName}.json", optional: true, reloadOnChange: true)
             .AddConsul(
                 "coscine/Coscine.Infrastructure/GraphDeployer/appsettings",
                 options =>
@@ -78,6 +85,18 @@ public class Program
                     options.OnLoadException = exceptionContext => exceptionContext.Ignore = true;
                 }
             )
+            .AddConsul(
+                $"coscine/Coscine.Infrastructure/GraphDeployer/appsettings.{_environmentName}.json",
+                options =>
+                {
+                    options.ConsulConfigurationOptions =
+                        cco => cco.Address = new Uri(consulUrl);
+                    options.Optional = true;
+                    options.ReloadOnChange = true;
+                    options.PollWaitTime = TimeSpan.FromSeconds(5);
+                    options.OnLoadException = exceptionContext => exceptionContext.Ignore = true;
+                }
+            )
             .AddEnvironmentVariables()
             .Build();
 
@@ -100,6 +119,17 @@ public class Program
 
         var graphDeployerConfiguration = new GraphDeployerConfiguration();
         configuration.Bind(GraphDeployerConfiguration.Section, graphDeployerConfiguration);
+
+        if (graphDeployerConfiguration.Endpoint is null)
+        {
+            throw new ArgumentNullException(nameof(graphDeployerConfiguration.Endpoint), "Endpoint cannot be null.");
+        }
+
+        if (graphDeployerConfiguration.ApiKey is null)
+        {
+            throw new ArgumentNullException(nameof(graphDeployerConfiguration.ApiKey), "ApiKey cannot be null.");
+        }
+
         // Set the default LogLevel
         LogManager.Configuration.Variables["logLevel"] = graphDeployerConfiguration.Logger?.LogLevel ?? "Trace";
         // Set the log location

src/GraphDeployer/appsettings.Example.json

+{
+  "GraphDeployerConfiguration": {
+    "ApiKey": null,
+    "Endpoint": null,
+    "GitLab": {
+      "Token": null
+    }
+  }
+}
\ No newline at end of file

src/GraphDeployer/appsettings.json

 {
   "GraphDeployerConfiguration": {
+    "ApiKey": null,
+    "Endpoint": null,
     "IsEnabled": true,
-    "WorkingFolder": "C:/coscine/GraphDeployer/output/",
+    "WorkingFolder": "./Output/",
     "Logger": {
       "LogLevel": "Information",
-      "LogHome": "C:/coscine/GraphDeployer/"
+      "LogHome": "./Logs"
     },
     "GitLab": {
       "HostUrl": "https://git.rwth-aachen.de/",
-      "Token": "glpat-iXFFtZSjmbhn8U4-YFzv",
+      "Token": null,
       "Repositories": [
         {
           "Name": "Application Profiles",
-          "Url": "https://git.rwth-aachen.de/coscine/graphs/applicationprofiles.git",
-          "Branch": "master"
+          "Url": "https://git.rwth-aachen.de/coscine/graphs/applicationprofiles.git"
         },
         {
           "Name": "Organisations",
-          "Url": "https://git.rwth-aachen.de/coscine/graphs/organizations.git",
-          "Branch": "master"
+          "Url": "https://git.rwth-aachen.de/coscine/graphs/organizations.git"
         },
         {
           "Name": "FH Bielefeld",

src/GraphDeployer/graph-deployer-cron

+* 3 * * * root /App/Coscine.GraphDeployer >> /var/log/cron.log 2>&1

src/GraphDeployer/nlog.config

@@ -3,7 +3,7 @@
       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         autoReload="true"
         internalLogLevel="Warn"
-        internalLogFile=".\internal_logs\internallog.txt">
+        internalLogFile="./internal_logs/internallog.txt">
 
 	<variable name="logHome" value="${basedir}/Logs" />
 	<variable name="logLevel" value="Warn" />
@@ -11,15 +11,8 @@
 	<!--Possible aspnet- variables: https://nlog-project.org/config/?tab=layout-renderers&search=package:nlog.web-->
 	<variable name="layout" value="${longdate} | [${level:uppercase=true}] ${message} ${exception:format=tostring}" />
 
-	<extensions>
-		<!--Enable NLog.Web for ASP.NET Core.-->
-		<add assembly="NLog.Web.AspNetCore" />
-	</extensions>
-
 	<targets>
 		<!-- Write logs to File -->
-		<target xsi:type="FallbackGroup"
-			name="fileGroup">
 		<target
 			name="fullfile"
 			xsi:type="File"
@@ -31,7 +24,6 @@
 			archiveEvery="Day"
 			maxArchiveFiles="7"
 		/>
-		</target>
 
 		<!-- Write colored logs to Console -->
 		<target name="consoleLog" xsi:type="ColoredConsole" layout="[${uppercase:${level}}]: ${message}">