diff --git a/.devcontainer/.vscode-server/.gitkeep b/.devcontainer/.vscode-server/.gitkeep
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..f22718318525e0a372ff92264a4f731beb9d658b
--- /dev/null
+++ b/.devcontainer/Dockerfile
@@ -0,0 +1,5 @@
+# Development container for dotnet
+
+FROM mcr.microsoft.com/devcontainers/dotnet:8.0 as develop
+
+USER vscode
\ No newline at end of file
diff --git a/.devcontainer/compose.yml b/.devcontainer/compose.yml
new file mode 100644
index 0000000000000000000000000000000000000000..50384ae06f8e2a50e7101eddd1c4adf35cd2a88a
--- /dev/null
+++ b/.devcontainer/compose.yml
@@ -0,0 +1,22 @@
+version: "3.9"
+
+services:
+
+ app:
+ build:
+ target: develop
+ user: vscode
+ userns_mode: keep-id:uid=1000,gid=1000
+ environment:
+ SSH_AUTH_SOCK: /.ssh/ssh-agent.sock
+ command: /bin/sh -c "while sleep 2s; do :; done"
+ volumes:
+ - ..:/workspace/graph-deployer:cached
+ - ./.vscode-server:/home/vscode/.vscode-server/:cached
+ - ${SSH_AUTH_SOCK}:/.ssh/ssh-agent.sock
+ networks:
+ - default
+
+networks:
+ default:
+ driver: bridge
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
new file mode 100644
index 0000000000000000000000000000000000000000..46ae9f2cddc85ca04fb315c81c6636e72897dd6b
--- /dev/null
+++ b/.devcontainer/devcontainer.json
@@ -0,0 +1,24 @@
+{
+ "name": "Graph Deployer",
+ "dockerComposeFile": [
+ "compose.yml"
+ ],
+ "service": "app",
+ "workspaceFolder": "/workspace/graph-deployer",
+ "updateRemoteUserUID": false,
+ "remoteUser": "vscode",
+ "containerUser": "vscode",
+ "customizations": {
+ "vscode": {
+ "settings": {},
+ "extensions": [
+ "ms-dotnettools.csdevkit",
+ "ms-azuretools.vscode-docker",
+ "ms-dotnettools.csharp",
+ "mhutchie.git-graph",
+ "mutantdino.resourcemonitor"
+ ]
+ }
+ },
+ "forwardPorts": []
+}
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 7cdf465cdfbcae98eaca0ef127878a9b08936ec9..0103a271330530a16c89b491897294e934f7d602 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,18 @@
+/.devcontainer/.vscode-server/*
+!/.devcontainer/.vscode-server/.gitkeep
+
+/.devcontainer/.data/*
+!/.devcontainer/.data/.gitkeep
+
+.logs/
+Output/*
+internallog.txt
+
+## Ignore the secrets settings
+appsettings.Development.json
+appsettings.Staging.json
+appsettings.Production.json
+
## Ignore Visual Studio temporary files, build results, and
## files generated by popular Visual Studio add-ons.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f42ae1c27ea20c0063524c1eb5f96693162dbb2c..08533ce34d20930fb3d3efd13200da911718b35c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -2,6 +2,7 @@ include:
- project: coscine/tools/gitlab-ci-templates
file:
- /dotnet.yml
+ - /container.yml
stages:
- build
@@ -21,3 +22,8 @@ publish-gitlab-release:
publish:
extends: .publish-artifact-release
+
+publish-container:
+ variables:
+ CONTAINER_CONTEXT_FOLDER_PATH: $CI_PROJECT_DIR/src/$DOTNET_MAIN_PROJECT_FOLDER
+ extends: .publish-container-gitlab
diff --git a/.vscode/launch.json b/.vscode/launch.json
new file mode 100644
index 0000000000000000000000000000000000000000..8d09799fe7a6c17af19a06b2ba3e4a249fe1c9b5
--- /dev/null
+++ b/.vscode/launch.json
@@ -0,0 +1,23 @@
+{
+ // Use IntelliSense to learn about possible attributes.
+ // Hover to view descriptions of existing attributes.
+ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+ "version": "0.2.0",
+ "configurations":
+ [
+ {
+ "name": "C#: GraphDeployer Debug",
+ "type": "coreclr",
+ "request": "launch",
+ "preLaunchTask": "dotnet: build",
+ "program": "${workspaceFolder}/src/GraphDeployer/bin/Debug/net8.0/Coscine.GraphDeployer.dll",
+ "args": [],
+ "cwd": "${workspaceFolder}/src/GraphDeployer",
+ "stopAtEntry": false,
+ "console": "internalConsole",
+ "env": {
+ "DOTNET_ENVIRONMENT": "Development"
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/.vscode/settings.json b/.vscode/settings.json
new file mode 100644
index 0000000000000000000000000000000000000000..bcf184968422b23b83ba23724646274adaa2faf2
--- /dev/null
+++ b/.vscode/settings.json
@@ -0,0 +1,7 @@
+{
+ "resmon.disk.format": "UsedOutOfTotal",
+ "resmon.show.battery": false,
+ "resmon.show.cpufreq": false,
+ "resmon.show.disk": true,
+ "resmon.disk.drives": null
+}
\ No newline at end of file
diff --git a/.vscode/tasks.json b/.vscode/tasks.json
new file mode 100644
index 0000000000000000000000000000000000000000..85beec3e7a8f83568d729a0d76f4b6ba0a05dd93
--- /dev/null
+++ b/.vscode/tasks.json
@@ -0,0 +1,15 @@
+{
+ "version": "2.0.0",
+ "tasks": [
+ {
+ "type": "dotnet",
+ "task": "build",
+ "group": {
+ "kind": "build",
+ "isDefault": true
+ },
+ "problemMatcher": [],
+ "label": "dotnet: build"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/README.md b/README.md
index 149ff1458143061b8cf18a8e67d8b3c663e172ec..fcea2a4b6f7fcba29e5045f234fb3055d87e5f06 100644
--- a/README.md
+++ b/README.md
@@ -6,6 +6,25 @@
The Graph Deployer, intended to be used as a CRON job, is a .NET application designed to manage and deploy RDF graphs. It interacts with configured GitLab repositories to pull RDF graph definitions and updates a centralized repository if changes are detected or redeployment is triggered. The job uses a series of configurations to determine the operational parameters including the execution environment, repository access, and deployment specifications.
+## Getting Started
+
+Clone the repository:
+
+```bash
+$ mkdir -p ~/code/git.rwth-aachen.de/coscine/backend/scripts && cd "$_"
+$ git clone git@git.rwth-aachen.de:coscine/backend/scripts/graphdeployer.git
+```
+
+Open the new folder in code and in a [devcontainer](https://code.visualstudio.com/docs/devcontainers/containers#_open-a-folder-on-a-remote-ssh-host-in-a-container).
+
+Insider the devcontainer, create a development settings file:
+
+```bash
+$ cp /workspace/graph-deployer/src/GraphDeployer/appsettings.Example.json /workspace/graph-deployer/src/GraphDeployer/appsettings.Development.json
+```
+
+In the `appsettings.Development.json` Replace any `null` value with a fitting value.
+
### Features
- **Configuration-driven**: Behavior driven by settings defined in configuration files and environment variables (see `appsettings.json`).
- **Supports Dummy Mode**: Can run in a non-operative mode for testing configurations and process flow without affecting live data (CLI arg `--dummy`).
diff --git a/src/GraphDeployer/Deployer.cs b/src/GraphDeployer/Deployer.cs
index 3895d42ac6a7dbc1527a3180509b98309e41743e..7349f513ae53259201212b7b5b096f2d900c5352 100644
--- a/src/GraphDeployer/Deployer.cs
+++ b/src/GraphDeployer/Deployer.cs
@@ -13,19 +13,24 @@ using Configuration = Coscine.ApiClient.Core.Client.Configuration;
namespace Coscine.GraphDeployer;
-public class Deployer(ILogger<Deployer> logger, IOptionsMonitor<GraphDeployerConfiguration> graphDeployerConfiguration)
+public class Deployer
{
- private readonly ILogger<Deployer> _logger = logger;
- private readonly GraphDeployerConfiguration _graphDeployerConfiguration = graphDeployerConfiguration.CurrentValue;
- private readonly AdminApi _adminApi = new(_configuration);
+ private readonly ILogger<Deployer> _logger;
+ private readonly GraphDeployerConfiguration _graphDeployerConfiguration;
+ private readonly AdminApi _adminApi;
- private static readonly Configuration _configuration = new()
+ public Deployer(ILogger<Deployer> logger, IOptionsMonitor<GraphDeployerConfiguration> graphDeployerConfiguration)
{
- BasePath = "http://localhost:7206/coscine",
- ApiKeyPrefix = { { "Authorization", "Bearer" } },
- ApiKey = { { "Authorization", ApiConfigurationUtil.GenerateAdminToken(ApiConfigurationUtil.RetrieveJwtConfiguration()) } },
- Timeout = 300000 // 5 minutes
- };
+ _logger = logger;
+ _graphDeployerConfiguration = graphDeployerConfiguration.CurrentValue;
+ _adminApi = new(new Configuration
+ {
+ BasePath = $"{_graphDeployerConfiguration.Endpoint.TrimEnd('/')}/coscine",
+ ApiKeyPrefix = { { "Authorization", "Bearer" } },
+ ApiKey = { { "Authorization", _graphDeployerConfiguration.ApiKey } },
+ Timeout = 300000 // 5 minutes
+ });
+ }
public static string WorkingFolder { get; set; } = "./output/";
public static List<string> DeployedGraphs { get; set; } = [];
diff --git a/src/GraphDeployer/Dockerfile b/src/GraphDeployer/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..d8e70b7de68547fefb86793c09e9a5d24165f612
--- /dev/null
+++ b/src/GraphDeployer/Dockerfile
@@ -0,0 +1,39 @@
+FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build-env
+WORKDIR /App
+
+# Copy everything
+COPY . ./
+
+# Restore as distinct layers
+RUN dotnet restore
+
+# Build and publish a release
+RUN dotnet publish -c Release -o out
+
+# Build runtime image
+FROM mcr.microsoft.com/dotnet/aspnet:8.0
+
+# Install cron and other necessary packages
+RUN apt-get update && \
+ apt-get install -y cron && \
+ apt-get clean
+
+WORKDIR /App
+COPY --from=build-env /App/out .
+
+# Copy the cron job file to /etc/cron.d/
+COPY graph-deployer-cron /etc/cron.d/
+
+# Give execution rights on the cron job file
+RUN chmod 0644 /etc/cron.d/graph-deployer-cron
+
+# Apply the cron job
+RUN crontab /etc/cron.d/graph-deployer-cron
+
+# Set the build-time argument and default environment variable
+ENV DOTNET_ENVIRONMENT=Development
+
+# Create the log file
+RUN touch /var/log/cron.log
+
+ENTRYPOINT /bin/sh -c "cron;/App/Coscine.GraphDeployer;tail -f /var/log/cron.log"
diff --git a/src/GraphDeployer/GraphDeployer.csproj b/src/GraphDeployer/GraphDeployer.csproj
index 16946bd18ff886d53435c912237150a757ddd4de..656adca43357136cd798e32ed7fdad103d62f340 100644
--- a/src/GraphDeployer/GraphDeployer.csproj
+++ b/src/GraphDeployer/GraphDeployer.csproj
@@ -22,7 +22,7 @@
<ItemGroup>
<PackageReference Include="CommandLineParser" Version="2.9.1" />
- <PackageReference Include="Coscine.ApiClient" Version="1.7.0-issue-2668-graph0001" />
+ <PackageReference Include="Coscine.ApiClient" Version="1.8.0" />
<PackageReference Include="dotNetRdf" Version="3.1.1" />
<PackageReference Include="GitLabApiClient" Version="1.8.1-beta.5" />
<PackageReference Include="LibGit2Sharp" Version="0.30.0" />
@@ -40,6 +40,9 @@
<None Update="appsettings.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
+ <None Update="appsettings.development.json">
+ <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+ </None>
<None Update="nlog.config">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
diff --git a/src/GraphDeployer/Models/ConfigurationModels/GraphDeployerConfiguration.cs b/src/GraphDeployer/Models/ConfigurationModels/GraphDeployerConfiguration.cs
index cf2e346374b4af78958f96d70fbdbd7ecf711920..7810fd211c2225fb68ffa6cb3dbcca7a95af4c54 100644
--- a/src/GraphDeployer/Models/ConfigurationModels/GraphDeployerConfiguration.cs
+++ b/src/GraphDeployer/Models/ConfigurationModels/GraphDeployerConfiguration.cs
@@ -17,6 +17,17 @@ public class GraphDeployerConfiguration
/// </summary>
public string? WorkingFolder { get; init; }
+ /// <summary>
+ /// API token for the coscine API.
+ /// Requires the administrator role.
+ /// </summary>
+ public string ApiKey { get; set; } = null!;
+
+ /// <summary>
+ /// Address of the Coscine API.
+ /// </summary>
+ public string Endpoint { get; set; } = null!;
+
/// <summary>
/// Logger configuration settings.
/// </summary>
diff --git a/src/GraphDeployer/Program.cs b/src/GraphDeployer/Program.cs
index c60e8b39113115cb922adf3ec808a443c45cdbf0..3b45eeab4a38c1de93bfdf5db7815c204ddf7924 100644
--- a/src/GraphDeployer/Program.cs
+++ b/src/GraphDeployer/Program.cs
@@ -15,8 +15,14 @@ public class Program
{
private static IServiceProvider _serviceProvider = null!;
+ private static string? _environmentName;
+
public static int Main(string[] args)
{
+ _environmentName = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT");
+
+ Console.WriteLine($"Running in environment: {_environmentName ?? "environment is null"}");
+
InitializeServices();
var logger = _serviceProvider.GetRequiredService<ILogger<Program>>();
@@ -65,7 +71,8 @@ public class Program
// Add Consul as a configuration source
var configuration = configBuilder
- .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true)
+ .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
+ .AddJsonFile($"appsettings.{_environmentName}.json", optional: true, reloadOnChange: true)
.AddConsul(
"coscine/Coscine.Infrastructure/GraphDeployer/appsettings",
options =>
@@ -78,6 +85,18 @@ public class Program
options.OnLoadException = exceptionContext => exceptionContext.Ignore = true;
}
)
+ .AddConsul(
+ $"coscine/Coscine.Infrastructure/GraphDeployer/appsettings.{_environmentName}.json",
+ options =>
+ {
+ options.ConsulConfigurationOptions =
+ cco => cco.Address = new Uri(consulUrl);
+ options.Optional = true;
+ options.ReloadOnChange = true;
+ options.PollWaitTime = TimeSpan.FromSeconds(5);
+ options.OnLoadException = exceptionContext => exceptionContext.Ignore = true;
+ }
+ )
.AddEnvironmentVariables()
.Build();
@@ -100,6 +119,17 @@ public class Program
var graphDeployerConfiguration = new GraphDeployerConfiguration();
configuration.Bind(GraphDeployerConfiguration.Section, graphDeployerConfiguration);
+
+ if (graphDeployerConfiguration.Endpoint is null)
+ {
+ throw new ArgumentNullException(nameof(graphDeployerConfiguration.Endpoint), "Endpoint cannot be null.");
+ }
+
+ if (graphDeployerConfiguration.ApiKey is null)
+ {
+ throw new ArgumentNullException(nameof(graphDeployerConfiguration.ApiKey), "ApiKey cannot be null.");
+ }
+
// Set the default LogLevel
LogManager.Configuration.Variables["logLevel"] = graphDeployerConfiguration.Logger?.LogLevel ?? "Trace";
// Set the log location
diff --git a/src/GraphDeployer/appsettings.Example.json b/src/GraphDeployer/appsettings.Example.json
new file mode 100644
index 0000000000000000000000000000000000000000..302c1b3f4bc76d276422a03ab569a77b5ff7c38c
--- /dev/null
+++ b/src/GraphDeployer/appsettings.Example.json
@@ -0,0 +1,9 @@
+{
+ "GraphDeployerConfiguration": {
+ "ApiKey": null,
+ "Endpoint": null,
+ "GitLab": {
+ "Token": null
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/GraphDeployer/appsettings.json b/src/GraphDeployer/appsettings.json
index 836defeaa1554902878f9cc20efac9728a18bff6..6fa53903c346bdc5e3ebef4439dba0c1f05054fc 100644
--- a/src/GraphDeployer/appsettings.json
+++ b/src/GraphDeployer/appsettings.json
@@ -1,24 +1,24 @@
{
"GraphDeployerConfiguration": {
+ "ApiKey": null,
+ "Endpoint": null,
"IsEnabled": true,
- "WorkingFolder": "C:/coscine/GraphDeployer/output/",
+ "WorkingFolder": "./Output/",
"Logger": {
"LogLevel": "Information",
- "LogHome": "C:/coscine/GraphDeployer/"
+ "LogHome": "./Logs"
},
"GitLab": {
"HostUrl": "https://git.rwth-aachen.de/",
- "Token": "glpat-iXFFtZSjmbhn8U4-YFzv",
+ "Token": null,
"Repositories": [
{
"Name": "Application Profiles",
- "Url": "https://git.rwth-aachen.de/coscine/graphs/applicationprofiles.git",
- "Branch": "master"
+ "Url": "https://git.rwth-aachen.de/coscine/graphs/applicationprofiles.git"
},
{
"Name": "Organisations",
- "Url": "https://git.rwth-aachen.de/coscine/graphs/organizations.git",
- "Branch": "master"
+ "Url": "https://git.rwth-aachen.de/coscine/graphs/organizations.git"
},
{
"Name": "FH Bielefeld",
diff --git a/src/GraphDeployer/graph-deployer-cron b/src/GraphDeployer/graph-deployer-cron
new file mode 100644
index 0000000000000000000000000000000000000000..9b333af7115505ba388f9946e0e2a4bb17b02154
--- /dev/null
+++ b/src/GraphDeployer/graph-deployer-cron
@@ -0,0 +1 @@
+* 3 * * * root /App/Coscine.GraphDeployer >> /var/log/cron.log 2>&1
diff --git a/src/GraphDeployer/nlog.config b/src/GraphDeployer/nlog.config
index c2fd7e323b320dadeb7946287dedab42a29a9c82..e57a1713090266ca649ee185a60362ec07956514 100644
--- a/src/GraphDeployer/nlog.config
+++ b/src/GraphDeployer/nlog.config
@@ -3,35 +3,27 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
autoReload="true"
internalLogLevel="Warn"
- internalLogFile=".\internal_logs\internallog.txt">
+ internalLogFile="./internal_logs/internallog.txt">
<variable name="logHome" value="${basedir}/Logs" />
<variable name="logLevel" value="Warn" />
<!--Possible aspnet- variables: https://nlog-project.org/config/?tab=layout-renderers&search=package:nlog.web-->
- <variable name="layout" value="${longdate} | [${level:uppercase=true}] ${message} ${exception:format=tostring}" />
-
- <extensions>
- <!--Enable NLog.Web for ASP.NET Core.-->
- <add assembly="NLog.Web.AspNetCore" />
- </extensions>
+ <variable name="layout" value="${longdate} | [${level:uppercase=true}] ${message} ${exception:format=tostring}" />
<targets>
<!-- Write logs to File -->
- <target xsi:type="FallbackGroup"
- name="fileGroup">
- <target
- name="fullfile"
- xsi:type="File"
- layout="${var:layout}"
- fileName="${var:logHome}/full_${shortdate}.log"
- keepFileOpen="false"
- archiveFileName="${var:logHome}/Archive/full_${shortdate}.log"
- archiveNumbering="Sequence"
- archiveEvery="Day"
- maxArchiveFiles="7"
- />
- </target>
+ <target
+ name="fullfile"
+ xsi:type="File"
+ layout="${var:layout}"
+ fileName="${var:logHome}/full_${shortdate}.log"
+ keepFileOpen="false"
+ archiveFileName="${var:logHome}/Archive/full_${shortdate}.log"
+ archiveNumbering="Sequence"
+ archiveEvery="Day"
+ maxArchiveFiles="7"
+ />
<!-- Write colored logs to Console -->
<target name="consoleLog" xsi:type="ColoredConsole" layout="[${uppercase:${level}}]: ${message}">