Skip to content
Snippets Groups Projects
Commit b0f1ef2c authored by Sandra Westerhoff's avatar Sandra Westerhoff
Browse files

Merge branch 'dev' into 'master'

merge dev into master

See merge request !22
parents d6f42bb1 422c3742
Branches
Tags v1.28.0
1 merge request!22merge dev into master
Showing
with 266 additions and 40 deletions
# Development container for dotnet
FROM mcr.microsoft.com/devcontainers/dotnet:8.0 as develop
USER vscode
\ No newline at end of file
version: "3.9"
services:
app:
build:
target: develop
user: vscode
userns_mode: keep-id:uid=1000,gid=1000
environment:
SSH_AUTH_SOCK: /.ssh/ssh-agent.sock
command: /bin/sh -c "while sleep 2s; do :; done"
volumes:
- ..:/workspace/graph-deployer:cached
- ./.vscode-server:/home/vscode/.vscode-server/:cached
- ${SSH_AUTH_SOCK}:/.ssh/ssh-agent.sock
networks:
- default
networks:
default:
driver: bridge
{
"name": "Graph Deployer",
"dockerComposeFile": [
"compose.yml"
],
"service": "app",
"workspaceFolder": "/workspace/graph-deployer",
"updateRemoteUserUID": false,
"remoteUser": "vscode",
"containerUser": "vscode",
"customizations": {
"vscode": {
"settings": {},
"extensions": [
"ms-dotnettools.csdevkit",
"ms-azuretools.vscode-docker",
"ms-dotnettools.csharp",
"mhutchie.git-graph",
"mutantdino.resourcemonitor"
]
}
},
"forwardPorts": []
}
\ No newline at end of file
/.devcontainer/.vscode-server/*
!/.devcontainer/.vscode-server/.gitkeep
/.devcontainer/.data/*
!/.devcontainer/.data/.gitkeep
.logs/
Output/*
internallog.txt
## Ignore the secrets settings
appsettings.Development.json
appsettings.Staging.json
appsettings.Production.json
## Ignore Visual Studio temporary files, build results, and ## Ignore Visual Studio temporary files, build results, and
## files generated by popular Visual Studio add-ons. ## files generated by popular Visual Studio add-ons.
......
...@@ -2,6 +2,7 @@ include: ...@@ -2,6 +2,7 @@ include:
- project: coscine/tools/gitlab-ci-templates - project: coscine/tools/gitlab-ci-templates
file: file:
- /dotnet.yml - /dotnet.yml
- /container.yml
stages: stages:
- build - build
...@@ -21,3 +22,8 @@ publish-gitlab-release: ...@@ -21,3 +22,8 @@ publish-gitlab-release:
publish: publish:
extends: .publish-artifact-release extends: .publish-artifact-release
publish-container:
variables:
CONTAINER_CONTEXT_FOLDER_PATH: $CI_PROJECT_DIR/src/$DOTNET_MAIN_PROJECT_FOLDER
extends: .publish-container-gitlab
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations":
[
{
"name": "C#: GraphDeployer Debug",
"type": "coreclr",
"request": "launch",
"preLaunchTask": "dotnet: build",
"program": "${workspaceFolder}/src/GraphDeployer/bin/Debug/net8.0/Coscine.GraphDeployer.dll",
"args": [],
"cwd": "${workspaceFolder}/src/GraphDeployer",
"stopAtEntry": false,
"console": "internalConsole",
"env": {
"DOTNET_ENVIRONMENT": "Development"
}
}
]
}
\ No newline at end of file
{
"resmon.disk.format": "UsedOutOfTotal",
"resmon.show.battery": false,
"resmon.show.cpufreq": false,
"resmon.show.disk": true,
"resmon.disk.drives": null
}
\ No newline at end of file
{
"version": "2.0.0",
"tasks": [
{
"type": "dotnet",
"task": "build",
"group": {
"kind": "build",
"isDefault": true
},
"problemMatcher": [],
"label": "dotnet: build"
}
]
}
\ No newline at end of file
...@@ -6,6 +6,25 @@ ...@@ -6,6 +6,25 @@
The Graph Deployer, intended to be used as a CRON job, is a .NET application designed to manage and deploy RDF graphs. It interacts with configured GitLab repositories to pull RDF graph definitions and updates a centralized repository if changes are detected or redeployment is triggered. The job uses a series of configurations to determine the operational parameters including the execution environment, repository access, and deployment specifications. The Graph Deployer, intended to be used as a CRON job, is a .NET application designed to manage and deploy RDF graphs. It interacts with configured GitLab repositories to pull RDF graph definitions and updates a centralized repository if changes are detected or redeployment is triggered. The job uses a series of configurations to determine the operational parameters including the execution environment, repository access, and deployment specifications.
## Getting Started
Clone the repository:
```bash
$ mkdir -p ~/code/git.rwth-aachen.de/coscine/backend/scripts && cd "$_"
$ git clone git@git.rwth-aachen.de:coscine/backend/scripts/graphdeployer.git
```
Open the new folder in code and in a [devcontainer](https://code.visualstudio.com/docs/devcontainers/containers#_open-a-folder-on-a-remote-ssh-host-in-a-container).
Insider the devcontainer, create a development settings file:
```bash
$ cp /workspace/graph-deployer/src/GraphDeployer/appsettings.Example.json /workspace/graph-deployer/src/GraphDeployer/appsettings.Development.json
```
In the `appsettings.Development.json` Replace any `null` value with a fitting value.
### Features ### Features
- **Configuration-driven**: Behavior driven by settings defined in configuration files and environment variables (see `appsettings.json`). - **Configuration-driven**: Behavior driven by settings defined in configuration files and environment variables (see `appsettings.json`).
- **Supports Dummy Mode**: Can run in a non-operative mode for testing configurations and process flow without affecting live data (CLI arg `--dummy`). - **Supports Dummy Mode**: Can run in a non-operative mode for testing configurations and process flow without affecting live data (CLI arg `--dummy`).
......
...@@ -13,19 +13,24 @@ using Configuration = Coscine.ApiClient.Core.Client.Configuration; ...@@ -13,19 +13,24 @@ using Configuration = Coscine.ApiClient.Core.Client.Configuration;
namespace Coscine.GraphDeployer; namespace Coscine.GraphDeployer;
public class Deployer(ILogger<Deployer> logger, IOptionsMonitor<GraphDeployerConfiguration> graphDeployerConfiguration) public class Deployer
{ {
private readonly ILogger<Deployer> _logger = logger; private readonly ILogger<Deployer> _logger;
private readonly GraphDeployerConfiguration _graphDeployerConfiguration = graphDeployerConfiguration.CurrentValue; private readonly GraphDeployerConfiguration _graphDeployerConfiguration;
private readonly AdminApi _adminApi = new(_configuration); private readonly AdminApi _adminApi;
private static readonly Configuration _configuration = new() public Deployer(ILogger<Deployer> logger, IOptionsMonitor<GraphDeployerConfiguration> graphDeployerConfiguration)
{ {
BasePath = "http://localhost:7206/coscine", _logger = logger;
_graphDeployerConfiguration = graphDeployerConfiguration.CurrentValue;
_adminApi = new(new Configuration
{
BasePath = $"{_graphDeployerConfiguration.Endpoint.TrimEnd('/')}/coscine",
ApiKeyPrefix = { { "Authorization", "Bearer" } }, ApiKeyPrefix = { { "Authorization", "Bearer" } },
ApiKey = { { "Authorization", ApiConfigurationUtil.GenerateAdminToken(ApiConfigurationUtil.RetrieveJwtConfiguration()) } }, ApiKey = { { "Authorization", _graphDeployerConfiguration.ApiKey } },
Timeout = 300000 // 5 minutes Timeout = 300000 // 5 minutes
}; });
}
public static string WorkingFolder { get; set; } = "./output/"; public static string WorkingFolder { get; set; } = "./output/";
public static List<string> DeployedGraphs { get; set; } = []; public static List<string> DeployedGraphs { get; set; } = [];
......
FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build-env
WORKDIR /App
# Copy everything
COPY . ./
# Restore as distinct layers
RUN dotnet restore
# Build and publish a release
RUN dotnet publish -c Release -o out
# Build runtime image
FROM mcr.microsoft.com/dotnet/aspnet:8.0
# Install cron and other necessary packages
RUN apt-get update && \
apt-get install -y cron && \
apt-get clean
WORKDIR /App
COPY --from=build-env /App/out .
# Copy the cron job file to /etc/cron.d/
COPY graph-deployer-cron /etc/cron.d/
# Give execution rights on the cron job file
RUN chmod 0644 /etc/cron.d/graph-deployer-cron
# Apply the cron job
RUN crontab /etc/cron.d/graph-deployer-cron
# Set the build-time argument and default environment variable
ENV DOTNET_ENVIRONMENT=Development
# Create the log file
RUN touch /var/log/cron.log
ENTRYPOINT /bin/sh -c "cron;/App/Coscine.GraphDeployer;tail -f /var/log/cron.log"
...@@ -22,7 +22,7 @@ ...@@ -22,7 +22,7 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="CommandLineParser" Version="2.9.1" /> <PackageReference Include="CommandLineParser" Version="2.9.1" />
<PackageReference Include="Coscine.ApiClient" Version="1.7.0-issue-2668-graph0001" /> <PackageReference Include="Coscine.ApiClient" Version="1.8.0" />
<PackageReference Include="dotNetRdf" Version="3.1.1" /> <PackageReference Include="dotNetRdf" Version="3.1.1" />
<PackageReference Include="GitLabApiClient" Version="1.8.1-beta.5" /> <PackageReference Include="GitLabApiClient" Version="1.8.1-beta.5" />
<PackageReference Include="LibGit2Sharp" Version="0.30.0" /> <PackageReference Include="LibGit2Sharp" Version="0.30.0" />
...@@ -40,6 +40,9 @@ ...@@ -40,6 +40,9 @@
<None Update="appsettings.json"> <None Update="appsettings.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None> </None>
<None Update="appsettings.development.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="nlog.config"> <None Update="nlog.config">
<CopyToOutputDirectory>Always</CopyToOutputDirectory> <CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None> </None>
......
...@@ -17,6 +17,17 @@ public class GraphDeployerConfiguration ...@@ -17,6 +17,17 @@ public class GraphDeployerConfiguration
/// </summary> /// </summary>
public string? WorkingFolder { get; init; } public string? WorkingFolder { get; init; }
/// <summary>
/// API token for the coscine API.
/// Requires the administrator role.
/// </summary>
public string ApiKey { get; set; } = null!;
/// <summary>
/// Address of the Coscine API.
/// </summary>
public string Endpoint { get; set; } = null!;
/// <summary> /// <summary>
/// Logger configuration settings. /// Logger configuration settings.
/// </summary> /// </summary>
......
...@@ -15,8 +15,14 @@ public class Program ...@@ -15,8 +15,14 @@ public class Program
{ {
private static IServiceProvider _serviceProvider = null!; private static IServiceProvider _serviceProvider = null!;
private static string? _environmentName;
public static int Main(string[] args) public static int Main(string[] args)
{ {
_environmentName = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT");
Console.WriteLine($"Running in environment: {_environmentName ?? "environment is null"}");
InitializeServices(); InitializeServices();
var logger = _serviceProvider.GetRequiredService<ILogger<Program>>(); var logger = _serviceProvider.GetRequiredService<ILogger<Program>>();
...@@ -65,7 +71,8 @@ public class Program ...@@ -65,7 +71,8 @@ public class Program
// Add Consul as a configuration source // Add Consul as a configuration source
var configuration = configBuilder var configuration = configBuilder
.AddJsonFile("appsettings.json", optional: true, reloadOnChange: true) .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
.AddJsonFile($"appsettings.{_environmentName}.json", optional: true, reloadOnChange: true)
.AddConsul( .AddConsul(
"coscine/Coscine.Infrastructure/GraphDeployer/appsettings", "coscine/Coscine.Infrastructure/GraphDeployer/appsettings",
options => options =>
...@@ -78,6 +85,18 @@ public class Program ...@@ -78,6 +85,18 @@ public class Program
options.OnLoadException = exceptionContext => exceptionContext.Ignore = true; options.OnLoadException = exceptionContext => exceptionContext.Ignore = true;
} }
) )
.AddConsul(
$"coscine/Coscine.Infrastructure/GraphDeployer/appsettings.{_environmentName}.json",
options =>
{
options.ConsulConfigurationOptions =
cco => cco.Address = new Uri(consulUrl);
options.Optional = true;
options.ReloadOnChange = true;
options.PollWaitTime = TimeSpan.FromSeconds(5);
options.OnLoadException = exceptionContext => exceptionContext.Ignore = true;
}
)
.AddEnvironmentVariables() .AddEnvironmentVariables()
.Build(); .Build();
...@@ -100,6 +119,17 @@ public class Program ...@@ -100,6 +119,17 @@ public class Program
var graphDeployerConfiguration = new GraphDeployerConfiguration(); var graphDeployerConfiguration = new GraphDeployerConfiguration();
configuration.Bind(GraphDeployerConfiguration.Section, graphDeployerConfiguration); configuration.Bind(GraphDeployerConfiguration.Section, graphDeployerConfiguration);
if (graphDeployerConfiguration.Endpoint is null)
{
throw new ArgumentNullException(nameof(graphDeployerConfiguration.Endpoint), "Endpoint cannot be null.");
}
if (graphDeployerConfiguration.ApiKey is null)
{
throw new ArgumentNullException(nameof(graphDeployerConfiguration.ApiKey), "ApiKey cannot be null.");
}
// Set the default LogLevel // Set the default LogLevel
LogManager.Configuration.Variables["logLevel"] = graphDeployerConfiguration.Logger?.LogLevel ?? "Trace"; LogManager.Configuration.Variables["logLevel"] = graphDeployerConfiguration.Logger?.LogLevel ?? "Trace";
// Set the log location // Set the log location
......
{
"GraphDeployerConfiguration": {
"ApiKey": null,
"Endpoint": null,
"GitLab": {
"Token": null
}
}
}
\ No newline at end of file
{ {
"GraphDeployerConfiguration": { "GraphDeployerConfiguration": {
"ApiKey": null,
"Endpoint": null,
"IsEnabled": true, "IsEnabled": true,
"WorkingFolder": "C:/coscine/GraphDeployer/output/", "WorkingFolder": "./Output/",
"Logger": { "Logger": {
"LogLevel": "Information", "LogLevel": "Information",
"LogHome": "C:/coscine/GraphDeployer/" "LogHome": "./Logs"
}, },
"GitLab": { "GitLab": {
"HostUrl": "https://git.rwth-aachen.de/", "HostUrl": "https://git.rwth-aachen.de/",
"Token": "glpat-iXFFtZSjmbhn8U4-YFzv", "Token": null,
"Repositories": [ "Repositories": [
{ {
"Name": "Application Profiles", "Name": "Application Profiles",
"Url": "https://git.rwth-aachen.de/coscine/graphs/applicationprofiles.git", "Url": "https://git.rwth-aachen.de/coscine/graphs/applicationprofiles.git"
"Branch": "master"
}, },
{ {
"Name": "Organisations", "Name": "Organisations",
"Url": "https://git.rwth-aachen.de/coscine/graphs/organizations.git", "Url": "https://git.rwth-aachen.de/coscine/graphs/organizations.git"
"Branch": "master"
}, },
{ {
"Name": "FH Bielefeld", "Name": "FH Bielefeld",
......
* 3 * * * root /App/Coscine.GraphDeployer >> /var/log/cron.log 2>&1
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
autoReload="true" autoReload="true"
internalLogLevel="Warn" internalLogLevel="Warn"
internalLogFile=".\internal_logs\internallog.txt"> internalLogFile="./internal_logs/internallog.txt">
<variable name="logHome" value="${basedir}/Logs" /> <variable name="logHome" value="${basedir}/Logs" />
<variable name="logLevel" value="Warn" /> <variable name="logLevel" value="Warn" />
...@@ -11,15 +11,8 @@ ...@@ -11,15 +11,8 @@
<!--Possible aspnet- variables: https://nlog-project.org/config/?tab=layout-renderers&search=package:nlog.web--> <!--Possible aspnet- variables: https://nlog-project.org/config/?tab=layout-renderers&search=package:nlog.web-->
<variable name="layout" value="${longdate} | [${level:uppercase=true}] ${message} ${exception:format=tostring}" /> <variable name="layout" value="${longdate} | [${level:uppercase=true}] ${message} ${exception:format=tostring}" />
<extensions>
<!--Enable NLog.Web for ASP.NET Core.-->
<add assembly="NLog.Web.AspNetCore" />
</extensions>
<targets> <targets>
<!-- Write logs to File --> <!-- Write logs to File -->
<target xsi:type="FallbackGroup"
name="fileGroup">
<target <target
name="fullfile" name="fullfile"
xsi:type="File" xsi:type="File"
...@@ -31,7 +24,6 @@ ...@@ -31,7 +24,6 @@
archiveEvery="Day" archiveEvery="Day"
maxArchiveFiles="7" maxArchiveFiles="7"
/> />
</target>
<!-- Write colored logs to Console --> <!-- Write colored logs to Console -->
<target name="consoleLog" xsi:type="ColoredConsole" layout="[${uppercase:${level}}]: ${message}"> <target name="consoleLog" xsi:type="ColoredConsole" layout="[${uppercase:${level}}]: ${message}">
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment