diff --git a/.devcontainer/.vscode-server/.gitkeep b/.devcontainer/.vscode-server/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..10f31c397cd124eb98789e38124e931817a38f7f --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,27 @@ +# Development container for dotnet +FROM mcr.microsoft.com/devcontainers/dotnet:8.0 as develop + +ARG USERNAME=vscode +ARG USER_UID=1000 +ARG USER_GID=$USER_UID + +# Remove any existing user with the same UID (if it's not the specified user) +RUN if id -u $USER_UID >/dev/null 2>&1; then \ + existing_user=$(getent passwd $USER_UID | cut -d: -f1); \ + if [ "$existing_user" != "$USERNAME" ]; then \ + userdel -f $existing_user; \ + fi; \ + fi + +# Change the user's GID and UID +RUN groupmod -g $USER_GID $USERNAME \ + && usermod -u $USER_UID -g $USER_GID $USERNAME + +# Change user folder owner and group +RUN chown -R $USER_GID:$USER_GID /home/$USERNAME + +# [Optional] Set the default user. Omit if you want to keep the default as root. +USER $USERNAME + +# Add nuget sources for private packages (here: api-client) +RUN dotnet nuget add source "https://git.rwth-aachen.de/api/v4/projects/88930/packages/nuget/index.json" -n "api-client" \ No newline at end of file diff --git a/.devcontainer/compose.yml b/.devcontainer/compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..0dd80c1b7d2a0074eb8d8f423320ba248ad4a3a9 --- /dev/null +++ b/.devcontainer/compose.yml @@ -0,0 +1,22 @@ +version: "3.9" + +services: + + app: + build: + target: develop + user: vscode + userns_mode: keep-id:uid=1000,gid=1000 + environment: + SSH_AUTH_SOCK: /.ssh/ssh-agent.sock + command: /bin/sh -c "while sleep 2s; do :; done" + volumes: + - ..:/workspace/kpi-generator:cached + - ./.vscode-server:/home/vscode/.vscode-server/:cached + - ${SSH_AUTH_SOCK}:/.ssh/ssh-agent.sock + networks: + - default + +networks: + default: + driver: bridge diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000000000000000000000000000000000000..4373339fb31853490b801735458b004255860a9a --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,32 @@ +{ + "name": "KPI Reporting Generator", + "dockerComposeFile": [ + "compose.yml" + ], + "service": "app", + "workspaceFolder": "/workspace/kpi-generator", + "updateRemoteUserUID": false, + "remoteUser": "vscode", + "containerUser": "vscode", + "postAttachCommand": "bash", + "customizations": { + "vscode": { + "settings": { + "NugetGallery.sources": [ + "{\"name\":\"nuget.org\",\"url\":\"https://api.nuget.org/v3/index.json\"}", + "{\"name\":\"api-client\",\"url\":\"https://git.rwth-aachen.de/api/v4/projects/88930/packages/nuget/index.json\"}" + ], + "nuget.includePrereleasePackageVersions": true + }, + "extensions": [ + "ms-dotnettools.csdevkit", + "ms-azuretools.vscode-docker", + "ms-dotnettools.csharp", + "mhutchie.git-graph", + "mutantdino.resourcemonitor", + "patcx.vscode-nuget-gallery" + ] + } + }, + "forwardPorts": [] +} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 7cdf465cdfbcae98eaca0ef127878a9b08936ec9..0103a271330530a16c89b491897294e934f7d602 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,18 @@ +/.devcontainer/.vscode-server/* +!/.devcontainer/.vscode-server/.gitkeep + +/.devcontainer/.data/* +!/.devcontainer/.data/.gitkeep + +.logs/ +Output/* +internallog.txt + +## Ignore the secrets settings +appsettings.Development.json +appsettings.Staging.json +appsettings.Production.json + ## Ignore Visual Studio temporary files, build results, and ## files generated by popular Visual Studio add-ons. diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eac91234990eb2a63539ee9889615772305e2904..e819ecc763496f81ce8511e028ca3c47078704ca 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,14 +2,14 @@ include: - project: coscine/tools/gitlab-ci-templates file: - /dotnet.yml + - /container.yml stages: - build - publish - variables: - DOTNET_MAIN_PROJECT_FOLDER: "KPI Generator" + DOTNET_MAIN_PROJECT_FOLDER: "KpiGenerator" build-branch: extends: .build-branch @@ -22,3 +22,8 @@ publish-gitlab-release: publish: extends: .publish-artifact-release + +publish-container: + variables: + CONTAINER_CONTEXT_FOLDER_PATH: $CI_PROJECT_DIR/src/$DOTNET_MAIN_PROJECT_FOLDER + extends: .publish-container-gitlab diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000000000000000000000000000000000000..df8dcbce0ff274ce0fab959c545ad396dd84e847 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,22 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "C#: KPI Reporting Generator Debug", + "type": "coreclr", + "request": "launch", + "preLaunchTask": "dotnet: build", + "program": "${workspaceFolder}/src/KpiGenerator/bin/Debug/net8.0/Coscine.KpiGenerator.dll", + "args": [], + "cwd": "${workspaceFolder}/src/KpiGenerator", + "stopAtEntry": false, + "console": "internalConsole", + "env": { + "DOTNET_ENVIRONMENT": "Development" + } + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000000000000000000000000000000000000..9e26dfeeb6e641a33dae4961196235bdb965b21b --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/src/KPI Generator/Models/PublicationRequestReport.cs b/src/KPI Generator/Models/PublicationRequestReport.cs deleted file mode 100644 index e166ee48a31fc4ae5f3671d9028730ff6e2297ee..0000000000000000000000000000000000000000 --- a/src/KPI Generator/Models/PublicationRequestReport.cs +++ /dev/null @@ -1,9 +0,0 @@ -using System.Text.Json.Serialization; - -namespace Coscine.KpiGenerator.Models; - -public record PublicationRequestReport -{ - [JsonPropertyName("Timestamp")] - public DateTime Timestamp { get; set; } -} \ No newline at end of file diff --git a/src/KPI Generator/nlog.config b/src/KPI Generator/nlog.config deleted file mode 100644 index 0a330532e1b935e2a337d62fb973da7d7e50a04f..0000000000000000000000000000000000000000 --- a/src/KPI Generator/nlog.config +++ /dev/null @@ -1,81 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<nlog xmlns="http://www.nlog-project.org/schemas/NLog.xsd" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - throwConfigExceptions="true" - throwExceptions="false" - internalLogFile="Logs/internal-nlog.txt" - internalLogLevel="Trace"> - - <targets> - <!-- Write logs to File --> - <target xsi:type="FallbackGroup" - name="fileGroup"> - <target - xsi:type="File" - name="fileLogD" - fileName="D:/coscine/logs/${assembly-name}/${assembly-version}/log-${shortdate}.log" - maxArchiveFiles="7" - > - <layout xsi:type="CompoundLayout"> - <layout xsi:type="JsonLayout" EscapeForwardSlash="true"> - <attribute layout="${longdate}" name="Timestamp"/> - <attribute layout="${level:upperCase=true}" name="Level"/> - <attribute layout="${message}" name="Message"/> - <attribute layout="${exception:format=tostring,StackTrace}" name="Exception"/> - <attribute layout="${ndlc}" name="Context"/> - <attribute layout="${event-properties:item=Metric}" name="Alarm" encode="false"/> - <attribute name="EventProperties" encode="false" > - <layout xsi:type='JsonLayout' includeAllProperties="true" maxRecursionLimit="2"/> - </attribute> - </layout> - <layout xsi:type='SimpleLayout' text="," /> - </layout> - </target> - <target - xsi:type="File" - name="fileLogC" - fileName="C:/coscine/logs/${assembly-name}/${assembly-version}/log-${shortdate}.log" - maxArchiveFiles="7" - > - <layout xsi:type="CompoundLayout"> - <layout xsi:type="JsonLayout" EscapeForwardSlash="true"> - <attribute layout="${longdate}" name="Timestamp"/> - <attribute layout="${level:upperCase=true}" name="Level"/> - <attribute layout="${message}" name="Message"/> - <attribute layout="${exception:format=tostring,StackTrace}" name="Exception"/> - <attribute layout="${ndlc}" name="Context"/> - <attribute layout="${event-properties:item=Metric}" name="Alarm" encode="false"/> - <attribute name="EventProperties" encode="false" > - <layout xsi:type='JsonLayout' includeAllProperties="true" maxRecursionLimit="2"/> - </attribute> - </layout> - <layout xsi:type='SimpleLayout' text="," /> - </layout> - </target> - </target> - - <!-- Write colored logs to Console --> - <target name="consoleLog" xsi:type="ColoredConsole" layout="[${uppercase:${level}}]: ${message}"> - <highlight-row condition="level == LogLevel.Debug" foregroundColor="DarkGray" /> - <highlight-row condition="level == LogLevel.Info" foregroundColor="White" /> - <highlight-row condition="level == LogLevel.Warn" foregroundColor="Yellow" /> - <highlight-row condition="level == LogLevel.Error" foregroundColor="DarkRed" /> - <highlight-row condition="level == LogLevel.Fatal" foregroundColor="Red" backgroundColor="White" /> - </target> - </targets> - - - <rules> - <!--All logs, including from Microsoft, Level Trace--> - <logger name="*" minlevel="Trace" writeTo="fileGroup"> - </logger> - - <!--All logs, including from Microsoft, Level Info--> - <logger name="*" minlevel="Info" writeTo="consoleLog"> - <filters defaultAction="Log"> - <when condition="contains('${ndlc}','/api/heartbeat')" action="Ignore"/> - </filters> - </logger> - - </rules> -</nlog> \ No newline at end of file diff --git a/src/KPI Generator.sln b/src/KpiGenerator.sln similarity index 86% rename from src/KPI Generator.sln rename to src/KpiGenerator.sln index c483cbc95ba311f5834f7a23f122dde4641d0b60..43ff3942319cc241b3601c9cdfa131cf11266b2a 100644 --- a/src/KPI Generator.sln +++ b/src/KpiGenerator.sln @@ -3,7 +3,7 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.2.32526.322 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KPI Generator", "KPI Generator\KPI Generator.csproj", "{2B402E93-467B-49C1-8350-9277BDEDA9C3}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KpiGenerator", "KpiGenerator\KpiGenerator.csproj", "{2B402E93-467B-49C1-8350-9277BDEDA9C3}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/src/KpiGenerator/Dockerfile b/src/KpiGenerator/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..343f59f05708803a1ee5c97db994d620b7460631 --- /dev/null +++ b/src/KpiGenerator/Dockerfile @@ -0,0 +1,33 @@ +FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build-env +WORKDIR /App + +# Copy everything +COPY . ./ + +# Add nuget sources for private packages (here: api-client) +RUN dotnet nuget add source "https://git.rwth-aachen.de/api/v4/projects/88930/packages/nuget/index.json" -n "api-client" + +# Restore as distinct layers +RUN dotnet restore + +# Build and publish a release +RUN dotnet publish -c Release -o out + +# Build runtime image +FROM mcr.microsoft.com/dotnet/aspnet:8.0 + +# Install cron and other necessary packages +RUN apt-get update && \ + apt-get install -y cron && \ + apt-get clean + +WORKDIR /App +COPY --from=build-env /App/out . + +# Set the build-time argument and default environment variable +ENV DOTNET_ENVIRONMENT=Development + +# Create the log file +RUN touch /var/log/cron.log + +ENTRYPOINT /bin/sh -c "/App/Coscine.KpiGenerator all" \ No newline at end of file diff --git a/src/KPI Generator/KPI Generator.csproj b/src/KpiGenerator/KpiGenerator.csproj similarity index 80% rename from src/KPI Generator/KPI Generator.csproj rename to src/KpiGenerator/KpiGenerator.csproj index 438351a03ac1859ef82c36badd74fcaca0232633..06b2fe24c15667b7e5f9660d3b3c745b157e5284 100644 --- a/src/KPI Generator/KPI Generator.csproj +++ b/src/KpiGenerator/KpiGenerator.csproj @@ -21,7 +21,7 @@ <PackageReference Include="AutoMapper" Version="12.0.1" /> <PackageReference Include="AutoMapper.Extensions.Microsoft.DependencyInjection" Version="12.0.1" /> <PackageReference Include="CommandLineParser" Version="2.9.1" /> - <PackageReference Include="Coscine.ApiClient" Version="1.6.0" /> + <PackageReference Include="Coscine.ApiClient" Version="1.9.2" /> <PackageReference Include="dotNetRdf.Core" Version="3.1.1" /> <PackageReference Include="GitLabApiClient" Version="1.8.1-beta.5" /> <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.0" /> @@ -35,8 +35,14 @@ </ItemGroup> <ItemGroup> - <None Update="nlog.config"> - <CopyToOutputDirectory>Always</CopyToOutputDirectory> - </None> + <None Update="appsettings.json"> + <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> + </None> + <None Update="appsettings.Development.json"> + <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> + </None> + <None Update="nlog.config"> + <CopyToOutputDirectory>Always</CopyToOutputDirectory> + </None> </ItemGroup> </Project> diff --git a/src/KPI Generator/Logging/AssemblyNameLayoutRenderer.cs b/src/KpiGenerator/Logging/AssemblyNameLayoutRenderer.cs similarity index 100% rename from src/KPI Generator/Logging/AssemblyNameLayoutRenderer.cs rename to src/KpiGenerator/Logging/AssemblyNameLayoutRenderer.cs diff --git a/src/KPI Generator/Logging/AssemblyVersionLayoutRenderer.cs b/src/KpiGenerator/Logging/AssemblyVersionLayoutRenderer.cs similarity index 100% rename from src/KPI Generator/Logging/AssemblyVersionLayoutRenderer.cs rename to src/KpiGenerator/Logging/AssemblyVersionLayoutRenderer.cs diff --git a/src/KPI Generator/MappingProfiles/MappingProfiles.cs b/src/KpiGenerator/MappingProfiles/MappingProfiles.cs similarity index 91% rename from src/KPI Generator/MappingProfiles/MappingProfiles.cs rename to src/KpiGenerator/MappingProfiles/MappingProfiles.cs index 06a048bdd2dde9b2462f112fc86429d5bc3e11ca..ca15d23ecc330164c5e96a37dedbb779a04441bd 100644 --- a/src/KPI Generator/MappingProfiles/MappingProfiles.cs +++ b/src/KpiGenerator/MappingProfiles/MappingProfiles.cs @@ -17,7 +17,8 @@ public class MappingProfiles : Profile .ForMember(pr => pr.Deleted, opt => opt.MapFrom(dto => dto.Deleted)) .ForMember(pr => pr.ProjectVisibility, opt => opt.MapFrom(dto => dto.Visibility)) .ForMember(pr => pr.Users, opt => opt.MapFrom(dto => dto.ProjectRoles.Count)) - .ForMember(pr => pr.ResourceTypeQuota, opt => opt.MapFrom(dto => dto.ProjectQuota)); + .ForMember(pr => pr.ResourceTypeQuota, opt => opt.MapFrom(dto => dto.ProjectQuota)) + .ForMember(pr => pr.PublicationRequests, opt => opt.MapFrom(dto => dto.PublicationRequests)); CreateMap<ResourceAdminDto, ResourceReport>() .ForMember(rr => rr.Id, opt => opt.MapFrom(dto => dto.Id)) @@ -50,12 +51,12 @@ public class MappingProfiles : Profile .ForMember(lp => lp.DisplayName, opt => opt.MapFrom(dto => dto.DisplayName)); CreateMap<UserInstituteDto, Organization>() - .ForMember(o => o.ReadOnly, opt => opt.MapFrom(dto => dto.VarReadOnly)) + .ForMember(o => o.ReadOnly, opt => opt.MapFrom(dto => dto.ReadOnly)) .ForMember(o => o.Name, opt => opt.MapFrom(dto => dto.DisplayName)) .ForMember(o => o.RorUrl, opt => opt.MapFrom(dto => dto.Uri)); CreateMap<UserOrganizationDto, Organization>() - .ForMember(o => o.ReadOnly, opt => opt.MapFrom(dto => dto.VarReadOnly)) + .ForMember(o => o.ReadOnly, opt => opt.MapFrom(dto => dto.ReadOnly)) .ForMember(o => o.Name, opt => opt.MapFrom(dto => dto.DisplayName)) .ForMember(o => o.RorUrl, opt => opt.MapFrom(dto => dto.Uri)); @@ -93,7 +94,9 @@ public class MappingProfiles : Profile .ForMember(rtq => rtq.Allocated, opt => opt.MapFrom(dto => dto.Allocated)) .ForMember(rtq => rtq.Maximum, opt => opt.MapFrom(dto => dto.Maximum)); - CreateMap<ActivityLogDto, PublicationRequestReport>() - .ForMember(pr => pr.Timestamp, opt => opt.MapFrom(dto => dto.ActivityTimestamp)); - } + CreateMap<ProjectPublicationRequestDto, PublicationRequestReport>() + .ForMember(prr => prr.PublicationServiceRorId, opt => opt.MapFrom(dto => dto.PublicationServiceRorId)) + .ForMember(prr => prr.DateCreated, opt => opt.MapFrom(dto => dto.DateCreated)) + .ForMember(prr => prr.Resources, opt => opt.MapFrom(dto => dto.Resources)); + } } diff --git a/src/KPI Generator/Models/ApplicationProfileReport.cs b/src/KpiGenerator/Models/ApplicationProfileReport.cs similarity index 86% rename from src/KPI Generator/Models/ApplicationProfileReport.cs rename to src/KpiGenerator/Models/ApplicationProfileReport.cs index fdbd63a0617bc82fa3a143665012b003ec398d05..fcada6cba731704e8beffbee5cab799e87f10941 100644 --- a/src/KPI Generator/Models/ApplicationProfileReport.cs +++ b/src/KpiGenerator/Models/ApplicationProfileReport.cs @@ -5,7 +5,7 @@ /// </summary> public class ApplicationProfileReport { - public List<string> Titles { get; set; } = new(); + public IReadOnlyList<string> Titles { get; set; } = []; public string Uri { get; set; } = null!; public string? Publisher { get; set; } = null!; public string? Rights { get; set; } = null!; diff --git a/src/KPI Generator/Models/ConfigurationModels/KpiConfiguration.cs b/src/KpiGenerator/Models/ConfigurationModels/KpiConfiguration.cs similarity index 100% rename from src/KPI Generator/Models/ConfigurationModels/KpiConfiguration.cs rename to src/KpiGenerator/Models/ConfigurationModels/KpiConfiguration.cs diff --git a/src/KPI Generator/Models/ConfigurationModels/ReportingConfiguration.cs b/src/KpiGenerator/Models/ConfigurationModels/ReportingConfiguration.cs similarity index 75% rename from src/KPI Generator/Models/ConfigurationModels/ReportingConfiguration.cs rename to src/KpiGenerator/Models/ConfigurationModels/ReportingConfiguration.cs index 84b7a722133690fc3d55d688c37bc77419a980f2..1b1c5a183ce6bd5dd8a6d3ce16db77dfdac8e331 100644 --- a/src/KPI Generator/Models/ConfigurationModels/ReportingConfiguration.cs +++ b/src/KpiGenerator/Models/ConfigurationModels/ReportingConfiguration.cs @@ -15,6 +15,17 @@ public class ReportingConfiguration /// </summary> public bool IsEnabled { get; init; } + /// <summary> + /// API token for the coscine API. + /// Requires the administrator role. + /// </summary> + public string ApiKey { get; set; } = null!; + + /// <summary> + /// Address of the Coscine API. + /// </summary> + public string Endpoint { get; set; } = null!; + /// <summary> /// Local storage configuration settings. /// </summary> @@ -23,6 +34,14 @@ public class ReportingConfiguration /// </value> public LocalStorageConfiguration? Local { get; init; } + /// <summary> + /// Logger configuration settings. + /// </summary> + /// <value> + /// The logger storage configuration settings, or <c>null</c> if not configured. + /// </value> + public LoggerConfiguration? Logger { get; init; } + /// <summary> /// GitLab configuration settings. /// </summary> @@ -67,4 +86,9 @@ public class ReportingConfiguration /// </summary> public Organization? OtherOrganization { get; init; } } + + /// <summary> + /// Represents the configuration settings for the logger. + /// </summary> + public record LoggerConfiguration(string? LogLevel, string? LogHome); } diff --git a/src/KPI Generator/Models/Discipline.cs b/src/KpiGenerator/Models/Discipline.cs similarity index 100% rename from src/KPI Generator/Models/Discipline.cs rename to src/KpiGenerator/Models/Discipline.cs diff --git a/src/KPI Generator/Models/LoginProvider.cs b/src/KpiGenerator/Models/LoginProvider.cs similarity index 100% rename from src/KPI Generator/Models/LoginProvider.cs rename to src/KpiGenerator/Models/LoginProvider.cs diff --git a/src/KPI Generator/Models/MaintenanceReport.cs b/src/KpiGenerator/Models/MaintenanceReport.cs similarity index 100% rename from src/KPI Generator/Models/MaintenanceReport.cs rename to src/KpiGenerator/Models/MaintenanceReport.cs diff --git a/src/KPI Generator/Models/MetadataVisibility.cs b/src/KpiGenerator/Models/MetadataVisibility.cs similarity index 100% rename from src/KPI Generator/Models/MetadataVisibility.cs rename to src/KpiGenerator/Models/MetadataVisibility.cs diff --git a/src/KPI Generator/Models/Organization.cs b/src/KpiGenerator/Models/Organization.cs similarity index 100% rename from src/KPI Generator/Models/Organization.cs rename to src/KpiGenerator/Models/Organization.cs diff --git a/src/KPI Generator/Models/ProjectReport.cs b/src/KpiGenerator/Models/ProjectReport.cs similarity index 100% rename from src/KPI Generator/Models/ProjectReport.cs rename to src/KpiGenerator/Models/ProjectReport.cs diff --git a/src/KPI Generator/Models/ProjectVisibility.cs b/src/KpiGenerator/Models/ProjectVisibility.cs similarity index 100% rename from src/KPI Generator/Models/ProjectVisibility.cs rename to src/KpiGenerator/Models/ProjectVisibility.cs diff --git a/src/KpiGenerator/Models/PublicationRequestReport.cs b/src/KpiGenerator/Models/PublicationRequestReport.cs new file mode 100644 index 0000000000000000000000000000000000000000..cbc209c35f8ca0e7fc1afc5cebd6e256add54067 --- /dev/null +++ b/src/KpiGenerator/Models/PublicationRequestReport.cs @@ -0,0 +1,16 @@ +using System.Text.Json.Serialization; +using Coscine.ApiClient.Core.Model; + +namespace Coscine.KpiGenerator.Models; + +public record PublicationRequestReport +{ + [JsonPropertyName("PublicationServiceRorId")] + public Uri PublicationServiceRorId { get; init; } = null!; + + [JsonPropertyName("DateCreated")] + public DateTime DateCreated { get; set; } + + [JsonPropertyName("Resources")] + public IReadOnlyList<ResourceMinimalDto> Resources { get; set; } = null!; +} \ No newline at end of file diff --git a/src/KPI Generator/Models/Quota.cs b/src/KpiGenerator/Models/Quota.cs similarity index 100% rename from src/KPI Generator/Models/Quota.cs rename to src/KpiGenerator/Models/Quota.cs diff --git a/src/KPI Generator/Models/RelatedProject.cs b/src/KpiGenerator/Models/RelatedProject.cs similarity index 100% rename from src/KPI Generator/Models/RelatedProject.cs rename to src/KpiGenerator/Models/RelatedProject.cs diff --git a/src/KPI Generator/Models/ReportingFile.cs b/src/KpiGenerator/Models/ReportingFile.cs similarity index 100% rename from src/KPI Generator/Models/ReportingFile.cs rename to src/KpiGenerator/Models/ReportingFile.cs diff --git a/src/KPI Generator/Models/ResourceQuota.cs b/src/KpiGenerator/Models/ResourceQuota.cs similarity index 100% rename from src/KPI Generator/Models/ResourceQuota.cs rename to src/KpiGenerator/Models/ResourceQuota.cs diff --git a/src/KPI Generator/Models/ResourceReport.cs b/src/KpiGenerator/Models/ResourceReport.cs similarity index 100% rename from src/KPI Generator/Models/ResourceReport.cs rename to src/KpiGenerator/Models/ResourceReport.cs diff --git a/src/KPI Generator/Models/ResourceTypeQuota.cs b/src/KpiGenerator/Models/ResourceTypeQuota.cs similarity index 100% rename from src/KPI Generator/Models/ResourceTypeQuota.cs rename to src/KpiGenerator/Models/ResourceTypeQuota.cs diff --git a/src/KPI Generator/Models/SystemReport.cs b/src/KpiGenerator/Models/SystemReport.cs similarity index 100% rename from src/KPI Generator/Models/SystemReport.cs rename to src/KpiGenerator/Models/SystemReport.cs diff --git a/src/KPI Generator/Models/UserReport.cs b/src/KpiGenerator/Models/UserReport.cs similarity index 100% rename from src/KPI Generator/Models/UserReport.cs rename to src/KpiGenerator/Models/UserReport.cs diff --git a/src/KPI Generator/Program.cs b/src/KpiGenerator/Program.cs similarity index 81% rename from src/KPI Generator/Program.cs rename to src/KpiGenerator/Program.cs index 100d8e4582db24284207acc48281ee3ea9a5fddb..c1f05f50b2e040150549ff2ffa2b04665c1f2ad0 100644 --- a/src/KPI Generator/Program.cs +++ b/src/KpiGenerator/Program.cs @@ -19,9 +19,14 @@ namespace Coscine.KpiGenerator; public class Program { private static IServiceProvider _serviceProvider = null!; + private static string? _environmentName; - public static int Main(string[] args) + public static async Task<int> Main(string[] args) { + _environmentName = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT"); + + Console.WriteLine($"Running in environment: {_environmentName ?? "environment is null"}"); + InitializeServices(); var logger = _serviceProvider.GetRequiredService<ILogger<Program>>(); @@ -37,13 +42,13 @@ public class Program SystemReportingOptions >(args); - var result = parserResult.MapResult( - (CompleteReportingOptions opts) => _serviceProvider.GetRequiredService<CompleteReporting>().RunAsync(opts).Result, - (ProjectReportingOptions opts) => _serviceProvider.GetRequiredService<ProjectReporting>().RunAsync(opts).Result, - (ResourceReportingOptions opts) => _serviceProvider.GetRequiredService<ResourceReporting>().RunAsync(opts).Result, - (UserReportingOptions opts) => _serviceProvider.GetRequiredService<UserReporting>().RunAsync(opts).Result, - (ApplicationProfileReportingOptions opts) => _serviceProvider.GetRequiredService<ApplicationProfileReporting>().RunAsync(opts).Result, - (SystemReportingOptions opts) => _serviceProvider.GetRequiredService<SystemReporting>().RunAsync(opts).Result, + var result = await parserResult.MapResult( + (CompleteReportingOptions opts) => _serviceProvider.GetRequiredService<CompleteReporting>().RunAsync(opts), + (ProjectReportingOptions opts) => _serviceProvider.GetRequiredService<ProjectReporting>().RunAsync(opts), + (ResourceReportingOptions opts) => _serviceProvider.GetRequiredService<ResourceReporting>().RunAsync(opts), + (UserReportingOptions opts) => _serviceProvider.GetRequiredService<UserReporting>().RunAsync(opts), + (ApplicationProfileReportingOptions opts) => _serviceProvider.GetRequiredService<ApplicationProfileReporting>().RunAsync(opts), + (SystemReportingOptions opts) => _serviceProvider.GetRequiredService<SystemReporting>().RunAsync(opts), HandleParseError ); @@ -89,8 +94,9 @@ public class Program // Add Consul as a configuration source var configuration = configBuilder - .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true) - .AddConsul( + .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true) + .AddJsonFile($"appsettings.{_environmentName}.json", optional: true, reloadOnChange: true) + .AddConsul( "coscine/Coscine.Infrastructure/KpiGenerator/appsettings", options => { @@ -102,6 +108,18 @@ public class Program options.OnLoadException = exceptionContext => exceptionContext.Ignore = true; } ) + .AddConsul( + $"coscine/Coscine.Infrastructure/KpiGenerator/appsettings.{_environmentName}.json", + options => + { + options.ConsulConfigurationOptions = + cco => cco.Address = new Uri(consulUrl); + options.Optional = true; + options.ReloadOnChange = true; + options.PollWaitTime = TimeSpan.FromSeconds(5); + options.OnLoadException = exceptionContext => exceptionContext.Ignore = true; + } + ) .AddEnvironmentVariables() .Build(); @@ -126,6 +144,14 @@ public class Program builder.AddNLog(); }); + var reportingConfiguration = new ReportingConfiguration(); + configuration.Bind(ReportingConfiguration.Section, reportingConfiguration); + + // Set the default LogLevel + LogManager.Configuration.Variables["logLevel"] = reportingConfiguration.Logger?.LogLevel ?? "Trace"; + // Set the log location + LogManager.Configuration.Variables["logHome"] = reportingConfiguration.Logger?.LogHome ?? Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Logs"); + // Add AutoMapper services.AddAutoMapper(typeof(Program)); @@ -162,7 +188,7 @@ public class Program } } - private static bool HandleParseError(IEnumerable<Error> errs) + private static Task<bool> HandleParseError(IEnumerable<Error> errs) { var logger = _serviceProvider.GetService<ILogger<Program>>(); @@ -183,10 +209,9 @@ public class Program } // Since there were errors, we typically return false to indicate that the program should not proceed - return false; + return Task.FromResult(false); } - public static TResult SanitizeOptions<TResult>(TResult unsanitizedOptions) { // Sanitize all input that accepts an array or is a list of inputs. diff --git a/src/KPI Generator/Reportings/ApplicationProfile/ApplicationProfileReporting.cs b/src/KpiGenerator/Reportings/ApplicationProfile/ApplicationProfileReporting.cs similarity index 100% rename from src/KPI Generator/Reportings/ApplicationProfile/ApplicationProfileReporting.cs rename to src/KpiGenerator/Reportings/ApplicationProfile/ApplicationProfileReporting.cs diff --git a/src/KPI Generator/Reportings/ApplicationProfile/ApplicationProfileReportingOptions.cs b/src/KpiGenerator/Reportings/ApplicationProfile/ApplicationProfileReportingOptions.cs similarity index 100% rename from src/KPI Generator/Reportings/ApplicationProfile/ApplicationProfileReportingOptions.cs rename to src/KpiGenerator/Reportings/ApplicationProfile/ApplicationProfileReportingOptions.cs diff --git a/src/KPI Generator/Reportings/Complete/CompleteReporting.cs b/src/KpiGenerator/Reportings/Complete/CompleteReporting.cs similarity index 100% rename from src/KPI Generator/Reportings/Complete/CompleteReporting.cs rename to src/KpiGenerator/Reportings/Complete/CompleteReporting.cs diff --git a/src/KPI Generator/Reportings/Complete/CompleteReportingOptions.cs b/src/KpiGenerator/Reportings/Complete/CompleteReportingOptions.cs similarity index 100% rename from src/KPI Generator/Reportings/Complete/CompleteReportingOptions.cs rename to src/KpiGenerator/Reportings/Complete/CompleteReportingOptions.cs diff --git a/src/KPI Generator/Reportings/Project/ProjectReporting.cs b/src/KpiGenerator/Reportings/Project/ProjectReporting.cs similarity index 74% rename from src/KPI Generator/Reportings/Project/ProjectReporting.cs rename to src/KpiGenerator/Reportings/Project/ProjectReporting.cs index 49005039ce60ea0ce16c0c218df44e3812bb5570..b2110a7ff415465e0cfb8c639f8aab971147220c 100644 --- a/src/KPI Generator/Reportings/Project/ProjectReporting.cs +++ b/src/KpiGenerator/Reportings/Project/ProjectReporting.cs @@ -49,10 +49,10 @@ public class ProjectReporting var configuration = new Configuration() { - BasePath = "http://localhost:7206/coscine", + BasePath = $"{_reportingConfiguration.Endpoint.TrimEnd('/')}/coscine", ApiKeyPrefix = { { "Authorization", "Bearer" } }, - ApiKey = { { "Authorization", ApiConfigurationUtil.GenerateAdminToken(ApiConfigurationUtil.RetrieveJwtConfiguration()) } }, - Timeout = 300000, // 5 minutes + ApiKey = { { "Authorization", _reportingConfiguration.ApiKey } }, + Timeout = 300000 // 5 minutes }; AdminApi = new AdminApi(configuration); @@ -81,7 +81,7 @@ public class ProjectReporting { _logger.LogDebug("Woring on projects asynchronously..."); var projects = PaginationHelper.GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>( - (currentPage) => AdminApi.GetAllProjectsAsync(includeDeleted: false, includeQuotas: true, pageNumber: currentPage, pageSize: 50)); + (currentPage) => AdminApi.GetAllProjectsAsync(includeDeleted: false, includeQuotas: true, includePublicationRequests: true, pageNumber: currentPage, pageSize: 50)); var reportingFiles = new List<ReportingFileObject>(); var returnObjects = new List<ProjectReport>(); @@ -91,33 +91,6 @@ public class ProjectReporting { _logger.LogDebug("Processing project {projectId}...", project.Id); var returnObject = _mapper.Map<ProjectReport>(project); - - // Get the activity logs for publication requests for the project - var utcTimestampBefore = DateTime.UtcNow.Date; - var utcTimestampAfter = utcTimestampBefore.AddDays(-1); - _logger.LogDebug("Processing activity logs from {timestampAfter} to {timestampBefore}...", utcTimestampAfter.ToString("yyyy-MM-dd HH:mm:ss"), utcTimestampBefore.ToString("yyyy-MM-dd HH:mm:ss")); - //lang=regex - var regex = @$"{project.Slug}.*publication"; // The slug is used to filter the logs to only those related to the project - _logger.LogDebug("Processing logs with regular expression {regex}...", regex); - var activityLogs = new List<ActivityLogDto>(); - await foreach (var log in PaginationHelper.GetAllAsync<ActivityLogDtoPagedResponse, ActivityLogDto>( - currentPage => AdminApi.GetAllActivityLogsAsync( - activityTimestampBefore: utcTimestampBefore, - activityTimestampAfter: utcTimestampAfter, - regularExpression: regex, - guid: null, // Note on project ID vs. slug - httpMethod: CoscineHttpMethod.POST, - pageNumber: currentPage, - pageSize: 50))) - { - if (log is not null) - { - activityLogs.Add(log); - } - } - _logger.LogDebug("{n} activity logs processed.", activityLogs.Count); - returnObject.PublicationRequests = _mapper.Map<List<PublicationRequestReport>>(activityLogs); - _logger.LogDebug("Processed publication requests..."); returnObjects.Add(returnObject); } _logger.LogInformation("{n} projects mapped.", returnObjects.Count); diff --git a/src/KPI Generator/Reportings/Project/ProjectReportingOptions.cs b/src/KpiGenerator/Reportings/Project/ProjectReportingOptions.cs similarity index 100% rename from src/KPI Generator/Reportings/Project/ProjectReportingOptions.cs rename to src/KpiGenerator/Reportings/Project/ProjectReportingOptions.cs diff --git a/src/KPI Generator/Reportings/Resource/ResourceReporting.cs b/src/KpiGenerator/Reportings/Resource/ResourceReporting.cs similarity index 100% rename from src/KPI Generator/Reportings/Resource/ResourceReporting.cs rename to src/KpiGenerator/Reportings/Resource/ResourceReporting.cs diff --git a/src/KPI Generator/Reportings/Resource/ResourceReportingOptions.cs b/src/KpiGenerator/Reportings/Resource/ResourceReportingOptions.cs similarity index 100% rename from src/KPI Generator/Reportings/Resource/ResourceReportingOptions.cs rename to src/KpiGenerator/Reportings/Resource/ResourceReportingOptions.cs diff --git a/src/KPI Generator/Reportings/System/SystemReporting.cs b/src/KpiGenerator/Reportings/System/SystemReporting.cs similarity index 100% rename from src/KPI Generator/Reportings/System/SystemReporting.cs rename to src/KpiGenerator/Reportings/System/SystemReporting.cs diff --git a/src/KPI Generator/Reportings/System/SystemReportingOptions.cs b/src/KpiGenerator/Reportings/System/SystemReportingOptions.cs similarity index 100% rename from src/KPI Generator/Reportings/System/SystemReportingOptions.cs rename to src/KpiGenerator/Reportings/System/SystemReportingOptions.cs diff --git a/src/KPI Generator/Reportings/User/UserReporting.cs b/src/KpiGenerator/Reportings/User/UserReporting.cs similarity index 100% rename from src/KPI Generator/Reportings/User/UserReporting.cs rename to src/KpiGenerator/Reportings/User/UserReporting.cs diff --git a/src/KPI Generator/Reportings/User/UserReportingOptions.cs b/src/KpiGenerator/Reportings/User/UserReportingOptions.cs similarity index 100% rename from src/KPI Generator/Reportings/User/UserReportingOptions.cs rename to src/KpiGenerator/Reportings/User/UserReportingOptions.cs diff --git a/src/KPI Generator/Utils/CommandLineOptions.cs b/src/KpiGenerator/Utils/CommandLineOptions.cs similarity index 100% rename from src/KPI Generator/Utils/CommandLineOptions.cs rename to src/KpiGenerator/Utils/CommandLineOptions.cs diff --git a/src/KPI Generator/Utils/FileSystemStorageService.cs b/src/KpiGenerator/Utils/FileSystemStorageService.cs similarity index 86% rename from src/KPI Generator/Utils/FileSystemStorageService.cs rename to src/KpiGenerator/Utils/FileSystemStorageService.cs index a9f6c9391244ed2183547f27c5fc28b9879572b6..381dc5539399fa4703ac121e93470872df1afd37 100644 --- a/src/KPI Generator/Utils/FileSystemStorageService.cs +++ b/src/KpiGenerator/Utils/FileSystemStorageService.cs @@ -5,21 +5,14 @@ using Microsoft.Extensions.Options; namespace Coscine.KpiGenerator.Utils; -public class FileSystemStorageService : IStorageService +public class FileSystemStorageService( + ILogger<FileSystemStorageService> logger, + IOptionsMonitor<ReportingConfiguration> reportingConfiguration) : IStorageService { - private readonly ReportingConfiguration _reportingConfiguration; - private readonly ILogger<FileSystemStorageService> _logger; + private readonly ReportingConfiguration _reportingConfiguration = reportingConfiguration.CurrentValue; + private readonly ILogger<FileSystemStorageService> _logger = logger; private string? _localStoragePath; - public FileSystemStorageService( - ILogger<FileSystemStorageService> logger, - IOptionsMonitor<ReportingConfiguration> reportingConfiguration - ) - { - _reportingConfiguration = reportingConfiguration.CurrentValue; - _logger = logger; - } - public Task EnsureInformationIsSetAndCorrectAsync() { // Ensure that the base file path from the configuration is valid diff --git a/src/KPI Generator/Utils/GitLabStorageService.cs b/src/KpiGenerator/Utils/GitLabStorageService.cs similarity index 100% rename from src/KPI Generator/Utils/GitLabStorageService.cs rename to src/KpiGenerator/Utils/GitLabStorageService.cs diff --git a/src/KPI Generator/Utils/GitLabUtils.cs b/src/KpiGenerator/Utils/GitLabUtils.cs similarity index 100% rename from src/KPI Generator/Utils/GitLabUtils.cs rename to src/KpiGenerator/Utils/GitLabUtils.cs diff --git a/src/KPI Generator/Utils/Helpers.cs b/src/KpiGenerator/Utils/Helpers.cs similarity index 100% rename from src/KPI Generator/Utils/Helpers.cs rename to src/KpiGenerator/Utils/Helpers.cs diff --git a/src/KPI Generator/Utils/IStorageService.cs b/src/KpiGenerator/Utils/IStorageService.cs similarity index 100% rename from src/KPI Generator/Utils/IStorageService.cs rename to src/KpiGenerator/Utils/IStorageService.cs diff --git a/src/KpiGenerator/appsettings.Example.json b/src/KpiGenerator/appsettings.Example.json new file mode 100644 index 0000000000000000000000000000000000000000..40be147020e2dc41b053c3d587183c12af4f100f --- /dev/null +++ b/src/KpiGenerator/appsettings.Example.json @@ -0,0 +1,19 @@ +{ + "ReportingConfiguration": { + "ApiKey": null, + "Endpoint": null, + "GitLab": { + "Token": null, + "Branch": null + } + }, + "KpiConfiguration": { + "SystemKpi": { + "Maintenance": { + "Username": null, + "Password": null, + "BasicAuthToken": null + } + } + } +} \ No newline at end of file diff --git a/src/KPI Generator/appsettings.json b/src/KpiGenerator/appsettings.json similarity index 71% rename from src/KPI Generator/appsettings.json rename to src/KpiGenerator/appsettings.json index 8ba75d42d1bef0c03d87b57efb356734b6c80517..f0ecd482c85f32d8b1ca2c0d43ea3c2883a2728c 100644 --- a/src/KPI Generator/appsettings.json +++ b/src/KpiGenerator/appsettings.json @@ -1,13 +1,19 @@ { "ReportingConfiguration": { "IsEnabled": true, + "ApiKey": null, + "Endpoint": null, "Local": { "StoragePath": "C:/coscine/reporting/" }, + "Logger": { + "LogLevel": "Information", + "LogHome": "./Logs" + }, "GitLab": { "HostUrl": "https://git.rwth-aachen.de/", - "Token": "", - "Branch": "", + "Token": null, + "Branch": null, "ProjectId": 76546 }, "Organizations": { @@ -34,10 +40,10 @@ "SystemKpi": { "FileName": "system_status.json", "Maintenance": { - "Url": "", - "Username": "", - "Password": "", - "BasicAuthToken": "" + "Url": "https://maintenance.itc.rwth-aachen.de/ticket/api/coscine/tickets", + "Username": null, + "Password": null, + "BasicAuthToken": null } }, "UserKpi": { diff --git a/src/KpiGenerator/nlog.config b/src/KpiGenerator/nlog.config new file mode 100644 index 0000000000000000000000000000000000000000..f7f07d333702fc7677ba59da0f39dafd6275097a --- /dev/null +++ b/src/KpiGenerator/nlog.config @@ -0,0 +1,54 @@ +<?xml version="1.0" encoding="utf-8" ?> +<nlog xmlns="http://www.nlog-project.org/schemas/NLog.xsd" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + autoReload="true" + internalLogLevel="Warn" + internalLogFile="./internal_logs/internallog.txt"> + + <variable name="logHome" value="${basedir}/Logs" /> + <variable name="logLevel" value="Warn" /> + + <!-- This variable is used to remove ANSI escape codes from the log message. --> + <variable name="message_raw" value="${replace:inner=${message}:searchFor=\x1B\[[0-9;]*[A-Za-z]:replaceWith=:regex=true}" /> + + <!--Possible aspnet- variables: https://nlog-project.org/config/?tab=layout-renderers&search=package:nlog.web--> + <variable name="layout" value="${longdate} | [${level:uppercase=true}] ${message_raw} ${exception:format=tostring}" /> + + <targets> + <!-- Write logs to File --> + <target + name="fullfile" + xsi:type="File" + layout="${var:layout}" + fileName="${var:logHome}/full_${shortdate}.log" + keepFileOpen="false" + archiveFileName="${var:logHome}/Archive/full_${shortdate}.log" + archiveNumbering="Sequence" + archiveEvery="Day" + maxArchiveFiles="7" + /> + + <!-- Write colored logs to Console --> + <target name="consoleLog" xsi:type="ColoredConsole" layout="[${uppercase:${level}}]: ${message}"> + <highlight-word text="[TRACE]" foregroundColor="DarkGray" /> + <highlight-word text="[Debug]" foregroundColor="Gray" /> + <highlight-word text="[INFO]" foregroundColor="Green" /> + <highlight-word text="[WARN]" foregroundColor="Yellow" /> + <highlight-word text="[ERROR]" foregroundColor="Red" /> + <highlight-word text="[FATAL]" foregroundColor="DarkRed" backgroundColor="White" /> + </target> + </targets> + + + <rules> + <!--All logs, including Microsoft--> + <logger name="*" minlevel="${var:logLevel}" writeTo="fullfile" /> + + <!--Skip non-critical Microsoft logs and so log only own logs (BlackHole).--> + <logger name="Microsoft.*" maxlevel="Info" final="true" /> + <logger name="System.Net.Http.*" maxlevel="Info" final="true" /> + + <!--All logs, including from Microsoft, Level Info--> + <logger name="*" minlevel="${var:logLevel}" writeTo="consoleLog" /> + </rules> +</nlog> \ No newline at end of file