From 1d61998e3c5f93b18bea382746dd2d592d5c79c8 Mon Sep 17 00:00:00 2001 From: Petar Hristov <hristov@itc.rwth-aachen.de> Date: Thu, 22 May 2025 09:29:41 +0000 Subject: [PATCH 1/5] Enhance logging for asynchronous project, resource, and user retrieval --- .../Reportings/Project/ProjectReporting.cs | 6 +++++- .../Reportings/Resource/ResourceReporting.cs | 21 +++++++++---------- .../Reportings/User/UserReporting.cs | 6 +++++- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/KpiGenerator/Reportings/Project/ProjectReporting.cs b/src/KpiGenerator/Reportings/Project/ProjectReporting.cs index e8bee5e..fd98154 100644 --- a/src/KpiGenerator/Reportings/Project/ProjectReporting.cs +++ b/src/KpiGenerator/Reportings/Project/ProjectReporting.cs @@ -82,7 +82,11 @@ public class ProjectReporting { _logger.LogDebug("Working on projects asynchronously..."); var projects = PaginationHelper.GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>( - (currentPage) => _adminApi.GetAllProjectsAsync(includeDeleted: false, includeQuotas: true, includePublicationRequests: true, pageNumber: currentPage, pageSize: 50)); + (currentPage) => + { + _logger.LogDebug("Getting page {page} of projects...", currentPage); + return _adminApi.GetAllProjectsAsync(includeDeleted: false, includeQuotas: true, includePublicationRequests: true, pageNumber: currentPage, pageSize: 50); + }); var reportingFiles = new List<ReportingFileObject>(); var returnObjects = new List<ProjectReport>(); diff --git a/src/KpiGenerator/Reportings/Resource/ResourceReporting.cs b/src/KpiGenerator/Reportings/Resource/ResourceReporting.cs index 48fb60b..f13df44 100644 --- a/src/KpiGenerator/Reportings/Resource/ResourceReporting.cs +++ b/src/KpiGenerator/Reportings/Resource/ResourceReporting.cs @@ -44,15 +44,6 @@ public class ResourceReporting _kpiConfiguration = kpiConfiguration.CurrentValue; _reportingConfiguration = reportingConfiguration.CurrentValue; ReportingFileName = _kpiConfiguration.ResourceKpi.FileName; - - var configuration = new Configuration() - { - BasePath = $"{_reportingConfiguration.Endpoint.TrimEnd('/')}/coscine", - ApiKeyPrefix = { { "Authorization", "Bearer" } }, - ApiKey = { { "Authorization", _reportingConfiguration.ApiKey } }, - Timeout = TimeSpan.FromSeconds(300) // 5 minutes - }; - _adminApi = adminApi; } @@ -90,12 +81,20 @@ public class ResourceReporting { _logger.LogDebug("Getting all projects..."); var projects = await PaginationHelper.GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>( - (currentPage) => _adminApi.GetAllProjectsAsync(includeDeleted: true, pageNumber: currentPage, pageSize: 50)).ToListAsync(); + (currentPage) => + { + _logger.LogDebug("Getting page {page} of projects...", currentPage); + return _adminApi.GetAllProjectsAsync(includeDeleted: true, pageNumber: currentPage, pageSize: 50); + }).ToListAsync(); _logger.LogDebug("Got all projects."); _logger.LogDebug("Working on resources asynchronously..."); var resources = PaginationHelper.GetAllAsync<ResourceAdminDtoPagedResponse, ResourceAdminDto>( - (currentPage) => _adminApi.GetAllResourcesAsync(includeDeleted: false, includeQuotas: true, pageNumber: currentPage, pageSize: 50)); + (currentPage) => + { + _logger.LogDebug("Getting page {page} of resources...", currentPage); + return _adminApi.GetAllResourcesAsync(includeDeleted: false, includeQuotas: true, pageNumber: currentPage, pageSize: 50); + }); var reportingFiles = new List<ReportingFileObject>(); var returnObjects = new List<ResourceReport>(); diff --git a/src/KpiGenerator/Reportings/User/UserReporting.cs b/src/KpiGenerator/Reportings/User/UserReporting.cs index 8db2ad4..65867ca 100644 --- a/src/KpiGenerator/Reportings/User/UserReporting.cs +++ b/src/KpiGenerator/Reportings/User/UserReporting.cs @@ -94,7 +94,11 @@ public class UserReporting _logger.LogDebug("Working on users asynchronously..."); var users = PaginationHelper.GetAllAsync<UserDtoPagedResponse, UserDto>( - (currentPage) => _adminApi.GetAllUsersAsync(tosAccepted: true, pageNumber: currentPage, pageSize: 50)); + (currentPage) => + { + _logger.LogDebug("Getting page {page} of users...", currentPage); + return _adminApi.GetAllUsersAsync(tosAccepted: true, pageNumber: currentPage, pageSize: 50); + }); var reportingFiles = new List<ReportingFileObject>(); var returnObjects = new List<UserReport>(); -- GitLab From 12492f2884b3dff4a272d3be4f239bf180629c4f Mon Sep 17 00:00:00 2001 From: Petar Hristov <hristov@itc.rwth-aachen.de> Date: Wed, 28 May 2025 10:28:04 +0000 Subject: [PATCH 2/5] Fix: ProjectReporting to fetch project quotas individually and update API client dependency to version 1.10.0 --- .../ProjectReportingTests.cs | 14 ++++++----- src/KpiGenerator/KpiGenerator.csproj | 2 +- src/KpiGenerator/Program.cs | 2 +- .../Reportings/Project/ProjectReporting.cs | 25 ++++++++++++++++--- 4 files changed, 32 insertions(+), 11 deletions(-) diff --git a/src/KpiGenerator.Tests/ProjectReportingTests.cs b/src/KpiGenerator.Tests/ProjectReportingTests.cs index 861c6fc..84093aa 100644 --- a/src/KpiGenerator.Tests/ProjectReportingTests.cs +++ b/src/KpiGenerator.Tests/ProjectReportingTests.cs @@ -26,6 +26,7 @@ public class ProjectReportingTests private IOptionsMonitor<ReportingConfiguration> _reportingConfiguration = null!; private IAdminApi _adminApi = null!; + private IProjectQuotaApi _projectQuotaApi = null!; private ProjectReporting _projectReporting = null!; // System Under Test @@ -70,6 +71,7 @@ public class ProjectReportingTests _reportingConfiguration.CurrentValue.Returns(reportingConfig); _adminApi = Substitute.For<IAdminApi>(); + _projectQuotaApi = Substitute.For<IProjectQuotaApi>(); } #region GenerateReportingAsync Tests @@ -94,7 +96,7 @@ public class ProjectReportingTests var pagination = new Pagination(currentPage: 1, pageSize: 2, totalCount: 2, totalPages: 1); return Task.FromResult(new ProjectAdminDtoPagedResponse(data: projects, pagination: pagination, statusCode: 200, traceId: "dummy-trace-id")); }); - _projectReporting = new ProjectReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi); + _projectReporting = new ProjectReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectQuotaApi); // Act var result = await _projectReporting.GenerateReportingAsync(); @@ -126,7 +128,7 @@ public class ProjectReportingTests var pagination = new Pagination(currentPage: 1, pageSize: 0, totalCount: 0, totalPages: 1); return Task.FromResult(new ProjectAdminDtoPagedResponse(data: [], pagination: pagination, statusCode: 200, traceId: "dummy-trace-id")); }); - _projectReporting = new ProjectReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi); + _projectReporting = new ProjectReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectQuotaApi); // Act var result = await _projectReporting.GenerateReportingAsync(); @@ -153,7 +155,7 @@ public class ProjectReportingTests }; // We want to ensure that GenerateReportingAsync returns some test objects - _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi); + _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectQuotaApi); _projectReporting .Configure() .GenerateReportingAsync() @@ -193,7 +195,7 @@ public class ProjectReportingTests }; // Partial mock to override GenerateReportingAsync - _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi); + _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectQuotaApi); _projectReporting .Configure() .GenerateReportingAsync() @@ -233,7 +235,7 @@ public class ProjectReportingTests }; // Partial mock to override GenerateReportingAsync - _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi); + _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectQuotaApi); _projectReporting .Configure() .GenerateReportingAsync() @@ -270,7 +272,7 @@ public class ProjectReportingTests }; // Partial mock - _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi); + _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectQuotaApi); _projectReporting .Configure() .GenerateReportingAsync() diff --git a/src/KpiGenerator/KpiGenerator.csproj b/src/KpiGenerator/KpiGenerator.csproj index c304988..bc810e6 100644 --- a/src/KpiGenerator/KpiGenerator.csproj +++ b/src/KpiGenerator/KpiGenerator.csproj @@ -21,7 +21,7 @@ <PackageReference Include="AutoMapper" Version="12.0.1" /> <PackageReference Include="AutoMapper.Extensions.Microsoft.DependencyInjection" Version="12.0.1" /> <PackageReference Include="CommandLineParser" Version="2.9.1" /> - <PackageReference Include="Coscine.ApiClient" Version="1.9.9" /> + <PackageReference Include="Coscine.ApiClient" Version="1.10.0" /> <PackageReference Include="dotNetRdf.Core" Version="3.1.1" /> <PackageReference Include="GitLabApiClient" Version="1.8.1-beta.5" /> <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.0" /> diff --git a/src/KpiGenerator/Program.cs b/src/KpiGenerator/Program.cs index 55e6679..cd1df9d 100644 --- a/src/KpiGenerator/Program.cs +++ b/src/KpiGenerator/Program.cs @@ -176,7 +176,7 @@ public class Program BasePath = $"{reportingConfiguration.Endpoint.TrimEnd('/')}/coscine", ApiKeyPrefix = { { "Authorization", "Bearer" } }, ApiKey = { { "Authorization", reportingConfiguration.ApiKey } }, - Timeout = TimeSpan.FromSeconds(300) // 5 minutes + Timeout = TimeSpan.FromSeconds(300), // 5 minutes }; services.AddSingleton<IAdminApi>(new AdminApi(apiConfiguration)); services.AddSingleton<IApplicationProfileApi>(new ApplicationProfileApi(apiConfiguration)); diff --git a/src/KpiGenerator/Reportings/Project/ProjectReporting.cs b/src/KpiGenerator/Reportings/Project/ProjectReporting.cs index fd98154..da977cb 100644 --- a/src/KpiGenerator/Reportings/Project/ProjectReporting.cs +++ b/src/KpiGenerator/Reportings/Project/ProjectReporting.cs @@ -22,6 +22,7 @@ public class ProjectReporting private readonly KpiConfiguration _kpiConfiguration; private readonly ReportingConfiguration _reportingConfiguration; private readonly IAdminApi _adminApi; + private readonly IProjectQuotaApi _projectQuotaApi; public ProjectReportingOptions Options { get; private set; } = null!; public string ReportingFileName { get; } @@ -33,7 +34,8 @@ public class ProjectReporting [FromKeyedServices("local")] IStorageService localStorageService, IOptionsMonitor<KpiConfiguration> kpiConfiguration, IOptionsMonitor<ReportingConfiguration> reportingConfiguration, - IAdminApi adminApi + IAdminApi adminApi, + IProjectQuotaApi projectQuotaApi ) { _mapper = mapper; @@ -45,6 +47,7 @@ public class ProjectReporting ReportingFileName = _kpiConfiguration.ProjectKpi.FileName; _adminApi = adminApi; + _projectQuotaApi = projectQuotaApi; } public async Task<bool> RunAsync(ProjectReportingOptions reportingOptions) @@ -84,8 +87,8 @@ public class ProjectReporting var projects = PaginationHelper.GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>( (currentPage) => { - _logger.LogDebug("Getting page {page} of projects...", currentPage); - return _adminApi.GetAllProjectsAsync(includeDeleted: false, includeQuotas: true, includePublicationRequests: true, pageNumber: currentPage, pageSize: 50); + _logger.LogDebug("Getting page {page} of projects...", currentPage); + return _adminApi.GetAllProjectsAsync(includeDeleted: false, includeQuotas: false, includePublicationRequests: true, pageNumber: currentPage, pageSize: 50); }); var reportingFiles = new List<ReportingFileObject>(); @@ -95,6 +98,22 @@ public class ProjectReporting await foreach (var project in projects) { _logger.LogDebug("Processing project {projectId}...", project.Id); + var quotas = PaginationHelper.GetAllAsync<ProjectQuotaDtoPagedResponse, ProjectQuotaDto>( + (currentPage) => + { + _logger.LogDebug("Getting page {page} of quotas for project {projectId}...", currentPage, project.Id); + return _projectQuotaApi.GetProjectQuotasAsync(project.Id.ToString(), pageNumber: currentPage, pageSize: 50); + }); + await foreach (var quota in quotas) + { + if (quota == null) + { + _logger.LogWarning("Quota for project {projectId} is null, skipping...", project.Id); + continue; + } + // Map the quota to the project + project.ProjectQuota.Add(quota); + } var returnObject = _mapper.Map<ProjectReport>(project); returnObjects.Add(returnObject); } -- GitLab From 9637bdd85b3176dc1551f0329848b95773bcbd2b Mon Sep 17 00:00:00 2001 From: Petar Hristov <hristov@itc.rwth-aachen.de> Date: Wed, 28 May 2025 15:15:14 +0200 Subject: [PATCH 3/5] Fix: Fetch only 25 users at once --- src/KpiGenerator/Reportings/User/UserReporting.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/KpiGenerator/Reportings/User/UserReporting.cs b/src/KpiGenerator/Reportings/User/UserReporting.cs index 65867ca..c1e8e57 100644 --- a/src/KpiGenerator/Reportings/User/UserReporting.cs +++ b/src/KpiGenerator/Reportings/User/UserReporting.cs @@ -97,7 +97,7 @@ public class UserReporting (currentPage) => { _logger.LogDebug("Getting page {page} of users...", currentPage); - return _adminApi.GetAllUsersAsync(tosAccepted: true, pageNumber: currentPage, pageSize: 50); + return _adminApi.GetAllUsersAsync(tosAccepted: true, pageNumber: currentPage, pageSize: 25); }); var reportingFiles = new List<ReportingFileObject>(); -- GitLab From 40dc29cc7c468a243357cc99ffe915404b64cd4d Mon Sep 17 00:00:00 2001 From: Petar Hristov <hristov@itc.rwth-aachen.de> Date: Fri, 30 May 2025 12:01:21 +0000 Subject: [PATCH 4/5] Update: Introduce ProjectCacheService for project retrieval and update reporting classes to use it --- .../ProjectReportingTests.cs | 43 ++++++------------- src/KpiGenerator.Tests/UserReportingTests.cs | 15 ++++--- src/KpiGenerator/Program.cs | 1 + .../Reportings/Project/ProjectReporting.cs | 24 ++++++----- .../Reportings/Resource/ResourceReporting.cs | 16 +++---- .../Reportings/User/UserReporting.cs | 14 +++--- src/KpiGenerator/Utils/ProjectCacheService.cs | 35 +++++++++++++++ 7 files changed, 87 insertions(+), 61 deletions(-) create mode 100644 src/KpiGenerator/Utils/ProjectCacheService.cs diff --git a/src/KpiGenerator.Tests/ProjectReportingTests.cs b/src/KpiGenerator.Tests/ProjectReportingTests.cs index 84093aa..5c92406 100644 --- a/src/KpiGenerator.Tests/ProjectReportingTests.cs +++ b/src/KpiGenerator.Tests/ProjectReportingTests.cs @@ -25,9 +25,8 @@ public class ProjectReportingTests private IOptionsMonitor<KpiConfiguration> _kpiConfiguration = null!; private IOptionsMonitor<ReportingConfiguration> _reportingConfiguration = null!; - private IAdminApi _adminApi = null!; private IProjectQuotaApi _projectQuotaApi = null!; - + private IProjectCacheService _projectCacheService = null!; private ProjectReporting _projectReporting = null!; // System Under Test [SetUp] @@ -70,7 +69,7 @@ public class ProjectReportingTests _reportingConfiguration = Substitute.For<IOptionsMonitor<ReportingConfiguration>>(); _reportingConfiguration.CurrentValue.Returns(reportingConfig); - _adminApi = Substitute.For<IAdminApi>(); + _projectCacheService = Substitute.For<IProjectCacheService>(); _projectQuotaApi = Substitute.For<IProjectQuotaApi>(); } @@ -82,21 +81,10 @@ public class ProjectReportingTests // Arrange var projects = TestData.ProjectAdminDtos; - _adminApi - .GetAllProjectsAsync( - includeDeleted: Arg.Any<bool>(), - includeQuotas: Arg.Any<bool>(), - includePublicationRequests: Arg.Any<bool>(), - pageNumber: Arg.Any<int>(), - pageSize: Arg.Any<int>() - ) - .Returns(ci => - { - // Return the test projects data, single page - var pagination = new Pagination(currentPage: 1, pageSize: 2, totalCount: 2, totalPages: 1); - return Task.FromResult(new ProjectAdminDtoPagedResponse(data: projects, pagination: pagination, statusCode: 200, traceId: "dummy-trace-id")); - }); - _projectReporting = new ProjectReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectQuotaApi); + _projectCacheService + .GetAllProjectsAsync() + .Returns(ci => Task.FromResult(projects)); + _projectReporting = new ProjectReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _projectCacheService, _projectQuotaApi); // Act var result = await _projectReporting.GenerateReportingAsync(); @@ -120,15 +108,10 @@ public class ProjectReportingTests public async Task GenerateReportingAsync_ReturnsOnlyGeneralFile_WhenNoProjects() { // Arrange - _adminApi + _projectCacheService .GetAllProjectsAsync() - .Returns(ci => - { - // No projects, empty data - var pagination = new Pagination(currentPage: 1, pageSize: 0, totalCount: 0, totalPages: 1); - return Task.FromResult(new ProjectAdminDtoPagedResponse(data: [], pagination: pagination, statusCode: 200, traceId: "dummy-trace-id")); - }); - _projectReporting = new ProjectReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectQuotaApi); + .Returns(ci => Task.FromResult(new List<ProjectAdminDto>())); + _projectReporting = new ProjectReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _projectCacheService, _projectQuotaApi); // Act var result = await _projectReporting.GenerateReportingAsync(); @@ -155,7 +138,7 @@ public class ProjectReportingTests }; // We want to ensure that GenerateReportingAsync returns some test objects - _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectQuotaApi); + _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _projectCacheService, _projectQuotaApi); _projectReporting .Configure() .GenerateReportingAsync() @@ -195,7 +178,7 @@ public class ProjectReportingTests }; // Partial mock to override GenerateReportingAsync - _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectQuotaApi); + _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _projectCacheService, _projectQuotaApi); _projectReporting .Configure() .GenerateReportingAsync() @@ -235,7 +218,7 @@ public class ProjectReportingTests }; // Partial mock to override GenerateReportingAsync - _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectQuotaApi); + _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _projectCacheService, _projectQuotaApi); _projectReporting .Configure() .GenerateReportingAsync() @@ -272,7 +255,7 @@ public class ProjectReportingTests }; // Partial mock - _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectQuotaApi); + _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _projectCacheService, _projectQuotaApi); _projectReporting .Configure() .GenerateReportingAsync() diff --git a/src/KpiGenerator.Tests/UserReportingTests.cs b/src/KpiGenerator.Tests/UserReportingTests.cs index 5386dbd..ec78546 100644 --- a/src/KpiGenerator.Tests/UserReportingTests.cs +++ b/src/KpiGenerator.Tests/UserReportingTests.cs @@ -27,7 +27,7 @@ public class UserReportingTests private IAdminApi _adminApi = null!; private IRoleApi _roleApi = null!; - + private IProjectCacheService _projectCacheService = null!; private UserReporting _userReporting = null!; // System Under Test [SetUp] @@ -72,6 +72,7 @@ public class UserReportingTests _adminApi = Substitute.For<IAdminApi>(); _roleApi = Substitute.For<IRoleApi>(); + _projectCacheService = Substitute.For<IProjectCacheService>(); } #region GenerateReportingAsync Tests @@ -119,7 +120,7 @@ public class UserReportingTests var pagination = new Pagination(currentPage: 1, pageSize: 2, totalCount: 2, totalPages: 1); return Task.FromResult(new RoleDtoPagedResponse(data: roles, pagination: pagination, statusCode: 200, traceId: "dummy-trace-id")); }); - _userReporting = new UserReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi); + _userReporting = new UserReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi, _projectCacheService); // Act var result = await _userReporting.GenerateReportingAsync(); @@ -167,7 +168,7 @@ public class UserReportingTests var pagination = new Pagination(currentPage: 1, pageSize: 0, totalCount: 0, totalPages: 1); return Task.FromResult(new RoleDtoPagedResponse(data: [], pagination: pagination, statusCode: 200, traceId: "dummy-trace-id")); }); - _userReporting = new UserReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi); + _userReporting = new UserReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi, _projectCacheService); // Act var result = await _userReporting.GenerateReportingAsync(); @@ -194,7 +195,7 @@ public class UserReportingTests }; // We want to ensure that GenerateReportingAsync returns some test objects - _userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi); + _userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi, _projectCacheService); _userReporting .Configure() .GenerateReportingAsync() @@ -234,7 +235,7 @@ public class UserReportingTests }; // Partial mock to override GenerateReportingAsync - _userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi); + _userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi, _projectCacheService); _userReporting .Configure() .GenerateReportingAsync() @@ -274,7 +275,7 @@ public class UserReportingTests }; // Partial mock to override GenerateReportingAsync - _userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi); + _userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi, _projectCacheService); _userReporting .Configure() .GenerateReportingAsync() @@ -311,7 +312,7 @@ public class UserReportingTests }; // Partial mock - _userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi); + _userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi, _projectCacheService); _userReporting .Configure() .GenerateReportingAsync() diff --git a/src/KpiGenerator/Program.cs b/src/KpiGenerator/Program.cs index cd1df9d..5c1aa63 100644 --- a/src/KpiGenerator/Program.cs +++ b/src/KpiGenerator/Program.cs @@ -180,6 +180,7 @@ public class Program }; services.AddSingleton<IAdminApi>(new AdminApi(apiConfiguration)); services.AddSingleton<IApplicationProfileApi>(new ApplicationProfileApi(apiConfiguration)); + services.AddSingleton<IProjectCacheService, ProjectCacheService>(); services.AddSingleton<IProjectApi>(new ProjectApi(apiConfiguration)); services.AddSingleton<IProjectQuotaApi>(new ProjectQuotaApi(apiConfiguration)); services.AddSingleton<IProjectResourceQuotaApi>(new ProjectResourceQuotaApi(apiConfiguration)); diff --git a/src/KpiGenerator/Reportings/Project/ProjectReporting.cs b/src/KpiGenerator/Reportings/Project/ProjectReporting.cs index da977cb..876926b 100644 --- a/src/KpiGenerator/Reportings/Project/ProjectReporting.cs +++ b/src/KpiGenerator/Reportings/Project/ProjectReporting.cs @@ -21,7 +21,7 @@ public class ProjectReporting private readonly IStorageService _localStorageService; private readonly KpiConfiguration _kpiConfiguration; private readonly ReportingConfiguration _reportingConfiguration; - private readonly IAdminApi _adminApi; + private readonly IProjectCacheService _projectCacheService; private readonly IProjectQuotaApi _projectQuotaApi; public ProjectReportingOptions Options { get; private set; } = null!; @@ -34,7 +34,7 @@ public class ProjectReporting [FromKeyedServices("local")] IStorageService localStorageService, IOptionsMonitor<KpiConfiguration> kpiConfiguration, IOptionsMonitor<ReportingConfiguration> reportingConfiguration, - IAdminApi adminApi, + IProjectCacheService projectCacheService, IProjectQuotaApi projectQuotaApi ) { @@ -46,7 +46,7 @@ public class ProjectReporting _reportingConfiguration = reportingConfiguration.CurrentValue; ReportingFileName = _kpiConfiguration.ProjectKpi.FileName; - _adminApi = adminApi; + _projectCacheService = projectCacheService; _projectQuotaApi = projectQuotaApi; } @@ -84,18 +84,22 @@ public class ProjectReporting public virtual async Task<IEnumerable<ReportingFileObject>> GenerateReportingAsync() { _logger.LogDebug("Working on projects asynchronously..."); - var projects = PaginationHelper.GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>( - (currentPage) => - { - _logger.LogDebug("Getting page {page} of projects...", currentPage); - return _adminApi.GetAllProjectsAsync(includeDeleted: false, includeQuotas: false, includePublicationRequests: true, pageNumber: currentPage, pageSize: 50); - }); + var projects = await _projectCacheService.GetAllProjectsAsync(); + _logger.LogInformation("Found {count} projects.", projects.Count); + if (projects.Count == 0) + { + _logger.LogWarning("No projects found. Exiting project reporting generation."); + return []; + } + // Filter out projects that are deleted + projects = [.. projects.Where(p => !p.Deleted)]; + _logger.LogInformation("Filtered out deleted projects. Remaining projects: {count}", projects.Count); var reportingFiles = new List<ReportingFileObject>(); var returnObjects = new List<ProjectReport>(); // Additional processing - await foreach (var project in projects) + foreach (var project in projects) { _logger.LogDebug("Processing project {projectId}...", project.Id); var quotas = PaginationHelper.GetAllAsync<ProjectQuotaDtoPagedResponse, ProjectQuotaDto>( diff --git a/src/KpiGenerator/Reportings/Resource/ResourceReporting.cs b/src/KpiGenerator/Reportings/Resource/ResourceReporting.cs index f13df44..e2d4cb5 100644 --- a/src/KpiGenerator/Reportings/Resource/ResourceReporting.cs +++ b/src/KpiGenerator/Reportings/Resource/ResourceReporting.cs @@ -23,6 +23,7 @@ public class ResourceReporting private readonly KpiConfiguration _kpiConfiguration; private readonly ReportingConfiguration _reportingConfiguration; private readonly IAdminApi _adminApi; + private readonly IProjectCacheService _projectCacheService; public ResourceReportingOptions Options { get; private set; } = null!; public string ReportingFileName { get; } @@ -34,7 +35,8 @@ public class ResourceReporting [FromKeyedServices("local")] IStorageService localStorageService, IOptionsMonitor<KpiConfiguration> kpiConfiguration, IOptionsMonitor<ReportingConfiguration> reportingConfiguration, - IAdminApi adminApi + IAdminApi adminApi, + IProjectCacheService projectCacheService ) { _mapper = mapper; @@ -45,6 +47,7 @@ public class ResourceReporting _reportingConfiguration = reportingConfiguration.CurrentValue; ReportingFileName = _kpiConfiguration.ResourceKpi.FileName; _adminApi = adminApi; + _projectCacheService = projectCacheService; } public async Task<bool> RunAsync(ResourceReportingOptions reportingOptions) @@ -80,20 +83,15 @@ public class ResourceReporting public virtual async Task<IEnumerable<ReportingFileObject>> GenerateReportingAsync() { _logger.LogDebug("Getting all projects..."); - var projects = await PaginationHelper.GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>( - (currentPage) => - { - _logger.LogDebug("Getting page {page} of projects...", currentPage); - return _adminApi.GetAllProjectsAsync(includeDeleted: true, pageNumber: currentPage, pageSize: 50); - }).ToListAsync(); - _logger.LogDebug("Got all projects."); + var projects = await _projectCacheService.GetAllProjectsAsync(); + _logger.LogDebug("Got all {count} projects, including deleted ones.", projects.Count); _logger.LogDebug("Working on resources asynchronously..."); var resources = PaginationHelper.GetAllAsync<ResourceAdminDtoPagedResponse, ResourceAdminDto>( (currentPage) => { _logger.LogDebug("Getting page {page} of resources...", currentPage); - return _adminApi.GetAllResourcesAsync(includeDeleted: false, includeQuotas: true, pageNumber: currentPage, pageSize: 50); + return _adminApi.GetAllResourcesAsync(includeDeleted: false, includeQuotas: true, pageNumber: currentPage, pageSize: 10); }); var reportingFiles = new List<ReportingFileObject>(); diff --git a/src/KpiGenerator/Reportings/User/UserReporting.cs b/src/KpiGenerator/Reportings/User/UserReporting.cs index c1e8e57..e7a1836 100644 --- a/src/KpiGenerator/Reportings/User/UserReporting.cs +++ b/src/KpiGenerator/Reportings/User/UserReporting.cs @@ -23,6 +23,7 @@ public class UserReporting private readonly ReportingConfiguration _reportingConfiguration; private readonly IAdminApi _adminApi; private readonly IRoleApi _roleApi; + private readonly IProjectCacheService _projectCacheService; public UserReportingOptions Options { get; private set; } = null!; public string ReportingFileName { get; } @@ -35,7 +36,8 @@ public class UserReporting IOptionsMonitor<KpiConfiguration> kpiConfiguration, IOptionsMonitor<ReportingConfiguration> reportingConfiguration, IAdminApi adminApi, - IRoleApi roleApi + IRoleApi roleApi, + IProjectCacheService projectCacheService ) { _mapper = mapper; @@ -48,6 +50,7 @@ public class UserReporting _adminApi = adminApi; _roleApi = roleApi; + _projectCacheService = projectCacheService; } public async Task<bool> RunAsync(UserReportingOptions reportingOptions) @@ -83,9 +86,10 @@ public class UserReporting public virtual async Task<IEnumerable<ReportingFileObject>> GenerateReportingAsync() { _logger.LogDebug("Getting all projects..."); - var projects = await PaginationHelper.GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>( - (currentPage) => _adminApi.GetAllProjectsAsync(includeDeleted: false, pageNumber: currentPage, pageSize: 50)).ToListAsync(); - _logger.LogDebug("Got all projects."); + var projects = await _projectCacheService.GetAllProjectsAsync(); + // Filter out projects that are deleted + projects = [.. projects.Where(p => !p.Deleted)]; + _logger.LogInformation("Filtered out deleted projects. Remaining projects: {count}", projects.Count); _logger.LogDebug("Getting all roles..."); var roles = await PaginationHelper.GetAllAsync<RoleDtoPagedResponse, RoleDto>( @@ -97,7 +101,7 @@ public class UserReporting (currentPage) => { _logger.LogDebug("Getting page {page} of users...", currentPage); - return _adminApi.GetAllUsersAsync(tosAccepted: true, pageNumber: currentPage, pageSize: 25); + return _adminApi.GetAllUsersAsync(tosAccepted: true, pageNumber: currentPage, pageSize: 10); }); var reportingFiles = new List<ReportingFileObject>(); diff --git a/src/KpiGenerator/Utils/ProjectCacheService.cs b/src/KpiGenerator/Utils/ProjectCacheService.cs new file mode 100644 index 0000000..5416492 --- /dev/null +++ b/src/KpiGenerator/Utils/ProjectCacheService.cs @@ -0,0 +1,35 @@ +using Coscine.ApiClient; +using Coscine.ApiClient.Core.Api; +using Coscine.ApiClient.Core.Model; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; + +namespace Coscine.KpiGenerator.Utils; + +public interface IProjectCacheService +{ + Task<List<ProjectAdminDto>> GetAllProjectsAsync(); +} + +public class ProjectCacheService(IAdminApi adminApi, IMemoryCache cache, ILogger<ProjectCacheService> logger) : IProjectCacheService +{ + private const string CacheKey = "AllProjects"; + private readonly IAdminApi _adminApi = adminApi; + private readonly IMemoryCache _cache = cache; + private readonly ILogger<ProjectCacheService> _logger = logger; + + public async Task<List<ProjectAdminDto>> GetAllProjectsAsync() + { + return await _cache.GetOrCreateAsync(CacheKey, async entry => + { + entry.AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(120); + _logger.LogDebug("Fetching all projects from API"); + var list = await PaginationHelper + .GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>( + page => _adminApi.GetAllProjectsAsync(includeDeleted: true, includePublicationRequests: true, pageNumber: page, pageSize: 50)) + .ToListAsync(); + _logger.LogDebug("Cached {Count} projects", list.Count); + return list; + }) ?? []; + } +} -- GitLab From 40c10402a8713f0656c7de02b057a0a96f16afc6 Mon Sep 17 00:00:00 2001 From: Petar Hristov <hristov@itc.rwth-aachen.de> Date: Fri, 30 May 2025 13:35:10 +0000 Subject: [PATCH 5/5] Refactor: Integrate ProjectCacheService into Resource and User Reporting tests, replacing direct API calls with cached project retrieval --- .../ResourceReportingTests.cs | 41 +++++++------------ src/KpiGenerator.Tests/UserReportingTests.cs | 24 +++-------- .../Reportings/Project/ProjectReporting.cs | 6 +-- 3 files changed, 20 insertions(+), 51 deletions(-) diff --git a/src/KpiGenerator.Tests/ResourceReportingTests.cs b/src/KpiGenerator.Tests/ResourceReportingTests.cs index ebd6030..652e4c9 100644 --- a/src/KpiGenerator.Tests/ResourceReportingTests.cs +++ b/src/KpiGenerator.Tests/ResourceReportingTests.cs @@ -26,7 +26,7 @@ public class ResourceReportingTests private IOptionsMonitor<ReportingConfiguration> _reportingConfiguration = null!; private IAdminApi _adminApi = null!; - + private IProjectCacheService _projectCacheService = null!; private ResourceReporting _resourceReporting = null!; // System Under Test [SetUp] @@ -68,8 +68,9 @@ public class ResourceReportingTests }; _reportingConfiguration = Substitute.For<IOptionsMonitor<ReportingConfiguration>>(); _reportingConfiguration.CurrentValue.Returns(reportingConfig); - + _adminApi = Substitute.For<IAdminApi>(); + _projectCacheService = Substitute.For<IProjectCacheService>(); } #region GenerateReportingAsync Tests @@ -81,18 +82,9 @@ public class ResourceReportingTests var projects = TestData.ProjectAdminDtos; var resources = TestData.ResourceAdminDtos; - _adminApi - .GetAllProjectsAsync( - includeDeleted: Arg.Any<bool>(), - pageNumber: Arg.Any<int>(), - pageSize: Arg.Any<int>() - ) - .Returns(ci => - { - // Return the test projects data, single page - var pagination = new Pagination(currentPage: 1, pageSize: 2, totalCount: 2, totalPages: 1); - return Task.FromResult(new ProjectAdminDtoPagedResponse(data: projects, pagination: pagination, statusCode: 200, traceId: "dummy-trace-id")); - }); + _projectCacheService + .GetAllProjectsAsync() + .Returns(ci => Task.FromResult(projects)); _adminApi .GetAllResourcesAsync( includeDeleted: Arg.Any<bool>(), @@ -106,7 +98,7 @@ public class ResourceReportingTests var pagination = new Pagination(currentPage: 1, pageSize: 2, totalCount: 2, totalPages: 1); return Task.FromResult(new ResourceAdminDtoPagedResponse(data: resources, pagination: pagination, statusCode: 200, traceId: "dummy-trace-id")); }); - _resourceReporting = new ResourceReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi); + _resourceReporting = new ResourceReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectCacheService); // Act var result = await _resourceReporting.GenerateReportingAsync(); @@ -130,14 +122,9 @@ public class ResourceReportingTests public async Task GenerateReportingAsync_ReturnsOnlyGeneralFile_WhenNoProjects() { // Arrange - _adminApi + _projectCacheService .GetAllProjectsAsync() - .Returns(ci => - { - // No projects, empty data - var pagination = new Pagination(currentPage: 1, pageSize: 0, totalCount: 0, totalPages: 1); - return Task.FromResult(new ProjectAdminDtoPagedResponse(data: [], pagination: pagination, statusCode: 200, traceId: "dummy-trace-id")); - }); + .Returns(ci => Task.FromResult(new List<ProjectAdminDto>())); _adminApi .GetAllResourcesAsync() .Returns(ci => @@ -146,7 +133,7 @@ public class ResourceReportingTests var pagination = new Pagination(currentPage: 1, pageSize: 0, totalCount: 0, totalPages: 1); return Task.FromResult(new ResourceAdminDtoPagedResponse(data: [], pagination: pagination, statusCode: 200, traceId: "dummy-trace-id")); }); - _resourceReporting = new ResourceReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi); + _resourceReporting = new ResourceReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectCacheService); // Act var result = await _resourceReporting.GenerateReportingAsync(); @@ -173,7 +160,7 @@ public class ResourceReportingTests }; // We want to ensure that GenerateReportingAsync returns some test objects - _resourceReporting = Substitute.ForPartsOf<ResourceReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi); + _resourceReporting = Substitute.ForPartsOf<ResourceReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectCacheService); _resourceReporting .Configure() .GenerateReportingAsync() @@ -213,7 +200,7 @@ public class ResourceReportingTests }; // Partial mock to override GenerateReportingAsync - _resourceReporting = Substitute.ForPartsOf<ResourceReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi); + _resourceReporting = Substitute.ForPartsOf<ResourceReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectCacheService); _resourceReporting .Configure() .GenerateReportingAsync() @@ -253,7 +240,7 @@ public class ResourceReportingTests }; // Partial mock to override GenerateReportingAsync - _resourceReporting = Substitute.ForPartsOf<ResourceReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi); + _resourceReporting = Substitute.ForPartsOf<ResourceReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectCacheService); _resourceReporting .Configure() .GenerateReportingAsync() @@ -290,7 +277,7 @@ public class ResourceReportingTests }; // Partial mock - _resourceReporting = Substitute.ForPartsOf<ResourceReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi); + _resourceReporting = Substitute.ForPartsOf<ResourceReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectCacheService); _resourceReporting .Configure() .GenerateReportingAsync() diff --git a/src/KpiGenerator.Tests/UserReportingTests.cs b/src/KpiGenerator.Tests/UserReportingTests.cs index ec78546..e17434c 100644 --- a/src/KpiGenerator.Tests/UserReportingTests.cs +++ b/src/KpiGenerator.Tests/UserReportingTests.cs @@ -85,18 +85,9 @@ public class UserReportingTests var users = TestData.UserDtos; var roles = TestData.RoleDtos; - _adminApi - .GetAllProjectsAsync( - includeDeleted: Arg.Any<bool>(), - pageNumber: Arg.Any<int>(), - pageSize: Arg.Any<int>() - ) - .Returns(ci => - { - // Return the test projects data, single page - var pagination = new Pagination(currentPage: 1, pageSize: 2, totalCount: 2, totalPages: 1); - return Task.FromResult(new ProjectAdminDtoPagedResponse(data: projects, pagination: pagination, statusCode: 200, traceId: "dummy-trace-id")); - }); + _projectCacheService + .GetAllProjectsAsync() + .Returns(ci => Task.FromResult(projects)); _adminApi .GetAllUsersAsync( tosAccepted: Arg.Any<bool>(), @@ -144,14 +135,9 @@ public class UserReportingTests public async Task GenerateReportingAsync_ReturnsOnlyGeneralFile_WhenNoProjects() { // Arrange - _adminApi + _projectCacheService .GetAllProjectsAsync() - .Returns(ci => - { - // No projects, empty data - var pagination = new Pagination(currentPage: 1, pageSize: 0, totalCount: 0, totalPages: 1); - return Task.FromResult(new ProjectAdminDtoPagedResponse(data: [], pagination: pagination, statusCode: 200, traceId: "dummy-trace-id")); - }); + .Returns(ci => Task.FromResult(new List<ProjectAdminDto>())); _adminApi .GetAllUsersAsync() .Returns(ci => diff --git a/src/KpiGenerator/Reportings/Project/ProjectReporting.cs b/src/KpiGenerator/Reportings/Project/ProjectReporting.cs index 876926b..08787f7 100644 --- a/src/KpiGenerator/Reportings/Project/ProjectReporting.cs +++ b/src/KpiGenerator/Reportings/Project/ProjectReporting.cs @@ -86,11 +86,7 @@ public class ProjectReporting _logger.LogDebug("Working on projects asynchronously..."); var projects = await _projectCacheService.GetAllProjectsAsync(); _logger.LogInformation("Found {count} projects.", projects.Count); - if (projects.Count == 0) - { - _logger.LogWarning("No projects found. Exiting project reporting generation."); - return []; - } + // Filter out projects that are deleted projects = [.. projects.Where(p => !p.Deleted)]; _logger.LogInformation("Filtered out deleted projects. Remaining projects: {count}", projects.Count); -- GitLab