From 40dc29cc7c468a243357cc99ffe915404b64cd4d Mon Sep 17 00:00:00 2001 From: Petar Hristov <hristov@itc.rwth-aachen.de> Date: Fri, 30 May 2025 12:01:21 +0000 Subject: [PATCH] Update: Introduce ProjectCacheService for project retrieval and update reporting classes to use it --- .../ProjectReportingTests.cs | 43 ++++++------------- src/KpiGenerator.Tests/UserReportingTests.cs | 15 ++++--- src/KpiGenerator/Program.cs | 1 + .../Reportings/Project/ProjectReporting.cs | 24 ++++++----- .../Reportings/Resource/ResourceReporting.cs | 16 +++---- .../Reportings/User/UserReporting.cs | 14 +++--- src/KpiGenerator/Utils/ProjectCacheService.cs | 35 +++++++++++++++ 7 files changed, 87 insertions(+), 61 deletions(-) create mode 100644 src/KpiGenerator/Utils/ProjectCacheService.cs diff --git a/src/KpiGenerator.Tests/ProjectReportingTests.cs b/src/KpiGenerator.Tests/ProjectReportingTests.cs index 84093aa..5c92406 100644 --- a/src/KpiGenerator.Tests/ProjectReportingTests.cs +++ b/src/KpiGenerator.Tests/ProjectReportingTests.cs @@ -25,9 +25,8 @@ public class ProjectReportingTests private IOptionsMonitor<KpiConfiguration> _kpiConfiguration = null!; private IOptionsMonitor<ReportingConfiguration> _reportingConfiguration = null!; - private IAdminApi _adminApi = null!; private IProjectQuotaApi _projectQuotaApi = null!; - + private IProjectCacheService _projectCacheService = null!; private ProjectReporting _projectReporting = null!; // System Under Test [SetUp] @@ -70,7 +69,7 @@ public class ProjectReportingTests _reportingConfiguration = Substitute.For<IOptionsMonitor<ReportingConfiguration>>(); _reportingConfiguration.CurrentValue.Returns(reportingConfig); - _adminApi = Substitute.For<IAdminApi>(); + _projectCacheService = Substitute.For<IProjectCacheService>(); _projectQuotaApi = Substitute.For<IProjectQuotaApi>(); } @@ -82,21 +81,10 @@ public class ProjectReportingTests // Arrange var projects = TestData.ProjectAdminDtos; - _adminApi - .GetAllProjectsAsync( - includeDeleted: Arg.Any<bool>(), - includeQuotas: Arg.Any<bool>(), - includePublicationRequests: Arg.Any<bool>(), - pageNumber: Arg.Any<int>(), - pageSize: Arg.Any<int>() - ) - .Returns(ci => - { - // Return the test projects data, single page - var pagination = new Pagination(currentPage: 1, pageSize: 2, totalCount: 2, totalPages: 1); - return Task.FromResult(new ProjectAdminDtoPagedResponse(data: projects, pagination: pagination, statusCode: 200, traceId: "dummy-trace-id")); - }); - _projectReporting = new ProjectReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectQuotaApi); + _projectCacheService + .GetAllProjectsAsync() + .Returns(ci => Task.FromResult(projects)); + _projectReporting = new ProjectReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _projectCacheService, _projectQuotaApi); // Act var result = await _projectReporting.GenerateReportingAsync(); @@ -120,15 +108,10 @@ public class ProjectReportingTests public async Task GenerateReportingAsync_ReturnsOnlyGeneralFile_WhenNoProjects() { // Arrange - _adminApi + _projectCacheService .GetAllProjectsAsync() - .Returns(ci => - { - // No projects, empty data - var pagination = new Pagination(currentPage: 1, pageSize: 0, totalCount: 0, totalPages: 1); - return Task.FromResult(new ProjectAdminDtoPagedResponse(data: [], pagination: pagination, statusCode: 200, traceId: "dummy-trace-id")); - }); - _projectReporting = new ProjectReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectQuotaApi); + .Returns(ci => Task.FromResult(new List<ProjectAdminDto>())); + _projectReporting = new ProjectReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _projectCacheService, _projectQuotaApi); // Act var result = await _projectReporting.GenerateReportingAsync(); @@ -155,7 +138,7 @@ public class ProjectReportingTests }; // We want to ensure that GenerateReportingAsync returns some test objects - _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectQuotaApi); + _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _projectCacheService, _projectQuotaApi); _projectReporting .Configure() .GenerateReportingAsync() @@ -195,7 +178,7 @@ public class ProjectReportingTests }; // Partial mock to override GenerateReportingAsync - _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectQuotaApi); + _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _projectCacheService, _projectQuotaApi); _projectReporting .Configure() .GenerateReportingAsync() @@ -235,7 +218,7 @@ public class ProjectReportingTests }; // Partial mock to override GenerateReportingAsync - _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectQuotaApi); + _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _projectCacheService, _projectQuotaApi); _projectReporting .Configure() .GenerateReportingAsync() @@ -272,7 +255,7 @@ public class ProjectReportingTests }; // Partial mock - _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectQuotaApi); + _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _projectCacheService, _projectQuotaApi); _projectReporting .Configure() .GenerateReportingAsync() diff --git a/src/KpiGenerator.Tests/UserReportingTests.cs b/src/KpiGenerator.Tests/UserReportingTests.cs index 5386dbd..ec78546 100644 --- a/src/KpiGenerator.Tests/UserReportingTests.cs +++ b/src/KpiGenerator.Tests/UserReportingTests.cs @@ -27,7 +27,7 @@ public class UserReportingTests private IAdminApi _adminApi = null!; private IRoleApi _roleApi = null!; - + private IProjectCacheService _projectCacheService = null!; private UserReporting _userReporting = null!; // System Under Test [SetUp] @@ -72,6 +72,7 @@ public class UserReportingTests _adminApi = Substitute.For<IAdminApi>(); _roleApi = Substitute.For<IRoleApi>(); + _projectCacheService = Substitute.For<IProjectCacheService>(); } #region GenerateReportingAsync Tests @@ -119,7 +120,7 @@ public class UserReportingTests var pagination = new Pagination(currentPage: 1, pageSize: 2, totalCount: 2, totalPages: 1); return Task.FromResult(new RoleDtoPagedResponse(data: roles, pagination: pagination, statusCode: 200, traceId: "dummy-trace-id")); }); - _userReporting = new UserReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi); + _userReporting = new UserReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi, _projectCacheService); // Act var result = await _userReporting.GenerateReportingAsync(); @@ -167,7 +168,7 @@ public class UserReportingTests var pagination = new Pagination(currentPage: 1, pageSize: 0, totalCount: 0, totalPages: 1); return Task.FromResult(new RoleDtoPagedResponse(data: [], pagination: pagination, statusCode: 200, traceId: "dummy-trace-id")); }); - _userReporting = new UserReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi); + _userReporting = new UserReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi, _projectCacheService); // Act var result = await _userReporting.GenerateReportingAsync(); @@ -194,7 +195,7 @@ public class UserReportingTests }; // We want to ensure that GenerateReportingAsync returns some test objects - _userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi); + _userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi, _projectCacheService); _userReporting .Configure() .GenerateReportingAsync() @@ -234,7 +235,7 @@ public class UserReportingTests }; // Partial mock to override GenerateReportingAsync - _userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi); + _userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi, _projectCacheService); _userReporting .Configure() .GenerateReportingAsync() @@ -274,7 +275,7 @@ public class UserReportingTests }; // Partial mock to override GenerateReportingAsync - _userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi); + _userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi, _projectCacheService); _userReporting .Configure() .GenerateReportingAsync() @@ -311,7 +312,7 @@ public class UserReportingTests }; // Partial mock - _userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi); + _userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi, _projectCacheService); _userReporting .Configure() .GenerateReportingAsync() diff --git a/src/KpiGenerator/Program.cs b/src/KpiGenerator/Program.cs index cd1df9d..5c1aa63 100644 --- a/src/KpiGenerator/Program.cs +++ b/src/KpiGenerator/Program.cs @@ -180,6 +180,7 @@ public class Program }; services.AddSingleton<IAdminApi>(new AdminApi(apiConfiguration)); services.AddSingleton<IApplicationProfileApi>(new ApplicationProfileApi(apiConfiguration)); + services.AddSingleton<IProjectCacheService, ProjectCacheService>(); services.AddSingleton<IProjectApi>(new ProjectApi(apiConfiguration)); services.AddSingleton<IProjectQuotaApi>(new ProjectQuotaApi(apiConfiguration)); services.AddSingleton<IProjectResourceQuotaApi>(new ProjectResourceQuotaApi(apiConfiguration)); diff --git a/src/KpiGenerator/Reportings/Project/ProjectReporting.cs b/src/KpiGenerator/Reportings/Project/ProjectReporting.cs index da977cb..876926b 100644 --- a/src/KpiGenerator/Reportings/Project/ProjectReporting.cs +++ b/src/KpiGenerator/Reportings/Project/ProjectReporting.cs @@ -21,7 +21,7 @@ public class ProjectReporting private readonly IStorageService _localStorageService; private readonly KpiConfiguration _kpiConfiguration; private readonly ReportingConfiguration _reportingConfiguration; - private readonly IAdminApi _adminApi; + private readonly IProjectCacheService _projectCacheService; private readonly IProjectQuotaApi _projectQuotaApi; public ProjectReportingOptions Options { get; private set; } = null!; @@ -34,7 +34,7 @@ public class ProjectReporting [FromKeyedServices("local")] IStorageService localStorageService, IOptionsMonitor<KpiConfiguration> kpiConfiguration, IOptionsMonitor<ReportingConfiguration> reportingConfiguration, - IAdminApi adminApi, + IProjectCacheService projectCacheService, IProjectQuotaApi projectQuotaApi ) { @@ -46,7 +46,7 @@ public class ProjectReporting _reportingConfiguration = reportingConfiguration.CurrentValue; ReportingFileName = _kpiConfiguration.ProjectKpi.FileName; - _adminApi = adminApi; + _projectCacheService = projectCacheService; _projectQuotaApi = projectQuotaApi; } @@ -84,18 +84,22 @@ public class ProjectReporting public virtual async Task<IEnumerable<ReportingFileObject>> GenerateReportingAsync() { _logger.LogDebug("Working on projects asynchronously..."); - var projects = PaginationHelper.GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>( - (currentPage) => - { - _logger.LogDebug("Getting page {page} of projects...", currentPage); - return _adminApi.GetAllProjectsAsync(includeDeleted: false, includeQuotas: false, includePublicationRequests: true, pageNumber: currentPage, pageSize: 50); - }); + var projects = await _projectCacheService.GetAllProjectsAsync(); + _logger.LogInformation("Found {count} projects.", projects.Count); + if (projects.Count == 0) + { + _logger.LogWarning("No projects found. Exiting project reporting generation."); + return []; + } + // Filter out projects that are deleted + projects = [.. projects.Where(p => !p.Deleted)]; + _logger.LogInformation("Filtered out deleted projects. Remaining projects: {count}", projects.Count); var reportingFiles = new List<ReportingFileObject>(); var returnObjects = new List<ProjectReport>(); // Additional processing - await foreach (var project in projects) + foreach (var project in projects) { _logger.LogDebug("Processing project {projectId}...", project.Id); var quotas = PaginationHelper.GetAllAsync<ProjectQuotaDtoPagedResponse, ProjectQuotaDto>( diff --git a/src/KpiGenerator/Reportings/Resource/ResourceReporting.cs b/src/KpiGenerator/Reportings/Resource/ResourceReporting.cs index f13df44..e2d4cb5 100644 --- a/src/KpiGenerator/Reportings/Resource/ResourceReporting.cs +++ b/src/KpiGenerator/Reportings/Resource/ResourceReporting.cs @@ -23,6 +23,7 @@ public class ResourceReporting private readonly KpiConfiguration _kpiConfiguration; private readonly ReportingConfiguration _reportingConfiguration; private readonly IAdminApi _adminApi; + private readonly IProjectCacheService _projectCacheService; public ResourceReportingOptions Options { get; private set; } = null!; public string ReportingFileName { get; } @@ -34,7 +35,8 @@ public class ResourceReporting [FromKeyedServices("local")] IStorageService localStorageService, IOptionsMonitor<KpiConfiguration> kpiConfiguration, IOptionsMonitor<ReportingConfiguration> reportingConfiguration, - IAdminApi adminApi + IAdminApi adminApi, + IProjectCacheService projectCacheService ) { _mapper = mapper; @@ -45,6 +47,7 @@ public class ResourceReporting _reportingConfiguration = reportingConfiguration.CurrentValue; ReportingFileName = _kpiConfiguration.ResourceKpi.FileName; _adminApi = adminApi; + _projectCacheService = projectCacheService; } public async Task<bool> RunAsync(ResourceReportingOptions reportingOptions) @@ -80,20 +83,15 @@ public class ResourceReporting public virtual async Task<IEnumerable<ReportingFileObject>> GenerateReportingAsync() { _logger.LogDebug("Getting all projects..."); - var projects = await PaginationHelper.GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>( - (currentPage) => - { - _logger.LogDebug("Getting page {page} of projects...", currentPage); - return _adminApi.GetAllProjectsAsync(includeDeleted: true, pageNumber: currentPage, pageSize: 50); - }).ToListAsync(); - _logger.LogDebug("Got all projects."); + var projects = await _projectCacheService.GetAllProjectsAsync(); + _logger.LogDebug("Got all {count} projects, including deleted ones.", projects.Count); _logger.LogDebug("Working on resources asynchronously..."); var resources = PaginationHelper.GetAllAsync<ResourceAdminDtoPagedResponse, ResourceAdminDto>( (currentPage) => { _logger.LogDebug("Getting page {page} of resources...", currentPage); - return _adminApi.GetAllResourcesAsync(includeDeleted: false, includeQuotas: true, pageNumber: currentPage, pageSize: 50); + return _adminApi.GetAllResourcesAsync(includeDeleted: false, includeQuotas: true, pageNumber: currentPage, pageSize: 10); }); var reportingFiles = new List<ReportingFileObject>(); diff --git a/src/KpiGenerator/Reportings/User/UserReporting.cs b/src/KpiGenerator/Reportings/User/UserReporting.cs index c1e8e57..e7a1836 100644 --- a/src/KpiGenerator/Reportings/User/UserReporting.cs +++ b/src/KpiGenerator/Reportings/User/UserReporting.cs @@ -23,6 +23,7 @@ public class UserReporting private readonly ReportingConfiguration _reportingConfiguration; private readonly IAdminApi _adminApi; private readonly IRoleApi _roleApi; + private readonly IProjectCacheService _projectCacheService; public UserReportingOptions Options { get; private set; } = null!; public string ReportingFileName { get; } @@ -35,7 +36,8 @@ public class UserReporting IOptionsMonitor<KpiConfiguration> kpiConfiguration, IOptionsMonitor<ReportingConfiguration> reportingConfiguration, IAdminApi adminApi, - IRoleApi roleApi + IRoleApi roleApi, + IProjectCacheService projectCacheService ) { _mapper = mapper; @@ -48,6 +50,7 @@ public class UserReporting _adminApi = adminApi; _roleApi = roleApi; + _projectCacheService = projectCacheService; } public async Task<bool> RunAsync(UserReportingOptions reportingOptions) @@ -83,9 +86,10 @@ public class UserReporting public virtual async Task<IEnumerable<ReportingFileObject>> GenerateReportingAsync() { _logger.LogDebug("Getting all projects..."); - var projects = await PaginationHelper.GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>( - (currentPage) => _adminApi.GetAllProjectsAsync(includeDeleted: false, pageNumber: currentPage, pageSize: 50)).ToListAsync(); - _logger.LogDebug("Got all projects."); + var projects = await _projectCacheService.GetAllProjectsAsync(); + // Filter out projects that are deleted + projects = [.. projects.Where(p => !p.Deleted)]; + _logger.LogInformation("Filtered out deleted projects. Remaining projects: {count}", projects.Count); _logger.LogDebug("Getting all roles..."); var roles = await PaginationHelper.GetAllAsync<RoleDtoPagedResponse, RoleDto>( @@ -97,7 +101,7 @@ public class UserReporting (currentPage) => { _logger.LogDebug("Getting page {page} of users...", currentPage); - return _adminApi.GetAllUsersAsync(tosAccepted: true, pageNumber: currentPage, pageSize: 25); + return _adminApi.GetAllUsersAsync(tosAccepted: true, pageNumber: currentPage, pageSize: 10); }); var reportingFiles = new List<ReportingFileObject>(); diff --git a/src/KpiGenerator/Utils/ProjectCacheService.cs b/src/KpiGenerator/Utils/ProjectCacheService.cs new file mode 100644 index 0000000..5416492 --- /dev/null +++ b/src/KpiGenerator/Utils/ProjectCacheService.cs @@ -0,0 +1,35 @@ +using Coscine.ApiClient; +using Coscine.ApiClient.Core.Api; +using Coscine.ApiClient.Core.Model; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; + +namespace Coscine.KpiGenerator.Utils; + +public interface IProjectCacheService +{ + Task<List<ProjectAdminDto>> GetAllProjectsAsync(); +} + +public class ProjectCacheService(IAdminApi adminApi, IMemoryCache cache, ILogger<ProjectCacheService> logger) : IProjectCacheService +{ + private const string CacheKey = "AllProjects"; + private readonly IAdminApi _adminApi = adminApi; + private readonly IMemoryCache _cache = cache; + private readonly ILogger<ProjectCacheService> _logger = logger; + + public async Task<List<ProjectAdminDto>> GetAllProjectsAsync() + { + return await _cache.GetOrCreateAsync(CacheKey, async entry => + { + entry.AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(120); + _logger.LogDebug("Fetching all projects from API"); + var list = await PaginationHelper + .GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>( + page => _adminApi.GetAllProjectsAsync(includeDeleted: true, includePublicationRequests: true, pageNumber: page, pageSize: 50)) + .ToListAsync(); + _logger.LogDebug("Cached {Count} projects", list.Count); + return list; + }) ?? []; + } +} -- GitLab