Skip to content
Snippets Groups Projects
Commit 40dc29cc authored by Petar Hristov's avatar Petar Hristov :speech_balloon:
Browse files

Update: Introduce ProjectCacheService for project retrieval and update reporting classes to use it

parent 9637bdd8
No related branches found
No related tags found
1 merge request!46Fix: ProjectReporting to fetch project quotas individually and update API
Pipeline #1721017 failed
Stage: build
Stage: test
Stage: publish
Show whitespace changes
Inline Side-by-side
Showing
with 87 additions and 61 deletions
src/KpiGenerator.Tests/ProjectReportingTests.cs
+ 13
− 30
View file @ 40dc29cc
......@@ -25,9 +25,8 @@ public class ProjectReportingTests
private IOptionsMonitor<KpiConfiguration> _kpiConfiguration = null!;
private IOptionsMonitor<ReportingConfiguration> _reportingConfiguration = null!;
private IAdminApi _adminApi = null!;
private IProjectQuotaApi _projectQuotaApi = null!;
private IProjectCacheService _projectCacheService = null!;
private ProjectReporting _projectReporting = null!; // System Under Test
[SetUp]
......@@ -70,7 +69,7 @@ public class ProjectReportingTests
_reportingConfiguration = Substitute.For<IOptionsMonitor<ReportingConfiguration>>();
_reportingConfiguration.CurrentValue.Returns(reportingConfig);
_adminApi = Substitute.For<IAdminApi>();
_projectCacheService = Substitute.For<IProjectCacheService>();
_projectQuotaApi = Substitute.For<IProjectQuotaApi>();
}
......@@ -82,21 +81,10 @@ public class ProjectReportingTests
// Arrange
var projects = TestData.ProjectAdminDtos;
_adminApi
.GetAllProjectsAsync(
includeDeleted: Arg.Any<bool>(),
includeQuotas: Arg.Any<bool>(),
includePublicationRequests: Arg.Any<bool>(),
pageNumber: Arg.Any<int>(),
pageSize: Arg.Any<int>()
)
.Returns(ci =>
{
// Return the test projects data, single page
var pagination = new Pagination(currentPage: 1, pageSize: 2, totalCount: 2, totalPages: 1);
return Task.FromResult(new ProjectAdminDtoPagedResponse(data: projects, pagination: pagination, statusCode: 200, traceId: "dummy-trace-id"));
});
_projectReporting = new ProjectReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectQuotaApi);
_projectCacheService
.GetAllProjectsAsync()
.Returns(ci => Task.FromResult(projects));
_projectReporting = new ProjectReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _projectCacheService, _projectQuotaApi);
// Act
var result = await _projectReporting.GenerateReportingAsync();
......@@ -120,15 +108,10 @@ public class ProjectReportingTests
public async Task GenerateReportingAsync_ReturnsOnlyGeneralFile_WhenNoProjects()
{
// Arrange
_adminApi
_projectCacheService
.GetAllProjectsAsync()
.Returns(ci =>
{
// No projects, empty data
var pagination = new Pagination(currentPage: 1, pageSize: 0, totalCount: 0, totalPages: 1);
return Task.FromResult(new ProjectAdminDtoPagedResponse(data: [], pagination: pagination, statusCode: 200, traceId: "dummy-trace-id"));
});
_projectReporting = new ProjectReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectQuotaApi);
.Returns(ci => Task.FromResult(new List<ProjectAdminDto>()));
_projectReporting = new ProjectReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _projectCacheService, _projectQuotaApi);
// Act
var result = await _projectReporting.GenerateReportingAsync();
......@@ -155,7 +138,7 @@ public class ProjectReportingTests
};
// We want to ensure that GenerateReportingAsync returns some test objects
_projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectQuotaApi);
_projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _projectCacheService, _projectQuotaApi);
_projectReporting
.Configure()
.GenerateReportingAsync()
......@@ -195,7 +178,7 @@ public class ProjectReportingTests
};
// Partial mock to override GenerateReportingAsync
_projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectQuotaApi);
_projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _projectCacheService, _projectQuotaApi);
_projectReporting
.Configure()
.GenerateReportingAsync()
......@@ -235,7 +218,7 @@ public class ProjectReportingTests
};
// Partial mock to override GenerateReportingAsync
_projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectQuotaApi);
_projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _projectCacheService, _projectQuotaApi);
_projectReporting
.Configure()
.GenerateReportingAsync()
......@@ -272,7 +255,7 @@ public class ProjectReportingTests
};
// Partial mock
_projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectQuotaApi);
_projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _projectCacheService, _projectQuotaApi);
_projectReporting
.Configure()
.GenerateReportingAsync()
......
src/KpiGenerator.Tests/UserReportingTests.cs
+ 8
− 7
View file @ 40dc29cc
......@@ -27,7 +27,7 @@ public class UserReportingTests
private IAdminApi _adminApi = null!;
private IRoleApi _roleApi = null!;
private IProjectCacheService _projectCacheService = null!;
private UserReporting _userReporting = null!; // System Under Test
[SetUp]
......@@ -72,6 +72,7 @@ public class UserReportingTests
_adminApi = Substitute.For<IAdminApi>();
_roleApi = Substitute.For<IRoleApi>();
_projectCacheService = Substitute.For<IProjectCacheService>();
}
#region GenerateReportingAsync Tests
......@@ -119,7 +120,7 @@ public class UserReportingTests
var pagination = new Pagination(currentPage: 1, pageSize: 2, totalCount: 2, totalPages: 1);
return Task.FromResult(new RoleDtoPagedResponse(data: roles, pagination: pagination, statusCode: 200, traceId: "dummy-trace-id"));
});
_userReporting = new UserReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi);
_userReporting = new UserReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi, _projectCacheService);
// Act
var result = await _userReporting.GenerateReportingAsync();
......@@ -167,7 +168,7 @@ public class UserReportingTests
var pagination = new Pagination(currentPage: 1, pageSize: 0, totalCount: 0, totalPages: 1);
return Task.FromResult(new RoleDtoPagedResponse(data: [], pagination: pagination, statusCode: 200, traceId: "dummy-trace-id"));
});
_userReporting = new UserReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi);
_userReporting = new UserReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi, _projectCacheService);
// Act
var result = await _userReporting.GenerateReportingAsync();
......@@ -194,7 +195,7 @@ public class UserReportingTests
};
// We want to ensure that GenerateReportingAsync returns some test objects
_userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi);
_userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi, _projectCacheService);
_userReporting
.Configure()
.GenerateReportingAsync()
......@@ -234,7 +235,7 @@ public class UserReportingTests
};
// Partial mock to override GenerateReportingAsync
_userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi);
_userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi, _projectCacheService);
_userReporting
.Configure()
.GenerateReportingAsync()
......@@ -274,7 +275,7 @@ public class UserReportingTests
};
// Partial mock to override GenerateReportingAsync
_userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi);
_userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi, _projectCacheService);
_userReporting
.Configure()
.GenerateReportingAsync()
......@@ -311,7 +312,7 @@ public class UserReportingTests
};
// Partial mock
_userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi);
_userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi, _projectCacheService);
_userReporting
.Configure()
.GenerateReportingAsync()
......
src/KpiGenerator/Program.cs
+ 1
− 0
View file @ 40dc29cc
......@@ -180,6 +180,7 @@ public class Program
};
services.AddSingleton<IAdminApi>(new AdminApi(apiConfiguration));
services.AddSingleton<IApplicationProfileApi>(new ApplicationProfileApi(apiConfiguration));
services.AddSingleton<IProjectCacheService, ProjectCacheService>();
services.AddSingleton<IProjectApi>(new ProjectApi(apiConfiguration));
services.AddSingleton<IProjectQuotaApi>(new ProjectQuotaApi(apiConfiguration));
services.AddSingleton<IProjectResourceQuotaApi>(new ProjectResourceQuotaApi(apiConfiguration));
......
src/KpiGenerator/Reportings/Project/ProjectReporting.cs
+ 14
− 10
View file @ 40dc29cc
......@@ -21,7 +21,7 @@ public class ProjectReporting
private readonly IStorageService _localStorageService;
private readonly KpiConfiguration _kpiConfiguration;
private readonly ReportingConfiguration _reportingConfiguration;
private readonly IAdminApi _adminApi;
private readonly IProjectCacheService _projectCacheService;
private readonly IProjectQuotaApi _projectQuotaApi;
public ProjectReportingOptions Options { get; private set; } = null!;
......@@ -34,7 +34,7 @@ public class ProjectReporting
[FromKeyedServices("local")] IStorageService localStorageService,
IOptionsMonitor<KpiConfiguration> kpiConfiguration,
IOptionsMonitor<ReportingConfiguration> reportingConfiguration,
IAdminApi adminApi,
IProjectCacheService projectCacheService,
IProjectQuotaApi projectQuotaApi
)
{
......@@ -46,7 +46,7 @@ public class ProjectReporting
_reportingConfiguration = reportingConfiguration.CurrentValue;
ReportingFileName = _kpiConfiguration.ProjectKpi.FileName;
_adminApi = adminApi;
_projectCacheService = projectCacheService;
_projectQuotaApi = projectQuotaApi;
}
......@@ -84,18 +84,22 @@ public class ProjectReporting
public virtual async Task<IEnumerable<ReportingFileObject>> GenerateReportingAsync()
{
_logger.LogDebug("Working on projects asynchronously...");
var projects = PaginationHelper.GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>(
(currentPage) =>
var projects = await _projectCacheService.GetAllProjectsAsync();
_logger.LogInformation("Found {count} projects.", projects.Count);
if (projects.Count == 0)
{
_logger.LogDebug("Getting page {page} of projects...", currentPage);
return _adminApi.GetAllProjectsAsync(includeDeleted: false, includeQuotas: false, includePublicationRequests: true, pageNumber: currentPage, pageSize: 50);
});
_logger.LogWarning("No projects found. Exiting project reporting generation.");
return [];
}
// Filter out projects that are deleted
projects = [.. projects.Where(p => !p.Deleted)];
_logger.LogInformation("Filtered out deleted projects. Remaining projects: {count}", projects.Count);
var reportingFiles = new List<ReportingFileObject>();
var returnObjects = new List<ProjectReport>();
// Additional processing
await foreach (var project in projects)
foreach (var project in projects)
{
_logger.LogDebug("Processing project {projectId}...", project.Id);
var quotas = PaginationHelper.GetAllAsync<ProjectQuotaDtoPagedResponse, ProjectQuotaDto>(
......
src/KpiGenerator/Reportings/Resource/ResourceReporting.cs
+ 7
− 9
View file @ 40dc29cc
......@@ -23,6 +23,7 @@ public class ResourceReporting
private readonly KpiConfiguration _kpiConfiguration;
private readonly ReportingConfiguration _reportingConfiguration;
private readonly IAdminApi _adminApi;
private readonly IProjectCacheService _projectCacheService;
public ResourceReportingOptions Options { get; private set; } = null!;
public string ReportingFileName { get; }
......@@ -34,7 +35,8 @@ public class ResourceReporting
[FromKeyedServices("local")] IStorageService localStorageService,
IOptionsMonitor<KpiConfiguration> kpiConfiguration,
IOptionsMonitor<ReportingConfiguration> reportingConfiguration,
IAdminApi adminApi
IAdminApi adminApi,
IProjectCacheService projectCacheService
)
{
_mapper = mapper;
......@@ -45,6 +47,7 @@ public class ResourceReporting
_reportingConfiguration = reportingConfiguration.CurrentValue;
ReportingFileName = _kpiConfiguration.ResourceKpi.FileName;
_adminApi = adminApi;
_projectCacheService = projectCacheService;
}
public async Task<bool> RunAsync(ResourceReportingOptions reportingOptions)
......@@ -80,20 +83,15 @@ public class ResourceReporting
public virtual async Task<IEnumerable<ReportingFileObject>> GenerateReportingAsync()
{
_logger.LogDebug("Getting all projects...");
var projects = await PaginationHelper.GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>(
(currentPage) =>
{
_logger.LogDebug("Getting page {page} of projects...", currentPage);
return _adminApi.GetAllProjectsAsync(includeDeleted: true, pageNumber: currentPage, pageSize: 50);
}).ToListAsync();
_logger.LogDebug("Got all projects.");
var projects = await _projectCacheService.GetAllProjectsAsync();
_logger.LogDebug("Got all {count} projects, including deleted ones.", projects.Count);
_logger.LogDebug("Working on resources asynchronously...");
var resources = PaginationHelper.GetAllAsync<ResourceAdminDtoPagedResponse, ResourceAdminDto>(
(currentPage) =>
{
_logger.LogDebug("Getting page {page} of resources...", currentPage);
return _adminApi.GetAllResourcesAsync(includeDeleted: false, includeQuotas: true, pageNumber: currentPage, pageSize: 50);
return _adminApi.GetAllResourcesAsync(includeDeleted: false, includeQuotas: true, pageNumber: currentPage, pageSize: 10);
});
var reportingFiles = new List<ReportingFileObject>();
......
src/KpiGenerator/Reportings/User/UserReporting.cs
+ 9
− 5
View file @ 40dc29cc
......@@ -23,6 +23,7 @@ public class UserReporting
private readonly ReportingConfiguration _reportingConfiguration;
private readonly IAdminApi _adminApi;
private readonly IRoleApi _roleApi;
private readonly IProjectCacheService _projectCacheService;
public UserReportingOptions Options { get; private set; } = null!;
public string ReportingFileName { get; }
......@@ -35,7 +36,8 @@ public class UserReporting
IOptionsMonitor<KpiConfiguration> kpiConfiguration,
IOptionsMonitor<ReportingConfiguration> reportingConfiguration,
IAdminApi adminApi,
IRoleApi roleApi
IRoleApi roleApi,
IProjectCacheService projectCacheService
)
{
_mapper = mapper;
......@@ -48,6 +50,7 @@ public class UserReporting
_adminApi = adminApi;
_roleApi = roleApi;
_projectCacheService = projectCacheService;
}
public async Task<bool> RunAsync(UserReportingOptions reportingOptions)
......@@ -83,9 +86,10 @@ public class UserReporting
public virtual async Task<IEnumerable<ReportingFileObject>> GenerateReportingAsync()
{
_logger.LogDebug("Getting all projects...");
var projects = await PaginationHelper.GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>(
(currentPage) => _adminApi.GetAllProjectsAsync(includeDeleted: false, pageNumber: currentPage, pageSize: 50)).ToListAsync();
_logger.LogDebug("Got all projects.");
var projects = await _projectCacheService.GetAllProjectsAsync();
// Filter out projects that are deleted
projects = [.. projects.Where(p => !p.Deleted)];
_logger.LogInformation("Filtered out deleted projects. Remaining projects: {count}", projects.Count);
_logger.LogDebug("Getting all roles...");
var roles = await PaginationHelper.GetAllAsync<RoleDtoPagedResponse, RoleDto>(
......@@ -97,7 +101,7 @@ public class UserReporting
(currentPage) =>
{
_logger.LogDebug("Getting page {page} of users...", currentPage);
return _adminApi.GetAllUsersAsync(tosAccepted: true, pageNumber: currentPage, pageSize: 25);
return _adminApi.GetAllUsersAsync(tosAccepted: true, pageNumber: currentPage, pageSize: 10);
});
var reportingFiles = new List<ReportingFileObject>();
......
src/KpiGenerator/Utils/ProjectCacheService.cs 0 → 100644
+ 35
− 0
View file @ 40dc29cc
using Coscine.ApiClient;
using Coscine.ApiClient.Core.Api;
using Coscine.ApiClient.Core.Model;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
namespace Coscine.KpiGenerator.Utils;
public interface IProjectCacheService
{
Task<List<ProjectAdminDto>> GetAllProjectsAsync();
}
public class ProjectCacheService(IAdminApi adminApi, IMemoryCache cache, ILogger<ProjectCacheService> logger) : IProjectCacheService
{
private const string CacheKey = "AllProjects";
private readonly IAdminApi _adminApi = adminApi;
private readonly IMemoryCache _cache = cache;
private readonly ILogger<ProjectCacheService> _logger = logger;
public async Task<List<ProjectAdminDto>> GetAllProjectsAsync()
{
return await _cache.GetOrCreateAsync(CacheKey, async entry =>
{
entry.AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(120);
_logger.LogDebug("Fetching all projects from API");
var list = await PaginationHelper
.GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>(
page => _adminApi.GetAllProjectsAsync(includeDeleted: true, includePublicationRequests: true, pageNumber: page, pageSize: 50))
.ToListAsync();
_logger.LogDebug("Cached {Count} projects", list.Count);
return list;
}) ?? [];
}
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment