diff --git a/src/KpiGenerator.Tests/ProjectReportingTests.cs b/src/KpiGenerator.Tests/ProjectReportingTests.cs
index 861c6fc046885a1b48d1d0ce3c2c307b8365b53c..5c9240680e8b0f9f0c106af6c4beaf3b3917343a 100644
--- a/src/KpiGenerator.Tests/ProjectReportingTests.cs
+++ b/src/KpiGenerator.Tests/ProjectReportingTests.cs
@@ -25,8 +25,8 @@ public class ProjectReportingTests
private IOptionsMonitor<KpiConfiguration> _kpiConfiguration = null!;
private IOptionsMonitor<ReportingConfiguration> _reportingConfiguration = null!;
- private IAdminApi _adminApi = null!;
-
+ private IProjectQuotaApi _projectQuotaApi = null!;
+ private IProjectCacheService _projectCacheService = null!;
private ProjectReporting _projectReporting = null!; // System Under Test
[SetUp]
@@ -69,7 +69,8 @@ public class ProjectReportingTests
_reportingConfiguration = Substitute.For<IOptionsMonitor<ReportingConfiguration>>();
_reportingConfiguration.CurrentValue.Returns(reportingConfig);
- _adminApi = Substitute.For<IAdminApi>();
+ _projectCacheService = Substitute.For<IProjectCacheService>();
+ _projectQuotaApi = Substitute.For<IProjectQuotaApi>();
}
#region GenerateReportingAsync Tests
@@ -80,21 +81,10 @@ public class ProjectReportingTests
// Arrange
var projects = TestData.ProjectAdminDtos;
- _adminApi
- .GetAllProjectsAsync(
- includeDeleted: Arg.Any<bool>(),
- includeQuotas: Arg.Any<bool>(),
- includePublicationRequests: Arg.Any<bool>(),
- pageNumber: Arg.Any<int>(),
- pageSize: Arg.Any<int>()
- )
- .Returns(ci =>
- {
- // Return the test projects data, single page
- var pagination = new Pagination(currentPage: 1, pageSize: 2, totalCount: 2, totalPages: 1);
- return Task.FromResult(new ProjectAdminDtoPagedResponse(data: projects, pagination: pagination, statusCode: 200, traceId: "dummy-trace-id"));
- });
- _projectReporting = new ProjectReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi);
+ _projectCacheService
+ .GetAllProjectsAsync()
+ .Returns(ci => Task.FromResult(projects));
+ _projectReporting = new ProjectReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _projectCacheService, _projectQuotaApi);
// Act
var result = await _projectReporting.GenerateReportingAsync();
@@ -118,15 +108,10 @@ public class ProjectReportingTests
public async Task GenerateReportingAsync_ReturnsOnlyGeneralFile_WhenNoProjects()
{
// Arrange
- _adminApi
+ _projectCacheService
.GetAllProjectsAsync()
- .Returns(ci =>
- {
- // No projects, empty data
- var pagination = new Pagination(currentPage: 1, pageSize: 0, totalCount: 0, totalPages: 1);
- return Task.FromResult(new ProjectAdminDtoPagedResponse(data: [], pagination: pagination, statusCode: 200, traceId: "dummy-trace-id"));
- });
- _projectReporting = new ProjectReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi);
+ .Returns(ci => Task.FromResult(new List<ProjectAdminDto>()));
+ _projectReporting = new ProjectReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _projectCacheService, _projectQuotaApi);
// Act
var result = await _projectReporting.GenerateReportingAsync();
@@ -153,7 +138,7 @@ public class ProjectReportingTests
};
// We want to ensure that GenerateReportingAsync returns some test objects
- _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi);
+ _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _projectCacheService, _projectQuotaApi);
_projectReporting
.Configure()
.GenerateReportingAsync()
@@ -193,7 +178,7 @@ public class ProjectReportingTests
};
// Partial mock to override GenerateReportingAsync
- _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi);
+ _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _projectCacheService, _projectQuotaApi);
_projectReporting
.Configure()
.GenerateReportingAsync()
@@ -233,7 +218,7 @@ public class ProjectReportingTests
};
// Partial mock to override GenerateReportingAsync
- _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi);
+ _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _projectCacheService, _projectQuotaApi);
_projectReporting
.Configure()
.GenerateReportingAsync()
@@ -270,7 +255,7 @@ public class ProjectReportingTests
};
// Partial mock
- _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi);
+ _projectReporting = Substitute.ForPartsOf<ProjectReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _projectCacheService, _projectQuotaApi);
_projectReporting
.Configure()
.GenerateReportingAsync()
diff --git a/src/KpiGenerator.Tests/ResourceReportingTests.cs b/src/KpiGenerator.Tests/ResourceReportingTests.cs
index ebd6030c7a728c925b138f7a6d380d89ffd8022f..652e4c907baa6f0d875934a4150abe2e63d8a485 100644
--- a/src/KpiGenerator.Tests/ResourceReportingTests.cs
+++ b/src/KpiGenerator.Tests/ResourceReportingTests.cs
@@ -26,7 +26,7 @@ public class ResourceReportingTests
private IOptionsMonitor<ReportingConfiguration> _reportingConfiguration = null!;
private IAdminApi _adminApi = null!;
-
+ private IProjectCacheService _projectCacheService = null!;
private ResourceReporting _resourceReporting = null!; // System Under Test
[SetUp]
@@ -68,8 +68,9 @@ public class ResourceReportingTests
};
_reportingConfiguration = Substitute.For<IOptionsMonitor<ReportingConfiguration>>();
_reportingConfiguration.CurrentValue.Returns(reportingConfig);
-
+
_adminApi = Substitute.For<IAdminApi>();
+ _projectCacheService = Substitute.For<IProjectCacheService>();
}
#region GenerateReportingAsync Tests
@@ -81,18 +82,9 @@ public class ResourceReportingTests
var projects = TestData.ProjectAdminDtos;
var resources = TestData.ResourceAdminDtos;
- _adminApi
- .GetAllProjectsAsync(
- includeDeleted: Arg.Any<bool>(),
- pageNumber: Arg.Any<int>(),
- pageSize: Arg.Any<int>()
- )
- .Returns(ci =>
- {
- // Return the test projects data, single page
- var pagination = new Pagination(currentPage: 1, pageSize: 2, totalCount: 2, totalPages: 1);
- return Task.FromResult(new ProjectAdminDtoPagedResponse(data: projects, pagination: pagination, statusCode: 200, traceId: "dummy-trace-id"));
- });
+ _projectCacheService
+ .GetAllProjectsAsync()
+ .Returns(ci => Task.FromResult(projects));
_adminApi
.GetAllResourcesAsync(
includeDeleted: Arg.Any<bool>(),
@@ -106,7 +98,7 @@ public class ResourceReportingTests
var pagination = new Pagination(currentPage: 1, pageSize: 2, totalCount: 2, totalPages: 1);
return Task.FromResult(new ResourceAdminDtoPagedResponse(data: resources, pagination: pagination, statusCode: 200, traceId: "dummy-trace-id"));
});
- _resourceReporting = new ResourceReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi);
+ _resourceReporting = new ResourceReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectCacheService);
// Act
var result = await _resourceReporting.GenerateReportingAsync();
@@ -130,14 +122,9 @@ public class ResourceReportingTests
public async Task GenerateReportingAsync_ReturnsOnlyGeneralFile_WhenNoProjects()
{
// Arrange
- _adminApi
+ _projectCacheService
.GetAllProjectsAsync()
- .Returns(ci =>
- {
- // No projects, empty data
- var pagination = new Pagination(currentPage: 1, pageSize: 0, totalCount: 0, totalPages: 1);
- return Task.FromResult(new ProjectAdminDtoPagedResponse(data: [], pagination: pagination, statusCode: 200, traceId: "dummy-trace-id"));
- });
+ .Returns(ci => Task.FromResult(new List<ProjectAdminDto>()));
_adminApi
.GetAllResourcesAsync()
.Returns(ci =>
@@ -146,7 +133,7 @@ public class ResourceReportingTests
var pagination = new Pagination(currentPage: 1, pageSize: 0, totalCount: 0, totalPages: 1);
return Task.FromResult(new ResourceAdminDtoPagedResponse(data: [], pagination: pagination, statusCode: 200, traceId: "dummy-trace-id"));
});
- _resourceReporting = new ResourceReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi);
+ _resourceReporting = new ResourceReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectCacheService);
// Act
var result = await _resourceReporting.GenerateReportingAsync();
@@ -173,7 +160,7 @@ public class ResourceReportingTests
};
// We want to ensure that GenerateReportingAsync returns some test objects
- _resourceReporting = Substitute.ForPartsOf<ResourceReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi);
+ _resourceReporting = Substitute.ForPartsOf<ResourceReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectCacheService);
_resourceReporting
.Configure()
.GenerateReportingAsync()
@@ -213,7 +200,7 @@ public class ResourceReportingTests
};
// Partial mock to override GenerateReportingAsync
- _resourceReporting = Substitute.ForPartsOf<ResourceReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi);
+ _resourceReporting = Substitute.ForPartsOf<ResourceReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectCacheService);
_resourceReporting
.Configure()
.GenerateReportingAsync()
@@ -253,7 +240,7 @@ public class ResourceReportingTests
};
// Partial mock to override GenerateReportingAsync
- _resourceReporting = Substitute.ForPartsOf<ResourceReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi);
+ _resourceReporting = Substitute.ForPartsOf<ResourceReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectCacheService);
_resourceReporting
.Configure()
.GenerateReportingAsync()
@@ -290,7 +277,7 @@ public class ResourceReportingTests
};
// Partial mock
- _resourceReporting = Substitute.ForPartsOf<ResourceReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi);
+ _resourceReporting = Substitute.ForPartsOf<ResourceReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _projectCacheService);
_resourceReporting
.Configure()
.GenerateReportingAsync()
diff --git a/src/KpiGenerator.Tests/UserReportingTests.cs b/src/KpiGenerator.Tests/UserReportingTests.cs
index 5386dbd1b25154449f4d568d8308f4e094bb2fef..e17434cb0f22a2c497c4ac3a7428212dfc79f593 100644
--- a/src/KpiGenerator.Tests/UserReportingTests.cs
+++ b/src/KpiGenerator.Tests/UserReportingTests.cs
@@ -27,7 +27,7 @@ public class UserReportingTests
private IAdminApi _adminApi = null!;
private IRoleApi _roleApi = null!;
-
+ private IProjectCacheService _projectCacheService = null!;
private UserReporting _userReporting = null!; // System Under Test
[SetUp]
@@ -72,6 +72,7 @@ public class UserReportingTests
_adminApi = Substitute.For<IAdminApi>();
_roleApi = Substitute.For<IRoleApi>();
+ _projectCacheService = Substitute.For<IProjectCacheService>();
}
#region GenerateReportingAsync Tests
@@ -84,18 +85,9 @@ public class UserReportingTests
var users = TestData.UserDtos;
var roles = TestData.RoleDtos;
- _adminApi
- .GetAllProjectsAsync(
- includeDeleted: Arg.Any<bool>(),
- pageNumber: Arg.Any<int>(),
- pageSize: Arg.Any<int>()
- )
- .Returns(ci =>
- {
- // Return the test projects data, single page
- var pagination = new Pagination(currentPage: 1, pageSize: 2, totalCount: 2, totalPages: 1);
- return Task.FromResult(new ProjectAdminDtoPagedResponse(data: projects, pagination: pagination, statusCode: 200, traceId: "dummy-trace-id"));
- });
+ _projectCacheService
+ .GetAllProjectsAsync()
+ .Returns(ci => Task.FromResult(projects));
_adminApi
.GetAllUsersAsync(
tosAccepted: Arg.Any<bool>(),
@@ -119,7 +111,7 @@ public class UserReportingTests
var pagination = new Pagination(currentPage: 1, pageSize: 2, totalCount: 2, totalPages: 1);
return Task.FromResult(new RoleDtoPagedResponse(data: roles, pagination: pagination, statusCode: 200, traceId: "dummy-trace-id"));
});
- _userReporting = new UserReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi);
+ _userReporting = new UserReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi, _projectCacheService);
// Act
var result = await _userReporting.GenerateReportingAsync();
@@ -143,14 +135,9 @@ public class UserReportingTests
public async Task GenerateReportingAsync_ReturnsOnlyGeneralFile_WhenNoProjects()
{
// Arrange
- _adminApi
+ _projectCacheService
.GetAllProjectsAsync()
- .Returns(ci =>
- {
- // No projects, empty data
- var pagination = new Pagination(currentPage: 1, pageSize: 0, totalCount: 0, totalPages: 1);
- return Task.FromResult(new ProjectAdminDtoPagedResponse(data: [], pagination: pagination, statusCode: 200, traceId: "dummy-trace-id"));
- });
+ .Returns(ci => Task.FromResult(new List<ProjectAdminDto>()));
_adminApi
.GetAllUsersAsync()
.Returns(ci =>
@@ -167,7 +154,7 @@ public class UserReportingTests
var pagination = new Pagination(currentPage: 1, pageSize: 0, totalCount: 0, totalPages: 1);
return Task.FromResult(new RoleDtoPagedResponse(data: [], pagination: pagination, statusCode: 200, traceId: "dummy-trace-id"));
});
- _userReporting = new UserReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi);
+ _userReporting = new UserReporting(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi, _projectCacheService);
// Act
var result = await _userReporting.GenerateReportingAsync();
@@ -194,7 +181,7 @@ public class UserReportingTests
};
// We want to ensure that GenerateReportingAsync returns some test objects
- _userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi);
+ _userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi, _projectCacheService);
_userReporting
.Configure()
.GenerateReportingAsync()
@@ -234,7 +221,7 @@ public class UserReportingTests
};
// Partial mock to override GenerateReportingAsync
- _userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi);
+ _userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi, _projectCacheService);
_userReporting
.Configure()
.GenerateReportingAsync()
@@ -274,7 +261,7 @@ public class UserReportingTests
};
// Partial mock to override GenerateReportingAsync
- _userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi);
+ _userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi, _projectCacheService);
_userReporting
.Configure()
.GenerateReportingAsync()
@@ -311,7 +298,7 @@ public class UserReportingTests
};
// Partial mock
- _userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi);
+ _userReporting = Substitute.ForPartsOf<UserReporting>(_mapper, _logger, _gitlabStorageService, _localStorageService, _kpiConfiguration, _reportingConfiguration, _adminApi, _roleApi, _projectCacheService);
_userReporting
.Configure()
.GenerateReportingAsync()
diff --git a/src/KpiGenerator/KpiGenerator.csproj b/src/KpiGenerator/KpiGenerator.csproj
index c3049884b6b06f479f47aa56b123d9f6d6304911..bc810e6d91d17f9020f35692816848df1a432215 100644
--- a/src/KpiGenerator/KpiGenerator.csproj
+++ b/src/KpiGenerator/KpiGenerator.csproj
@@ -21,7 +21,7 @@
<PackageReference Include="AutoMapper" Version="12.0.1" />
<PackageReference Include="AutoMapper.Extensions.Microsoft.DependencyInjection" Version="12.0.1" />
<PackageReference Include="CommandLineParser" Version="2.9.1" />
- <PackageReference Include="Coscine.ApiClient" Version="1.9.9" />
+ <PackageReference Include="Coscine.ApiClient" Version="1.10.0" />
<PackageReference Include="dotNetRdf.Core" Version="3.1.1" />
<PackageReference Include="GitLabApiClient" Version="1.8.1-beta.5" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.0" />
diff --git a/src/KpiGenerator/Program.cs b/src/KpiGenerator/Program.cs
index 55e66793b76afaaeea3961f678a21b01d3e89561..5c1aa636da821e4613ba998f391b3f30d7b5c3e9 100644
--- a/src/KpiGenerator/Program.cs
+++ b/src/KpiGenerator/Program.cs
@@ -176,10 +176,11 @@ public class Program
BasePath = $"{reportingConfiguration.Endpoint.TrimEnd('/')}/coscine",
ApiKeyPrefix = { { "Authorization", "Bearer" } },
ApiKey = { { "Authorization", reportingConfiguration.ApiKey } },
- Timeout = TimeSpan.FromSeconds(300) // 5 minutes
+ Timeout = TimeSpan.FromSeconds(300), // 5 minutes
};
services.AddSingleton<IAdminApi>(new AdminApi(apiConfiguration));
services.AddSingleton<IApplicationProfileApi>(new ApplicationProfileApi(apiConfiguration));
+ services.AddSingleton<IProjectCacheService, ProjectCacheService>();
services.AddSingleton<IProjectApi>(new ProjectApi(apiConfiguration));
services.AddSingleton<IProjectQuotaApi>(new ProjectQuotaApi(apiConfiguration));
services.AddSingleton<IProjectResourceQuotaApi>(new ProjectResourceQuotaApi(apiConfiguration));
diff --git a/src/KpiGenerator/Reportings/Project/ProjectReporting.cs b/src/KpiGenerator/Reportings/Project/ProjectReporting.cs
index e8bee5ee1d5235deb95e48cac58a5b1ed03fe0da..08787f782bf484caba645a5e62d35d392cb6fdd9 100644
--- a/src/KpiGenerator/Reportings/Project/ProjectReporting.cs
+++ b/src/KpiGenerator/Reportings/Project/ProjectReporting.cs
@@ -21,7 +21,8 @@ public class ProjectReporting
private readonly IStorageService _localStorageService;
private readonly KpiConfiguration _kpiConfiguration;
private readonly ReportingConfiguration _reportingConfiguration;
- private readonly IAdminApi _adminApi;
+ private readonly IProjectCacheService _projectCacheService;
+ private readonly IProjectQuotaApi _projectQuotaApi;
public ProjectReportingOptions Options { get; private set; } = null!;
public string ReportingFileName { get; }
@@ -33,7 +34,8 @@ public class ProjectReporting
[FromKeyedServices("local")] IStorageService localStorageService,
IOptionsMonitor<KpiConfiguration> kpiConfiguration,
IOptionsMonitor<ReportingConfiguration> reportingConfiguration,
- IAdminApi adminApi
+ IProjectCacheService projectCacheService,
+ IProjectQuotaApi projectQuotaApi
)
{
_mapper = mapper;
@@ -44,7 +46,8 @@ public class ProjectReporting
_reportingConfiguration = reportingConfiguration.CurrentValue;
ReportingFileName = _kpiConfiguration.ProjectKpi.FileName;
- _adminApi = adminApi;
+ _projectCacheService = projectCacheService;
+ _projectQuotaApi = projectQuotaApi;
}
public async Task<bool> RunAsync(ProjectReportingOptions reportingOptions)
@@ -81,16 +84,36 @@ public class ProjectReporting
public virtual async Task<IEnumerable<ReportingFileObject>> GenerateReportingAsync()
{
_logger.LogDebug("Working on projects asynchronously...");
- var projects = PaginationHelper.GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>(
- (currentPage) => _adminApi.GetAllProjectsAsync(includeDeleted: false, includeQuotas: true, includePublicationRequests: true, pageNumber: currentPage, pageSize: 50));
+ var projects = await _projectCacheService.GetAllProjectsAsync();
+ _logger.LogInformation("Found {count} projects.", projects.Count);
+
+ // Filter out projects that are deleted
+ projects = [.. projects.Where(p => !p.Deleted)];
+ _logger.LogInformation("Filtered out deleted projects. Remaining projects: {count}", projects.Count);
var reportingFiles = new List<ReportingFileObject>();
var returnObjects = new List<ProjectReport>();
// Additional processing
- await foreach (var project in projects)
+ foreach (var project in projects)
{
_logger.LogDebug("Processing project {projectId}...", project.Id);
+ var quotas = PaginationHelper.GetAllAsync<ProjectQuotaDtoPagedResponse, ProjectQuotaDto>(
+ (currentPage) =>
+ {
+ _logger.LogDebug("Getting page {page} of quotas for project {projectId}...", currentPage, project.Id);
+ return _projectQuotaApi.GetProjectQuotasAsync(project.Id.ToString(), pageNumber: currentPage, pageSize: 50);
+ });
+ await foreach (var quota in quotas)
+ {
+ if (quota == null)
+ {
+ _logger.LogWarning("Quota for project {projectId} is null, skipping...", project.Id);
+ continue;
+ }
+ // Map the quota to the project
+ project.ProjectQuota.Add(quota);
+ }
var returnObject = _mapper.Map<ProjectReport>(project);
returnObjects.Add(returnObject);
}
diff --git a/src/KpiGenerator/Reportings/Resource/ResourceReporting.cs b/src/KpiGenerator/Reportings/Resource/ResourceReporting.cs
index 48fb60b78c12173a504a2c19233f4601be17f7be..e2d4cb552b46682dc3ef31536d84d528ede2d3a5 100644
--- a/src/KpiGenerator/Reportings/Resource/ResourceReporting.cs
+++ b/src/KpiGenerator/Reportings/Resource/ResourceReporting.cs
@@ -23,6 +23,7 @@ public class ResourceReporting
private readonly KpiConfiguration _kpiConfiguration;
private readonly ReportingConfiguration _reportingConfiguration;
private readonly IAdminApi _adminApi;
+ private readonly IProjectCacheService _projectCacheService;
public ResourceReportingOptions Options { get; private set; } = null!;
public string ReportingFileName { get; }
@@ -34,7 +35,8 @@ public class ResourceReporting
[FromKeyedServices("local")] IStorageService localStorageService,
IOptionsMonitor<KpiConfiguration> kpiConfiguration,
IOptionsMonitor<ReportingConfiguration> reportingConfiguration,
- IAdminApi adminApi
+ IAdminApi adminApi,
+ IProjectCacheService projectCacheService
)
{
_mapper = mapper;
@@ -44,16 +46,8 @@ public class ResourceReporting
_kpiConfiguration = kpiConfiguration.CurrentValue;
_reportingConfiguration = reportingConfiguration.CurrentValue;
ReportingFileName = _kpiConfiguration.ResourceKpi.FileName;
-
- var configuration = new Configuration()
- {
- BasePath = $"{_reportingConfiguration.Endpoint.TrimEnd('/')}/coscine",
- ApiKeyPrefix = { { "Authorization", "Bearer" } },
- ApiKey = { { "Authorization", _reportingConfiguration.ApiKey } },
- Timeout = TimeSpan.FromSeconds(300) // 5 minutes
- };
-
_adminApi = adminApi;
+ _projectCacheService = projectCacheService;
}
public async Task<bool> RunAsync(ResourceReportingOptions reportingOptions)
@@ -89,13 +83,16 @@ public class ResourceReporting
public virtual async Task<IEnumerable<ReportingFileObject>> GenerateReportingAsync()
{
_logger.LogDebug("Getting all projects...");
- var projects = await PaginationHelper.GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>(
- (currentPage) => _adminApi.GetAllProjectsAsync(includeDeleted: true, pageNumber: currentPage, pageSize: 50)).ToListAsync();
- _logger.LogDebug("Got all projects.");
+ var projects = await _projectCacheService.GetAllProjectsAsync();
+ _logger.LogDebug("Got all {count} projects, including deleted ones.", projects.Count);
_logger.LogDebug("Working on resources asynchronously...");
var resources = PaginationHelper.GetAllAsync<ResourceAdminDtoPagedResponse, ResourceAdminDto>(
- (currentPage) => _adminApi.GetAllResourcesAsync(includeDeleted: false, includeQuotas: true, pageNumber: currentPage, pageSize: 50));
+ (currentPage) =>
+ {
+ _logger.LogDebug("Getting page {page} of resources...", currentPage);
+ return _adminApi.GetAllResourcesAsync(includeDeleted: false, includeQuotas: true, pageNumber: currentPage, pageSize: 10);
+ });
var reportingFiles = new List<ReportingFileObject>();
var returnObjects = new List<ResourceReport>();
diff --git a/src/KpiGenerator/Reportings/User/UserReporting.cs b/src/KpiGenerator/Reportings/User/UserReporting.cs
index 8db2ad4d1e6c4720e521f501d80a401e2d6bbda6..e7a18367a50360299395628b3e6115e0965e5249 100644
--- a/src/KpiGenerator/Reportings/User/UserReporting.cs
+++ b/src/KpiGenerator/Reportings/User/UserReporting.cs
@@ -23,6 +23,7 @@ public class UserReporting
private readonly ReportingConfiguration _reportingConfiguration;
private readonly IAdminApi _adminApi;
private readonly IRoleApi _roleApi;
+ private readonly IProjectCacheService _projectCacheService;
public UserReportingOptions Options { get; private set; } = null!;
public string ReportingFileName { get; }
@@ -35,7 +36,8 @@ public class UserReporting
IOptionsMonitor<KpiConfiguration> kpiConfiguration,
IOptionsMonitor<ReportingConfiguration> reportingConfiguration,
IAdminApi adminApi,
- IRoleApi roleApi
+ IRoleApi roleApi,
+ IProjectCacheService projectCacheService
)
{
_mapper = mapper;
@@ -48,6 +50,7 @@ public class UserReporting
_adminApi = adminApi;
_roleApi = roleApi;
+ _projectCacheService = projectCacheService;
}
public async Task<bool> RunAsync(UserReportingOptions reportingOptions)
@@ -83,9 +86,10 @@ public class UserReporting
public virtual async Task<IEnumerable<ReportingFileObject>> GenerateReportingAsync()
{
_logger.LogDebug("Getting all projects...");
- var projects = await PaginationHelper.GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>(
- (currentPage) => _adminApi.GetAllProjectsAsync(includeDeleted: false, pageNumber: currentPage, pageSize: 50)).ToListAsync();
- _logger.LogDebug("Got all projects.");
+ var projects = await _projectCacheService.GetAllProjectsAsync();
+ // Filter out projects that are deleted
+ projects = [.. projects.Where(p => !p.Deleted)];
+ _logger.LogInformation("Filtered out deleted projects. Remaining projects: {count}", projects.Count);
_logger.LogDebug("Getting all roles...");
var roles = await PaginationHelper.GetAllAsync<RoleDtoPagedResponse, RoleDto>(
@@ -94,7 +98,11 @@ public class UserReporting
_logger.LogDebug("Working on users asynchronously...");
var users = PaginationHelper.GetAllAsync<UserDtoPagedResponse, UserDto>(
- (currentPage) => _adminApi.GetAllUsersAsync(tosAccepted: true, pageNumber: currentPage, pageSize: 50));
+ (currentPage) =>
+ {
+ _logger.LogDebug("Getting page {page} of users...", currentPage);
+ return _adminApi.GetAllUsersAsync(tosAccepted: true, pageNumber: currentPage, pageSize: 10);
+ });
var reportingFiles = new List<ReportingFileObject>();
var returnObjects = new List<UserReport>();
diff --git a/src/KpiGenerator/Utils/ProjectCacheService.cs b/src/KpiGenerator/Utils/ProjectCacheService.cs
new file mode 100644
index 0000000000000000000000000000000000000000..5416492f7cde8bdac9cf64f4a2fde71156422b4e
--- /dev/null
+++ b/src/KpiGenerator/Utils/ProjectCacheService.cs
@@ -0,0 +1,35 @@
+using Coscine.ApiClient;
+using Coscine.ApiClient.Core.Api;
+using Coscine.ApiClient.Core.Model;
+using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.Logging;
+
+namespace Coscine.KpiGenerator.Utils;
+
+public interface IProjectCacheService
+{
+ Task<List<ProjectAdminDto>> GetAllProjectsAsync();
+}
+
+public class ProjectCacheService(IAdminApi adminApi, IMemoryCache cache, ILogger<ProjectCacheService> logger) : IProjectCacheService
+{
+ private const string CacheKey = "AllProjects";
+ private readonly IAdminApi _adminApi = adminApi;
+ private readonly IMemoryCache _cache = cache;
+ private readonly ILogger<ProjectCacheService> _logger = logger;
+
+ public async Task<List<ProjectAdminDto>> GetAllProjectsAsync()
+ {
+ return await _cache.GetOrCreateAsync(CacheKey, async entry =>
+ {
+ entry.AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(120);
+ _logger.LogDebug("Fetching all projects from API");
+ var list = await PaginationHelper
+ .GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>(
+ page => _adminApi.GetAllProjectsAsync(includeDeleted: true, includePublicationRequests: true, pageNumber: page, pageSize: 50))
+ .ToListAsync();
+ _logger.LogDebug("Cached {Count} projects", list.Count);
+ return list;
+ }) ?? [];
+ }
+}