Skip to main content
Explore
Sign in
Explore
Snippets Groups Projects
Commit 27eff841 authored by Petar Hristov's avatar Petar Hristov :speech_balloon:
Browse files

Fix: Removed Cache and fixed resource reporting for deleted projects

parent 42af8511
No related tags found
1 merge request!37Update: Using AsyncEnums for processing entities and utilising it in the memory cache
Pipeline #1301796 passed
Stage: build
Show whitespace changes
Inline Side-by-side
Showing
with 10 additions and 129 deletions
src/KPI Generator/Program.cs
+ 0
− 3
View file @ 27eff841
......@@ -132,9 +132,6 @@ public class Program
// Register the in-memory cache service
services.AddMemoryCache();
// Register your caching service
services.AddSingleton<ICacheService, MemoryCacheService>();
// Register the storage services
services.AddKeyedSingleton<IStorageService, GitLabStorageService>("gitlab");
services.AddKeyedSingleton<IStorageService, FileSystemStorageService>("local");
......
......
src/KPI Generator/Reportings/Complete/CompleteReporting.cs
+ 1
− 0
View file @ 27eff841
......@@ -64,6 +64,7 @@ public class CompleteReporting
{
var reportingFiles = new List<ReportingFileObject>();
var projectReportingFiles = await _projectReporting.GenerateReportingAsync();
_logger.LogInformation("Project Reporting generated successfully.\n\n");
var resourceReportingFiles = await _resourceReporting.GenerateReportingAsync();
......
......
src/KPI Generator/Reportings/Project/ProjectReporting.cs
+ 3
− 6
View file @ 27eff841
......@@ -17,7 +17,6 @@ namespace Coscine.KpiGenerator.Reportings.Project;
public class ProjectReporting
{
private readonly IMapper _mapper;
private readonly ICacheService _cacheService;
private readonly ILogger<ProjectReporting> _logger;
private readonly IStorageService _gitlabStorageService;
private readonly IStorageService _localStorageService;
......@@ -33,7 +32,6 @@ public class ProjectReporting
public ProjectReporting(
IMapper mapper,
ICacheService cacheService,
ILogger<ProjectReporting> logger,
[FromKeyedServices("gitlab")] IStorageService gitlabStorageService,
[FromKeyedServices("local")] IStorageService localStorageService,
......@@ -42,7 +40,6 @@ public class ProjectReporting
)
{
_mapper = mapper;
_cacheService = cacheService;
_logger = logger;
_gitlabStorageService = gitlabStorageService;
_localStorageService = localStorageService;
......@@ -66,6 +63,7 @@ public class ProjectReporting
{
_logger.LogInformation("Generating Project Reporting...");
Options = reportingOptions;
var reportingFiles = await GenerateReportingAsync();
_logger.LogInformation("Publishing to GitLab...");
......@@ -82,9 +80,8 @@ public class ProjectReporting
public async Task<IEnumerable<ReportingFileObject>> GenerateReportingAsync()
{
// Use the cache service to get the projects
var projects = _cacheService.GetOrAddAsync(MemoryCacheService.ProjectsCacheKey,
() => PaginationHelper.GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>(
(currentPage) => AdminApi.GetAllProjectsAsync(includeDeleted: false, includeQuotas: true, pageNumber: currentPage, pageSize: 50)));
var projects = PaginationHelper.GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>(
(currentPage) => AdminApi.GetAllProjectsAsync(includeDeleted: false, includeQuotas: true, pageNumber: currentPage, pageSize: 50));
var reportingFiles = new List<ReportingFileObject>();
var returnObjects = new List<ProjectReport>();
......
......
src/KPI Generator/Reportings/Resource/ResourceReporting.cs
+ 4
− 9
View file @ 27eff841
......@@ -17,7 +17,6 @@ namespace Coscine.KpiGenerator.Reportings.Resource;
public class ResourceReporting
{
private readonly IMapper _mapper;
private readonly ICacheService _cacheService;
private readonly ILogger<ResourceReporting> _logger;
private readonly IStorageService _gitlabStorageService;
private readonly IStorageService _localStorageService;
......@@ -34,7 +33,6 @@ public class ResourceReporting
public ResourceReporting(
IMapper mapper,
ICacheService cacheService,
ILogger<ResourceReporting> logger,
[FromKeyedServices("gitlab")] IStorageService gitlabStorageService,
[FromKeyedServices("local")] IStorageService localStorageService,
......@@ -43,7 +41,6 @@ public class ResourceReporting
)
{
_mapper = mapper;
_cacheService = cacheService;
_logger = logger;
_gitlabStorageService = gitlabStorageService;
_localStorageService = localStorageService;
......@@ -84,14 +81,12 @@ public class ResourceReporting
public async Task<IEnumerable<ReportingFileObject>> GenerateReportingAsync()
{
// Use the cache service to get the projects
var projects = _cacheService.GetOrAddAsync(MemoryCacheService.ProjectsCacheKey,
() => PaginationHelper.GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>(
(currentPage) => AdminApi.GetAllProjectsAsync(includeDeleted: false, pageNumber: currentPage, pageSize: 250)));
var projects = PaginationHelper.GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>(
(currentPage) => AdminApi.GetAllProjectsAsync(includeDeleted: true, pageNumber: currentPage, pageSize: 50));
// Use the cache service to get the resources
var resources = _cacheService.GetOrAddAsync(MemoryCacheService.ResourceCacheKey,
() => PaginationHelper.GetAllAsync<ResourceAdminDtoPagedResponse, ResourceAdminDto>(
(currentPage) => AdminApi.GetAllResourcesAsync(includeDeleted: false, includeQuotas: true, pageNumber: currentPage, pageSize: 50)));
var resources = PaginationHelper.GetAllAsync<ResourceAdminDtoPagedResponse, ResourceAdminDto>(
(currentPage) => AdminApi.GetAllResourcesAsync(includeDeleted: false, includeQuotas: true, pageNumber: currentPage, pageSize: 50));
var reportingFiles = new List<ReportingFileObject>();
var returnObjects = new List<ResourceReport>();
......
......
src/KPI Generator/Reportings/User/UserReporting.cs
+ 2
− 6
View file @ 27eff841
......@@ -17,7 +17,6 @@ namespace Coscine.KpiGenerator.Reportings.Resource;
public class UserReporting
{
private readonly IMapper _mapper;
private readonly ICacheService _cacheService;
private readonly ILogger<UserReporting> _logger;
private readonly IStorageService _gitlabStorageService;
private readonly IStorageService _localStorageService;
......@@ -34,7 +33,6 @@ public class UserReporting
public UserReporting(
IMapper mapper,
ICacheService cacheService,
ILogger<UserReporting> logger,
[FromKeyedServices("gitlab")] IStorageService gitlabStorageService,
[FromKeyedServices("local")] IStorageService localStorageService,
......@@ -43,7 +41,6 @@ public class UserReporting
)
{
_mapper = mapper;
_cacheService = cacheService;
_logger = logger;
_gitlabStorageService = gitlabStorageService;
_localStorageService = localStorageService;
......@@ -84,9 +81,8 @@ public class UserReporting
public async Task<IEnumerable<ReportingFileObject>> GenerateReportingAsync()
{
// Use the cache service to get the projects
var projects = _cacheService.GetOrAddAsync(MemoryCacheService.ProjectsCacheKey,
() => PaginationHelper.GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>(
(currentPage) => AdminApi.GetAllProjectsAsync(includeDeleted: false, pageNumber: currentPage, pageSize: 250)));
var projects = PaginationHelper.GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>(
(currentPage) => AdminApi.GetAllProjectsAsync(includeDeleted: false, pageNumber: currentPage, pageSize: 50));
var users = PaginationHelper.GetAllAsync<UserDtoPagedResponse, UserDto>(
(currentPage) => AdminApi.GetAllUsersAsync(tosAccepted: true, pageNumber: currentPage, pageSize: 250));
......
......
src/KPI Generator/Utils/ICacheService.cs deleted 100644 → 0
+ 0
− 13
View file @ 42af8511
using Coscine.ApiClient.Core.Model;
namespace Coscine.KpiGenerator.Utils;
public interface ICacheService
{
T? Get<T>(string key);
void Set<T>(string key, T item, TimeSpan? absoluteExpireTime = null, TimeSpan? unusedExpireTime = null);
Task<IEnumerable<ProjectAdminDto>> GetProjectsAsync(Func<Task<IEnumerable<ProjectAdminDto>>> apiCall);
Task<IEnumerable<ResourceAdminDto>> GetResourcesAsync(Func<Task<IEnumerable<ResourceAdminDto>>> apiCall);
IAsyncEnumerable<T> GetOrAddAsync<T>(string key, Func<IAsyncEnumerable<T>> fetchFunction, TimeSpan? expirationRelativeToNow = null);
}
src/KPI Generator/Utils/MemoryCacheService.cs deleted 100644 → 0
+ 0
− 92
View file @ 42af8511
using Coscine.ApiClient.Core.Model;
using Microsoft.Extensions.Caching.Memory;
namespace Coscine.KpiGenerator.Utils;
public class MemoryCacheService(IMemoryCache memoryCache) : ICacheService
{
public static string ProjectsCacheKey { get; } = "projects";
public static string ResourceCacheKey { get; } = "resources";
private readonly IMemoryCache _memoryCache = memoryCache;
private readonly SemaphoreSlim _lock = new(1, 1);
public T? Get<T>(string key)
{
return _memoryCache.Get<T>(key);
}
public void Set<T>(string key, T item, TimeSpan? absoluteExpireTime = null, TimeSpan? unusedExpireTime = null)
{
var options = new MemoryCacheEntryOptions();
if (absoluteExpireTime.HasValue)
{
options.SetAbsoluteExpiration(absoluteExpireTime.Value);
}
if (unusedExpireTime.HasValue)
{
options.SetSlidingExpiration(unusedExpireTime.Value);
}
_memoryCache.Set(key, item, options);
}
[Obsolete($"Use {nameof(GetOrAddAsync)} instead.")]
public async Task<IEnumerable<ProjectAdminDto>> GetProjectsAsync(Func<Task<IEnumerable<ProjectAdminDto>>> apiCall)
{
var projects = Get<IEnumerable<ProjectAdminDto>>(ProjectsCacheKey);
if (projects == null)
{
projects = await apiCall();
Set(ProjectsCacheKey, projects, TimeSpan.FromHours(1)); // Set cache with 1-hour expiration
}
return projects;
}
[Obsolete($"Use {nameof(GetOrAddAsync)} instead.")]
public async Task<IEnumerable<ResourceAdminDto>> GetResourcesAsync(Func<Task<IEnumerable<ResourceAdminDto>>> apiCall)
{
var resources = Get<IEnumerable<ResourceAdminDto>>(ResourceCacheKey);
if (resources == null)
{
resources = await apiCall();
Set(ResourceCacheKey, resources, TimeSpan.FromHours(1)); // Set cache with 1-hour expiration
}
return resources;
}
public async IAsyncEnumerable<T> GetOrAddAsync<T>(string key, Func<IAsyncEnumerable<T>> fetchFunction, TimeSpan? expirationRelativeToNow = null)
{
if (_memoryCache.TryGetValue(key, out List<T>? cachedValues) && cachedValues is not null)
{
// Serve from cache
foreach (var item in cachedValues)
{
yield return item;
}
}
else
{
var buffer = new List<T>();
await foreach (var item in fetchFunction())
{
buffer.Add(item);
yield return item;
}
// Ensure only one caller updates the cache for a given key
await _lock.WaitAsync();
try
{
// Double-check locking in case the cache was populated while fetching items
if (!_memoryCache.TryGetValue(key, out _))
{
_memoryCache.Set(key, buffer, expirationRelativeToNow ?? TimeSpan.FromHours(1));
}
}
finally
{
_lock.Release();
}
}
}
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment