BREAKING: Migrated KPI Generator to the APIv2 infrastructure
15 open threads
coscine/issues#2666
Merge request reports
Activity
3 11 using Newtonsoft.Json; 4 using NLog.Extensions.Logging; 5 using VDS.RDF.Query; 12 using VDS.RDF; 13 using VDS.RDF.Nodes; 14 using VDS.RDF.Parsing; 6 15 using static KPIGenerator.Utils.CommandLineOptions; 7 16 8 namespace Coscine.KpiGenerator.Reportings.ApplicationProfile; 17 namespace Coscine.KpiGenerator.Reportings.Resource; 9 18 10 public class ApplicationProfileReporting : Reporting<ApplicationProfileReportingOptions> 19 public class ApplicationProfileReporting 11 20 { 12 private readonly ILogger<ApplicationProfileReporting> _logger; 21 private readonly IMapper _mapper; 105 returnObjects.Add(new ApplicationProfileReport 73 106 { 74 Uri = ap.Key.Uri, 75 Publisher = ap.Key.Publisher, 76 Rights = ap.Key.Rights, 77 License = ap.Key.License, 78 Titles = ap.Select(t => 79 { 80 if (t is not null) 81 { 82 return t[..t.IndexOf('@')]; 83 } 84 return t; 85 }).ToList() 107 Uri = ap.Uri, 108 Titles = titles ?? new List<string>(), 17 private readonly IStorageService _gitlabStorageService; 18 private readonly IStorageService _localStorageService; 16 19 17 public CompleteReporting(CompleteReportingOptions options) : base(options) 20 public CompleteReportingOptions Options { get; private set; } = null!; 21 22 23 public CompleteReporting( 24 ProjectReporting projectReporting, 25 ResourceReporting resourceReporting, 26 UserReporting userReporting, 27 ApplicationProfileReporting applicationProfileReporting, 28 SystemReporting systemReporting, 29 [FromKeyedServices("gitlab")] IStorageService gitlabStorageService, 30 [FromKeyedServices("local")] IStorageService localStorageService 31 ) - Comment on lines +23 to +31
Primary constructor candidate (https://learn.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/ide0290).
65 { 66 Console.WriteLine("Failed to publish to GitLab. Publishing to local storage instead."); 67 success = await _localStorageService.PublishAsync("Project Reporting", reportingFiles); 68 } 69 return success; 32 70 } 33 71 34 public override IEnumerable<ReportingFileObject> GenerateReporting() 72 public async Task<IEnumerable<ReportingFileObject>> GenerateReportingAsync() 35 73 { 36 Console.WriteLine($" - {GetType().Name}: Begin reporting generation"); 37 _logger.LogInformation("{Name}: Begin reporting generation", GetType().Name); 38 var projects = _projectModel.GetAllWhere(r => r.Deleted.Equals(true) || r.Deleted.Equals(false)); 74 // Use the cache service to get the projects 75 var projects = await _cacheService.GetProjectsAsync(async () => await RequestUtil.WrapPagedRequest<ProjectAdminDtoPagedResponse, ProjectAdminDto>( 76 (currentPage) => AdminApi.GetAllProjectsAsync(includeDeleted: false, pageNumber: currentPage, pageSize: 250))); 67 { 68 Console.WriteLine("Failed to publish to GitLab. Publishing to local storage instead."); 69 success = await _localStorageService.PublishAsync("Resources Reporting", reportingFiles); 70 } 71 return success; 27 72 } 28 73 29 public override IEnumerable<ReportingFileObject> GenerateReporting() 74 public async Task<IEnumerable<ReportingFileObject>> GenerateReportingAsync() 30 75 { 31 Console.WriteLine($" - {GetType().Name}: Begin reporting generation"); 32 _logger.LogInformation("{Name}: Begin reporting generation", GetType().Name); 33 var resources = _resourceModel.GetAllWhere(r => r.Deleted.Equals(true) || r.Deleted.Equals(false)); 76 // Use the cache service to get the projects 77 var projects = await _cacheService.GetProjectsAsync(async () => await RequestUtil.WrapPagedRequest<ProjectAdminDtoPagedResponse, ProjectAdminDto>( 78 (currentPage) => AdminApi.GetAllProjectsAsync(includeDeleted: false, pageNumber: currentPage, pageSize: 250))); 71 return success; 27 72 } 28 73 29 public override IEnumerable<ReportingFileObject> GenerateReporting() 74 public async Task<IEnumerable<ReportingFileObject>> GenerateReportingAsync() 30 75 { 31 Console.WriteLine($" - {GetType().Name}: Begin reporting generation"); 32 _logger.LogInformation("{Name}: Begin reporting generation", GetType().Name); 33 var resources = _resourceModel.GetAllWhere(r => r.Deleted.Equals(true) || r.Deleted.Equals(false)); 76 // Use the cache service to get the projects 77 var projects = await _cacheService.GetProjectsAsync(async () => await RequestUtil.WrapPagedRequest<ProjectAdminDtoPagedResponse, ProjectAdminDto>( 78 (currentPage) => AdminApi.GetAllProjectsAsync(includeDeleted: false, pageNumber: currentPage, pageSize: 250))); 79 80 // Use the cache service to get the resources 81 var resources = await _cacheService.GetResourcesAsync(async () => await RequestUtil.WrapPagedRequest<ResourceAdminDtoPagedResponse, ResourceAdminDto>( 82 (currentPage) => AdminApi.GetAllResourcesAsync(includeDeleted: false, includeQuotas: true, pageNumber: currentPage, pageSize: 250))); 8 using Microsoft.Extensions.DependencyInjection; 9 using Microsoft.Extensions.Options; 4 10 using Newtonsoft.Json; 5 using NLog.Extensions.Logging; 6 11 using System.Net.Http.Headers; 7 12 using System.Text; 8 13 using static KPIGenerator.Utils.CommandLineOptions; 9 14 10 namespace Coscine.KpiGenerator.Reportings.System; 15 namespace Coscine.KpiGenerator.Reportings.Resource; 11 16 12 public class SystemReporting : Reporting<SystemReportingOptions> 17 public class SystemReporting 13 18 { 14 private readonly ILogger<SystemReporting> _logger; 19 private readonly IMapper _mapper; 62 101 63 102 // Add Basic Authentication Headers 64 var authenticationString = $"{_configuration.GetStringAndWait("coscine/global/maintenance/user")}:{_configuration.GetStringAndWait("coscine/global/maintenance/password")}"; 103 var authenticationString = $"{_kpiConfiguration.SystemKpi.Maintenance.Username}:{_kpiConfiguration.SystemKpi.Maintenance.Password}"; 65 104 var base64EncodedAuthenticationString = Convert.ToBase64String(Encoding.ASCII.GetBytes(authenticationString)); 66 105 requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Basic", base64EncodedAuthenticationString); 67 106 68 var result = await httpClient.SendAsync(requestMessage); 69 var responseBody = JsonConvert.DeserializeObject<List<MaintenanceBannerObject>>(result.Content.ReadAsStringAsync().Result); 107 var result = await _httpClient.SendAsync(requestMessage); 108 var responseBody = JsonConvert.DeserializeObject<List<MaintenanceReport>>(result.Content.ReadAsStringAsync().Result); 70 109 if (responseBody is null) 71 110 { 72 111 return null; 73 112 } 74 113 return responseBody.OrderBy(e => e.StartAt).ToList(); 87 { 88 List<ReturnObject> returnObjectsForOrganization; 89 90 // Handling of "Other" organization 91 if (entry.RorUrl.Equals("https://ror.org/") || entry.RorUrl.Equals("http://ror.org/")) 92 { 93 returnObjectsForOrganization = returnObjects.Where(ro => ro.Organizations.Select(o => o.RorUrl).Any(e => e.Equals("https://ror.org/") || e.Equals("http://ror.org/"))).ToList(); 94 entry.RorUrl = _otherOrganization.RorUrl; 95 } 96 else 97 { 98 returnObjectsForOrganization = returnObjects.Where(ro => ro.Organizations.Select(o => o.RorUrl).Any(e => e.Contains(entry.RorUrl))).ToList(); 99 } 76 // Use the cache service to get the projects 77 var projects = await _cacheService.GetProjectsAsync(async () => await RequestUtil.WrapPagedRequest<ProjectAdminDtoPagedResponse, ProjectAdminDto>( 78 (currentPage) => AdminApi.GetAllProjectsAsync(includeDeleted: false, pageNumber: currentPage, pageSize: 250))); 95 } 96 else 97 { 98 returnObjectsForOrganization = returnObjects.Where(ro => ro.Organizations.Select(o => o.RorUrl).Any(e => e.Contains(entry.RorUrl))).ToList(); 99 } 76 // Use the cache service to get the projects 77 var projects = await _cacheService.GetProjectsAsync(async () => await RequestUtil.WrapPagedRequest<ProjectAdminDtoPagedResponse, ProjectAdminDto>( 78 (currentPage) => AdminApi.GetAllProjectsAsync(includeDeleted: false, pageNumber: currentPage, pageSize: 250))); 100 79 101 var reportingFile = new ReportingFileObject 102 { 103 Path = GetReportingPathOrganization(entry.RorUrl, ReportingFileName), 104 Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjectsForOrganization, Formatting.Indented)) 105 }; 80 var users = await RequestUtil.WrapPagedRequest<UserDtoPagedResponse, UserDto>( 81 (currentPage) => AdminApi.GetAllUsersAsync(tosAccepted: true, pageNumber: currentPage, pageSize: 250)); 116 { 117 var result = new List<ReturnObject.RelatedProject>(); 118 var projectRoles = _projectRoleModel.GetAllWhere(role => role.UserId.Equals(id)); 119 if (projectRoles.Any()) 89 // Do additional processing 90 foreach (var returnObject in returnObjects) 120 91 { 121 foreach (var projectRole in projectRoles) 92 var relatedProjects = new List<RelatedProject>(); 93 94 // Set the user project roles from the related projects 95 var relatedProjectDtos = projects.Where(p => p.ProjectRoles.Any(pr => pr.UserId == returnObject.Id)); 96 foreach (var relatedProject in relatedProjectDtos) 122 97 { 123 if (_projectModel.GetById(projectRole.ProjectId) is not null) // null if project has been deleted 98 var roleId = relatedProject.ProjectRoles.FirstOrDefault(pr => pr.UserId == returnObject.Id)?.RoleId; 187 119 188 private Organization? TryGetOrganizationByLabel(string rdfsLabel) 120 private List<ReportingFileObject> GeneratePerOrganization(List<UserReport> returnObjects) 189 121 { 190 if (string.IsNullOrWhiteSpace(rdfsLabel)) 191 { 192 return null; 193 } 194 var _queryString = new SparqlParameterizedString() 122 var reportingFilesPerOrganization = new List<ReportingFileObject>(); 123 var organizationsFromProjects = Helpers.GetTopLevelOrganizationsFromEntries(returnObjects.SelectMany(ro => ro.Organizations)); 124 foreach (var entry in organizationsFromProjects) 195 125 { 196 CommandText = $@" 197 PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> 126 var returnObjectsForOrganization = returnObjects.Where(ro => ro.Organizations.Select(o => o.RorUrl).Any(e => e is not null && e.Contains(entry.RorUrl))).ToList(); 43 await EnsureInformationIsSetAndCorrectAsync(); 44 try 45 { 46 foreach (var file in files) 47 { 48 var filePath = Path.Combine(_localStoragePath ?? $"C:/coscine/reporting/temp/{reportingInstanceName}/", file.Path); 49 var directoryPath = Path.GetDirectoryName(filePath); 50 51 // Ensure the directory exists 52 if (!Directory.Exists(directoryPath)) 53 { 54 Directory.CreateDirectory(directoryPath); 55 } 56 57 // Write file contents to disk 58 using var fileStream = new FileStream(filePath, FileMode.Create, FileAccess.Write); 64 foreach (var fileInProject in projectTree.Where(file => file.Type.Equals("blob"))) 65 { 66 // Delete files, that are not part of "files" and are not any of the files to keep 67 if (!files.Any(f => f.Path.Equals(fileInProject.Path)) && !_reportingConfiguration.FilesToKeepInRepo.Any(f => f.Equals(fileInProject.Path))) 68 { 69 // Add Action 70 actions.Add(new CreateCommitRequestAction(CreateCommitRequestActionType.Delete, fileInProject.Path)); 71 } 72 } 73 74 // Create a commit per file with its contents 75 foreach (var file in files) 76 { 77 // Write file contents to bytes 78 byte[] bytes; 79 using (var ms = new MemoryStream()) 38 (ProjectReportingOptions options) => new ProjectReporting(SanitizeOptions(options)).Run(), 39 (ResourceReportingOptions options) => new ResourceReporting(SanitizeOptions(options)).Run(), 40 (UserReportingOptions options) => new UserReporting(SanitizeOptions(options)).Run(), 41 (ApplicationProfileReportingOptions options) => new ApplicationProfileReporting(SanitizeOptions(options)).Run(), 42 (SystemReportingOptions options) => new SystemReporting(SanitizeOptions(options)).Run(), 43 _ => false); 37 >(args); 38 39 var result = parserResult.MapResult( 40 (CompleteReportingOptions opts) => _serviceProvider.GetRequiredService<CompleteReporting>().RunAsync(opts).Result, 41 (ProjectReportingOptions opts) => _serviceProvider.GetRequiredService<ProjectReporting>().RunAsync(opts).Result, 42 (ResourceReportingOptions opts) => _serviceProvider.GetRequiredService<ResourceReporting>().RunAsync(opts).Result, 43 (UserReportingOptions opts) => _serviceProvider.GetRequiredService<UserReporting>().RunAsync(opts).Result, 44 (SystemReportingOptions opts) => _serviceProvider.GetRequiredService<SystemReporting>().RunAsync(opts).Result, 45 (ApplicationProfileReportingOptions opts) => _serviceProvider.GetRequiredService<ApplicationProfileReporting>().RunAsync(opts).Result, 46 HandleParseError -
The whole idea was not to have an overly complex script, that just gets the values from the API.
In essence, a shorter version of the API core was reconstructed instead. The whole code, while well implemented, is an absolute overkill for generating KPIs. So instead of simply querying the API, like we wanted, we now just rebuild a shorter version of the API and mimic its behavior, when the idea was not to have it inside the API.
That just went almost full circle.
mentioned in commit b5eff56d
Please register or sign in to reply