Skip to content
Snippets Groups Projects

BREAKING: Migrated KPI Generator to the APIv2 infrastructure

coscine/issues#2666

Merge request reports

Loading
Loading

Activity

Filter activity
  • Approvals
  • Assignees & reviewers
  • Comments (from bots)
  • Comments (from users)
  • Commits & branches
  • Edits
  • Labels
  • Lock status
  • Mentions
  • Merge request status
  • Tracking
3 11 using Newtonsoft.Json;
4 using NLog.Extensions.Logging;
5 using VDS.RDF.Query;
12 using VDS.RDF;
13 using VDS.RDF.Nodes;
14 using VDS.RDF.Parsing;
6 15 using static KPIGenerator.Utils.CommandLineOptions;
7 16
8 namespace Coscine.KpiGenerator.Reportings.ApplicationProfile;
17 namespace Coscine.KpiGenerator.Reportings.Resource;
9 18
10 public class ApplicationProfileReporting : Reporting<ApplicationProfileReportingOptions>
19 public class ApplicationProfileReporting
11 20 {
12 private readonly ILogger<ApplicationProfileReporting> _logger;
21 private readonly IMapper _mapper;
  • 105 returnObjects.Add(new ApplicationProfileReport
    73 106 {
    74 Uri = ap.Key.Uri,
    75 Publisher = ap.Key.Publisher,
    76 Rights = ap.Key.Rights,
    77 License = ap.Key.License,
    78 Titles = ap.Select(t =>
    79 {
    80 if (t is not null)
    81 {
    82 return t[..t.IndexOf('@')];
    83 }
    84 return t;
    85 }).ToList()
    107 Uri = ap.Uri,
    108 Titles = titles ?? new List<string>(),
  • 17 private readonly IStorageService _gitlabStorageService;
    18 private readonly IStorageService _localStorageService;
    16 19
    17 public CompleteReporting(CompleteReportingOptions options) : base(options)
    20 public CompleteReportingOptions Options { get; private set; } = null!;
    21
    22
    23 public CompleteReporting(
    24 ProjectReporting projectReporting,
    25 ResourceReporting resourceReporting,
    26 UserReporting userReporting,
    27 ApplicationProfileReporting applicationProfileReporting,
    28 SystemReporting systemReporting,
    29 [FromKeyedServices("gitlab")] IStorageService gitlabStorageService,
    30 [FromKeyedServices("local")] IStorageService localStorageService
    31 )
  • 65 {
    66 Console.WriteLine("Failed to publish to GitLab. Publishing to local storage instead.");
    67 success = await _localStorageService.PublishAsync("Project Reporting", reportingFiles);
    68 }
    69 return success;
    32 70 }
    33 71
    34 public override IEnumerable<ReportingFileObject> GenerateReporting()
    72 public async Task<IEnumerable<ReportingFileObject>> GenerateReportingAsync()
    35 73 {
    36 Console.WriteLine($" - {GetType().Name}: Begin reporting generation");
    37 _logger.LogInformation("{Name}: Begin reporting generation", GetType().Name);
    38 var projects = _projectModel.GetAllWhere(r => r.Deleted.Equals(true) || r.Deleted.Equals(false));
    74 // Use the cache service to get the projects
    75 var projects = await _cacheService.GetProjectsAsync(async () => await RequestUtil.WrapPagedRequest<ProjectAdminDtoPagedResponse, ProjectAdminDto>(
    76 (currentPage) => AdminApi.GetAllProjectsAsync(includeDeleted: false, pageNumber: currentPage, pageSize: 250)));
  • 67 {
    68 Console.WriteLine("Failed to publish to GitLab. Publishing to local storage instead.");
    69 success = await _localStorageService.PublishAsync("Resources Reporting", reportingFiles);
    70 }
    71 return success;
    27 72 }
    28 73
    29 public override IEnumerable<ReportingFileObject> GenerateReporting()
    74 public async Task<IEnumerable<ReportingFileObject>> GenerateReportingAsync()
    30 75 {
    31 Console.WriteLine($" - {GetType().Name}: Begin reporting generation");
    32 _logger.LogInformation("{Name}: Begin reporting generation", GetType().Name);
    33 var resources = _resourceModel.GetAllWhere(r => r.Deleted.Equals(true) || r.Deleted.Equals(false));
    76 // Use the cache service to get the projects
    77 var projects = await _cacheService.GetProjectsAsync(async () => await RequestUtil.WrapPagedRequest<ProjectAdminDtoPagedResponse, ProjectAdminDto>(
    78 (currentPage) => AdminApi.GetAllProjectsAsync(includeDeleted: false, pageNumber: currentPage, pageSize: 250)));
  • 71 return success;
    27 72 }
    28 73
    29 public override IEnumerable<ReportingFileObject> GenerateReporting()
    74 public async Task<IEnumerable<ReportingFileObject>> GenerateReportingAsync()
    30 75 {
    31 Console.WriteLine($" - {GetType().Name}: Begin reporting generation");
    32 _logger.LogInformation("{Name}: Begin reporting generation", GetType().Name);
    33 var resources = _resourceModel.GetAllWhere(r => r.Deleted.Equals(true) || r.Deleted.Equals(false));
    76 // Use the cache service to get the projects
    77 var projects = await _cacheService.GetProjectsAsync(async () => await RequestUtil.WrapPagedRequest<ProjectAdminDtoPagedResponse, ProjectAdminDto>(
    78 (currentPage) => AdminApi.GetAllProjectsAsync(includeDeleted: false, pageNumber: currentPage, pageSize: 250)));
    79
    80 // Use the cache service to get the resources
    81 var resources = await _cacheService.GetResourcesAsync(async () => await RequestUtil.WrapPagedRequest<ResourceAdminDtoPagedResponse, ResourceAdminDto>(
    82 (currentPage) => AdminApi.GetAllResourcesAsync(includeDeleted: false, includeQuotas: true, pageNumber: currentPage, pageSize: 250)));
  • 8 using Microsoft.Extensions.DependencyInjection;
    9 using Microsoft.Extensions.Options;
    4 10 using Newtonsoft.Json;
    5 using NLog.Extensions.Logging;
    6 11 using System.Net.Http.Headers;
    7 12 using System.Text;
    8 13 using static KPIGenerator.Utils.CommandLineOptions;
    9 14
    10 namespace Coscine.KpiGenerator.Reportings.System;
    15 namespace Coscine.KpiGenerator.Reportings.Resource;
    11 16
    12 public class SystemReporting : Reporting<SystemReportingOptions>
    17 public class SystemReporting
    13 18 {
    14 private readonly ILogger<SystemReporting> _logger;
    19 private readonly IMapper _mapper;
  • 62 101
    63 102 // Add Basic Authentication Headers
    64 var authenticationString = $"{_configuration.GetStringAndWait("coscine/global/maintenance/user")}:{_configuration.GetStringAndWait("coscine/global/maintenance/password")}";
    103 var authenticationString = $"{_kpiConfiguration.SystemKpi.Maintenance.Username}:{_kpiConfiguration.SystemKpi.Maintenance.Password}";
    65 104 var base64EncodedAuthenticationString = Convert.ToBase64String(Encoding.ASCII.GetBytes(authenticationString));
    66 105 requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Basic", base64EncodedAuthenticationString);
    67 106
    68 var result = await httpClient.SendAsync(requestMessage);
    69 var responseBody = JsonConvert.DeserializeObject<List<MaintenanceBannerObject>>(result.Content.ReadAsStringAsync().Result);
    107 var result = await _httpClient.SendAsync(requestMessage);
    108 var responseBody = JsonConvert.DeserializeObject<List<MaintenanceReport>>(result.Content.ReadAsStringAsync().Result);
    70 109 if (responseBody is null)
    71 110 {
    72 111 return null;
    73 112 }
    74 113 return responseBody.OrderBy(e => e.StartAt).ToList();
  • 87 {
    88 List<ReturnObject> returnObjectsForOrganization;
    89
    90 // Handling of "Other" organization
    91 if (entry.RorUrl.Equals("https://ror.org/") || entry.RorUrl.Equals("http://ror.org/"))
    92 {
    93 returnObjectsForOrganization = returnObjects.Where(ro => ro.Organizations.Select(o => o.RorUrl).Any(e => e.Equals("https://ror.org/") || e.Equals("http://ror.org/"))).ToList();
    94 entry.RorUrl = _otherOrganization.RorUrl;
    95 }
    96 else
    97 {
    98 returnObjectsForOrganization = returnObjects.Where(ro => ro.Organizations.Select(o => o.RorUrl).Any(e => e.Contains(entry.RorUrl))).ToList();
    99 }
    76 // Use the cache service to get the projects
    77 var projects = await _cacheService.GetProjectsAsync(async () => await RequestUtil.WrapPagedRequest<ProjectAdminDtoPagedResponse, ProjectAdminDto>(
    78 (currentPage) => AdminApi.GetAllProjectsAsync(includeDeleted: false, pageNumber: currentPage, pageSize: 250)));
  • 95 }
    96 else
    97 {
    98 returnObjectsForOrganization = returnObjects.Where(ro => ro.Organizations.Select(o => o.RorUrl).Any(e => e.Contains(entry.RorUrl))).ToList();
    99 }
    76 // Use the cache service to get the projects
    77 var projects = await _cacheService.GetProjectsAsync(async () => await RequestUtil.WrapPagedRequest<ProjectAdminDtoPagedResponse, ProjectAdminDto>(
    78 (currentPage) => AdminApi.GetAllProjectsAsync(includeDeleted: false, pageNumber: currentPage, pageSize: 250)));
    100 79
    101 var reportingFile = new ReportingFileObject
    102 {
    103 Path = GetReportingPathOrganization(entry.RorUrl, ReportingFileName),
    104 Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjectsForOrganization, Formatting.Indented))
    105 };
    80 var users = await RequestUtil.WrapPagedRequest<UserDtoPagedResponse, UserDto>(
    81 (currentPage) => AdminApi.GetAllUsersAsync(tosAccepted: true, pageNumber: currentPage, pageSize: 250));
  • 116 {
    117 var result = new List<ReturnObject.RelatedProject>();
    118 var projectRoles = _projectRoleModel.GetAllWhere(role => role.UserId.Equals(id));
    119 if (projectRoles.Any())
    89 // Do additional processing
    90 foreach (var returnObject in returnObjects)
    120 91 {
    121 foreach (var projectRole in projectRoles)
    92 var relatedProjects = new List<RelatedProject>();
    93
    94 // Set the user project roles from the related projects
    95 var relatedProjectDtos = projects.Where(p => p.ProjectRoles.Any(pr => pr.UserId == returnObject.Id));
    96 foreach (var relatedProject in relatedProjectDtos)
    122 97 {
    123 if (_projectModel.GetById(projectRole.ProjectId) is not null) // null if project has been deleted
    98 var roleId = relatedProject.ProjectRoles.FirstOrDefault(pr => pr.UserId == returnObject.Id)?.RoleId;
  • 187 119
    188 private Organization? TryGetOrganizationByLabel(string rdfsLabel)
    120 private List<ReportingFileObject> GeneratePerOrganization(List<UserReport> returnObjects)
    189 121 {
    190 if (string.IsNullOrWhiteSpace(rdfsLabel))
    191 {
    192 return null;
    193 }
    194 var _queryString = new SparqlParameterizedString()
    122 var reportingFilesPerOrganization = new List<ReportingFileObject>();
    123 var organizationsFromProjects = Helpers.GetTopLevelOrganizationsFromEntries(returnObjects.SelectMany(ro => ro.Organizations));
    124 foreach (var entry in organizationsFromProjects)
    195 125 {
    196 CommandText = $@"
    197 PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
    126 var returnObjectsForOrganization = returnObjects.Where(ro => ro.Organizations.Select(o => o.RorUrl).Any(e => e is not null && e.Contains(entry.RorUrl))).ToList();
    • User conditional access: var returnObjectsForOrganization = returnObjects.Where(ro => ro.Organizations.Select(o => o.RorUrl).Any(e => e?.Contains(entry.RorUrl) == true)).ToList();

    • Please register or sign in to reply
  • 43 await EnsureInformationIsSetAndCorrectAsync();
    44 try
    45 {
    46 foreach (var file in files)
    47 {
    48 var filePath = Path.Combine(_localStoragePath ?? $"C:/coscine/reporting/temp/{reportingInstanceName}/", file.Path);
    49 var directoryPath = Path.GetDirectoryName(filePath);
    50
    51 // Ensure the directory exists
    52 if (!Directory.Exists(directoryPath))
    53 {
    54 Directory.CreateDirectory(directoryPath);
    55 }
    56
    57 // Write file contents to disk
    58 using var fileStream = new FileStream(filePath, FileMode.Create, FileAccess.Write);
  • 64 foreach (var fileInProject in projectTree.Where(file => file.Type.Equals("blob")))
    65 {
    66 // Delete files, that are not part of "files" and are not any of the files to keep
    67 if (!files.Any(f => f.Path.Equals(fileInProject.Path)) && !_reportingConfiguration.FilesToKeepInRepo.Any(f => f.Equals(fileInProject.Path)))
    68 {
    69 // Add Action
    70 actions.Add(new CreateCommitRequestAction(CreateCommitRequestActionType.Delete, fileInProject.Path));
    71 }
    72 }
    73
    74 // Create a commit per file with its contents
    75 foreach (var file in files)
    76 {
    77 // Write file contents to bytes
    78 byte[] bytes;
    79 using (var ms = new MemoryStream())
  • 38 (ProjectReportingOptions options) => new ProjectReporting(SanitizeOptions(options)).Run(),
    39 (ResourceReportingOptions options) => new ResourceReporting(SanitizeOptions(options)).Run(),
    40 (UserReportingOptions options) => new UserReporting(SanitizeOptions(options)).Run(),
    41 (ApplicationProfileReportingOptions options) => new ApplicationProfileReporting(SanitizeOptions(options)).Run(),
    42 (SystemReportingOptions options) => new SystemReporting(SanitizeOptions(options)).Run(),
    43 _ => false);
    37 >(args);
    38
    39 var result = parserResult.MapResult(
    40 (CompleteReportingOptions opts) => _serviceProvider.GetRequiredService<CompleteReporting>().RunAsync(opts).Result,
    41 (ProjectReportingOptions opts) => _serviceProvider.GetRequiredService<ProjectReporting>().RunAsync(opts).Result,
    42 (ResourceReportingOptions opts) => _serviceProvider.GetRequiredService<ResourceReporting>().RunAsync(opts).Result,
    43 (UserReportingOptions opts) => _serviceProvider.GetRequiredService<UserReporting>().RunAsync(opts).Result,
    44 (SystemReportingOptions opts) => _serviceProvider.GetRequiredService<SystemReporting>().RunAsync(opts).Result,
    45 (ApplicationProfileReportingOptions opts) => _serviceProvider.GetRequiredService<ApplicationProfileReporting>().RunAsync(opts).Result,
    46 HandleParseError
  • The whole idea was not to have an overly complex script, that just gets the values from the API.

    In essence, a shorter version of the API core was reconstructed instead. The whole code, while well implemented, is an absolute overkill for generating KPIs. So instead of simply querying the API, like we wanted, we now just rebuild a shorter version of the API and mimic its behavior, when the idea was not to have it inside the API.

    That just went almost full circle.

  • Petar Hristov added 1 commit

    added 1 commit

    Compare with previous version

  • L. Ellenbeck approved this merge request

    approved this merge request

  • L. Ellenbeck mentioned in commit b5eff56d

    mentioned in commit b5eff56d

  • merged

  • Please register or sign in to reply
    Loading