From 256b0db12766064c4a38f9d6322659491036b7d3 Mon Sep 17 00:00:00 2001 From: sirieamhunke <hunke@itc.rwth-aachen.de> Date: Wed, 5 Oct 2022 11:22:06 +0200 Subject: [PATCH] WIP --- src/KPI Generator/KPI Generator.csproj | 3 +- src/KPI Generator/Reporting.cs | 73 +++++++------ .../Reportings/Project/ProjectReporting.cs | 103 ++++++++++++++++-- .../Reportings/Project/ReturnObject.cs | 15 ++- src/KPI Generator/Utils/Organization.cs | 2 +- 5 files changed, 150 insertions(+), 46 deletions(-) diff --git a/src/KPI Generator/KPI Generator.csproj b/src/KPI Generator/KPI Generator.csproj index 91ce6be..65be8a7 100644 --- a/src/KPI Generator/KPI Generator.csproj +++ b/src/KPI Generator/KPI Generator.csproj @@ -20,8 +20,9 @@ <ItemGroup> <PackageReference Include="CommandLineParser" Version="2.9.1" /> <PackageReference Include="Coscine.ApiCommons" Version="2.*-*" /> - <PackageReference Include="Coscine.Database" Version="2.*-*" /> + <PackageReference Include="Coscine.Database" Version="2.15.0-issue-2183-kpige0001" /> <PackageReference Include="Coscine.Metadata" Version="2.*-*" /> + <PackageReference Include="Coscine.ResourceTypes" Version="1.*-*" /> <PackageReference Include="GitLabApiClient" Version="1.8.1-beta.5" /> </ItemGroup> </Project> diff --git a/src/KPI Generator/Reporting.cs b/src/KPI Generator/Reporting.cs index e7abf1c..403295d 100644 --- a/src/KPI Generator/Reporting.cs +++ b/src/KPI Generator/Reporting.cs @@ -24,12 +24,20 @@ public abstract class Reporting<O> where O : class private static string InstanceName { get; set; } = null!; public virtual string ReportingFileName { get; init; } = null!; + + public readonly Organization _otherOrganization = new() + { + Name = "Other", + RorUrl = "https://ror.org/_other", + }; + /// <summary> /// Reporting Database GitLab Project URL /// </summary> /// <remarks>https://git.rwth-aachen.de/coscine/reporting/reporting-database</remarks> private static readonly int ReportingDatabaseProjectId = 75304; + public Reporting(O options) { InstanceName = this.GetType().Name; @@ -38,7 +46,7 @@ public abstract class Reporting<O> where O : class RdfStoreConnector = new RdfStoreConnector(Configuration.GetStringAndWait("coscine/local/virtuoso/additional/url")); QueryEndpoint = new SparqlRemoteEndpoint(new Uri(Configuration.GetStringAndWait("coscine/local/virtuoso/additional/url"))); GitLabClient = new GitLabClient(HostUrl, Configuration.GetStringAndWait("coscine/global/gitlabtoken")); - Organizations = FetchOrganizations(); + Organizations = new List<Organization>() { _otherOrganization }; } public abstract IEnumerable<ReportingFileObject> GenerateReporting(); @@ -132,49 +140,42 @@ public abstract class Reporting<O> where O : class } } - private List<Organization> FetchOrganizations() + public Organization FetchOrganizationByRor(string rorUrl) { - var organizations = new List<Organization>(); - var organizationsToFind = JsonConvert.DeserializeObject<IEnumerable<Uri>>( - Configuration.GetStringAndWait("coscine/local/organizations/list", - "['https://ror.org/', 'https://ror.org/04xfq0f34']") - ); - - var resultSet = new List<Triple>(); - foreach (var orgGraph in organizationsToFind) - { - resultSet.AddRange(RdfStoreConnector.GetTriples(orgGraph, null)); - } - var organizationTriples = resultSet.Where(r => !r.Subject.ToString().Contains('#')).Distinct().ToList(); - - foreach (var triple in organizationTriples) - { - organizations.Add(new Organization - { - Name = triple.Object.ToString(), - Ror = triple.Subject.ToString(), - }); - } - - // Organization "Other" - var organizationOther = organizations.Find(o => o.Name.Equals("Other")); - var organizationOtherRor = "https://ror.org/_other"; - if (organizationOther is null) + var result = new Organization(); + var organizationFound = Organizations.Find(o => o.RorUrl.Equals(rorUrl)); + if (organizationFound is not null) { - organizations.Add(new Organization + result = new Organization { - Name = "Other", - Ror = organizationOtherRor - }); + Name = organizationFound.Name, + RorUrl = organizationFound.RorUrl, + }; } else { - var index = organizations.IndexOf(organizationOther); - organizations[index].Ror = organizationOtherRor; + var organizationTriples = RdfStoreConnector.GetLabelForSubject(new Uri(Uri.UnescapeDataString(rorUrl))).ToList(); + if (organizationTriples.Any()) + { + result = new Organization + { + // Only one entry possible per organization, take 0th element + Name = organizationTriples[0].Object.ToString(), + RorUrl = organizationTriples[0].Subject.ToString(), + }; + Organizations.Add(result); // Cache the fetched organization + } + else + { + result = _otherOrganization; + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine($"WARNING!: Organization with ROR \"{rorUrl}\" could not be correctly identified. Will use \"{result.RorUrl}\" and \"{result.Name}\"."); + Console.ResetColor(); + } } - - return organizations; + return result; } + public static string GetReportingPathGeneral(string fileName) { return string.Format("General/{0}", fileName); diff --git a/src/KPI Generator/Reportings/Project/ProjectReporting.cs b/src/KPI Generator/Reportings/Project/ProjectReporting.cs index 0a1a086..5cce848 100644 --- a/src/KPI Generator/Reportings/Project/ProjectReporting.cs +++ b/src/KPI Generator/Reportings/Project/ProjectReporting.cs @@ -1,22 +1,111 @@ -using KPIGenerator.Utils; +using Coscine.Database.Models; +using Coscine.Database.ReturnObjects; + +using Newtonsoft.Json; +using KPIGenerator.Utils; using static KPIGenerator.Utils.CommandLineOptions; +using Coscine.Database.DataModel; namespace KPIGenerator.Reportings.Project; public class ProjectReporting : Reporting<ProjectReportingOptions> { + private readonly ProjectModel _projectModel; + private readonly ResourceModel _resourceModel; + private readonly ProjectRoleModel _projectRoleModel; + private readonly ProjectInstituteModel _projectInstituteModel; + public ProjectReporting(ProjectReportingOptions options) : base(options) { ReportingFileName = "projects.json"; + _projectModel = new ProjectModel(); + _projectRoleModel = new ProjectRoleModel(); + _resourceModel = new ResourceModel(); + _projectInstituteModel = new ProjectInstituteModel(); } public override IEnumerable<ReportingFileObject> GenerateReporting() { - /* - * 1. Collect the reporting for the whole database -- General/{ReportingReportingFileName} - * 2. Append to the list the same information per organization as folders -- Organizations/{OrgRorId}/{ReportingReportingFileName} - * --> See envisioned folder structure. - */ + var projects = _projectModel.GetAllWhere(r => r.Deleted.Equals(true) || r.Deleted.Equals(false)); + var reportingFiles = new List<ReportingFileObject>(); + var returnObjects = Generate(projects); + + //General File + reportingFiles.Add(new ReportingFileObject + { + Path = GetReportingPathGeneral(ReportingFileName), + Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjects, Formatting.Indented)), + }); + + // Per Organization + reportingFiles.AddRange(GeneratePerOrganization(returnObjects)); + + return reportingFiles; + } + + private List<ReturnObject> Generate(IEnumerable<Coscine.Database.DataModel.Project> projects) + { + var returnObjects = new List<ReturnObject>(); + foreach (var project in projects) + { + var projectReturnObject = _projectModel.CreateReturnObjectFromDatabaseObject(project); + + var projectReportEntry = new ReturnObject + { + Id = projectReturnObject.Id, + DateCreated = projectReturnObject.DateCreated, + Organizations = GetOrganizations(projectReturnObject.Id), + Disciplines = projectReturnObject.Disciplines.ToList(), + Deleted = projectReturnObject.Deleted, + ProjectVisibilityId = projectReturnObject.Visibility.Id, + GrantId = projectReturnObject.GrantId, + Members = _projectRoleModel.GetAllWhere(x => x.ProjectId == projectReturnObject.Id).Count(), + ResourceQuota = GetResourceQuota(project) + }; + } + return returnObjects; + } + + private IEnumerable<ReportingFileObject> GeneratePerOrganization(List<ReturnObject> returnObjects) + { + var reportingFilesPerOrganization = new List<ReportingFileObject>(); + var organizationsFromProjects = returnObjects.SelectMany(ro => ro.Organizations).DistinctBy(o => o.RorUrl); + foreach (var entry in organizationsFromProjects) + { + var organization = Organizations.Find(o => o.Equals(entry)); + if (organization is null) + { + organization = _otherOrganization; + Console.WriteLine($"WARNING!: Organization \"{entry.RorUrl}\" could not be correctly identified. Will use \"{_otherOrganization.RorUrl}\"."); + } + var returnObjectsForOrganization = returnObjects.Where(ro => ro.Organizations.Select(o => o.RorUrl).Any(e => e.Equals(entry.RorUrl))); + reportingFilesPerOrganization.Add(new ReportingFileObject + { + Path = GetReportingPathOrganization(organization.RorUrl.Replace("https://ror.org/", "").ToLower(), ReportingFileName), + Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjectsForOrganization, Formatting.Indented)) + }); + } + + return reportingFilesPerOrganization; + } + + private ResourceQuotaReturnObject GetResourceQuota(object resource) + { throw new NotImplementedException(); } -} + + private List<Organization> GetOrganizations(Guid projectID) + { + var result = new List<Organization>(); + var projectInstituteModel = new ProjectInstituteModel(); + var organizations = from projectInstitute in projectInstituteModel.GetAllWhere((ProjectInstitute projectInstitute) => projectInstitute.ProjectId == projectID) + select projectInstitute.OrganizationUrl; + + foreach (var entry in organizations) + { + result.Add(FetchOrganizationByRor(entry)); + } + + return result; + } +} \ No newline at end of file diff --git a/src/KPI Generator/Reportings/Project/ReturnObject.cs b/src/KPI Generator/Reportings/Project/ReturnObject.cs index 4263424..45148e6 100644 --- a/src/KPI Generator/Reportings/Project/ReturnObject.cs +++ b/src/KPI Generator/Reportings/Project/ReturnObject.cs @@ -1,9 +1,22 @@ -namespace KPIGenerator.Reportings.Project; +using Coscine.Database.ReturnObjects; +using KPIGenerator.Utils; + +namespace KPIGenerator.Reportings.Project; /// <summary> /// Object containing the JSON structure for the reporting /// </summary> public class ReturnObject { + public Guid Id { get; set; } + public DateTime? DateCreated { get; set; } = null; + public List<Organization>? Organizations { get; set; } = new(); + public List<DisciplineObject> Disciplines { get; set; } = new(); + public bool Deleted { get; set; } + public Guid ProjectVisibilityId { get; set; } + public string? GrantId { get; set; } + public int? Members { get; set; } = null; + public ResourceQuotaReturnObject? ResourceQuota { get; set; } = null!; + } diff --git a/src/KPI Generator/Utils/Organization.cs b/src/KPI Generator/Utils/Organization.cs index c5a674c..b33b0f4 100644 --- a/src/KPI Generator/Utils/Organization.cs +++ b/src/KPI Generator/Utils/Organization.cs @@ -11,7 +11,7 @@ public class Organization /// Organizaiton ROR from GitLab project's title /// </summary> /// <example>04xfq0f34</example> - public string Ror { get; set; } = null!; + public string RorUrl { get; set; } = null!; /// <summary> /// Organizaiton ROR URL from GitLab project's title /// </summary> -- GitLab