Skip to content
Snippets Groups Projects
Commit faef1a79 authored by Petar Hristov's avatar Petar Hristov :speech_balloon:
Browse files

Merge branch 'dev' into 'main'

Release: Sprint/2022 19 :robot:

See merge request !8
parents c12c7d25 d3f411f9
Branches
Tags
1 merge request!8Release: Sprint/2022 19 :robot:
Pipeline #820017 passed
......@@ -22,6 +22,7 @@
<PackageReference Include="Coscine.ApiCommons" Version="2.*-*" />
<PackageReference Include="Coscine.Database" Version="2.*-*" />
<PackageReference Include="Coscine.Metadata" Version="2.*-*" />
<PackageReference Include="Coscine.ResourceTypes" Version="1.*-*" />
<PackageReference Include="GitLabApiClient" Version="1.8.1-beta.5" />
</ItemGroup>
</Project>
......@@ -24,6 +24,12 @@ public abstract class Reporting<O> where O : class
private static string InstanceName { get; set; } = null!;
public virtual string ReportingFileName { get; init; } = null!;
public readonly Organization _otherOrganization = new()
{
Name = "Other",
RorUrl = "https://ror.org/_other",
};
/// <summary>
/// Reporting Database GitLab Project URL
/// </summary>
......@@ -38,7 +44,7 @@ public abstract class Reporting<O> where O : class
RdfStoreConnector = new RdfStoreConnector(Configuration.GetStringAndWait("coscine/local/virtuoso/additional/url"));
QueryEndpoint = new SparqlRemoteEndpoint(new Uri(Configuration.GetStringAndWait("coscine/local/virtuoso/additional/url")));
GitLabClient = new GitLabClient(HostUrl, Configuration.GetStringAndWait("coscine/global/gitlabtoken"));
Organizations = FetchOrganizations();
Organizations = new List<Organization>() { _otherOrganization };
}
public abstract IEnumerable<ReportingFileObject> GenerateReporting();
......@@ -63,7 +69,6 @@ public abstract class Reporting<O> where O : class
return success;
}
private async Task<bool> PublishAsync(IEnumerable<ReportingFileObject> files)
{
try
......@@ -132,53 +137,47 @@ public abstract class Reporting<O> where O : class
}
}
private List<Organization> FetchOrganizations()
public Organization FetchOrganizationByRor(string rorUrl)
{
var organizations = new List<Organization>();
var organizationsToFind = JsonConvert.DeserializeObject<IEnumerable<Uri>>(
Configuration.GetStringAndWait("coscine/local/organizations/list",
"['https://ror.org/', 'https://ror.org/04xfq0f34']")
);
var resultSet = new List<Triple>();
foreach (var orgGraph in organizationsToFind)
var result = new Organization();
var organizationFound = Organizations.Find(o => o.RorUrl.Equals(rorUrl));
if (organizationFound is not null)
{
resultSet.AddRange(RdfStoreConnector.GetTriples(orgGraph, null));
}
var organizationTriples = resultSet.Where(r => !r.Subject.ToString().Contains('#')).Distinct().ToList();
foreach (var triple in organizationTriples)
result = new Organization
{
organizations.Add(new Organization
{
Name = triple.Object.ToString(),
Ror = triple.Subject.ToString(),
});
Name = organizationFound.Name,
RorUrl = organizationFound.RorUrl,
};
}
// Organization "Other"
var organizationOther = organizations.Find(o => o.Name.Equals("Other"));
var organizationOtherRor = "https://ror.org/_other";
if (organizationOther is null)
else
{
organizations.Add(new Organization
var organizationTriples = RdfStoreConnector.GetLabelForSubject(new Uri(Uri.UnescapeDataString(rorUrl))).ToList();
if (organizationTriples.Any())
{
Name = "Other",
Ror = organizationOtherRor
});
result = new Organization
{
// Only one entry possible per organization, take 0th element
Name = organizationTriples[0].Object.ToString(),
RorUrl = organizationTriples[0].Subject.ToString(),
};
Organizations.Add(result); // Cache the fetched organization
}
else
{
var index = organizations.IndexOf(organizationOther);
organizations[index].Ror = organizationOtherRor;
result = _otherOrganization;
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine($"WARNING!: Organization with ROR \"{rorUrl}\" could not be correctly identified. Will use \"{result.RorUrl}\" and \"{result.Name}\".");
Console.ResetColor();
}
return organizations;
}
return result;
}
public static string GetReportingPathGeneral(string fileName)
{
return string.Format("General/{0}", fileName);
}
public static string GetReportingPathOrganization(string organizationRor, string fileName)
{
return string.Format("Organizations/{0}/{1}", organizationRor, fileName);
......
using KPIGenerator.Utils;
using Coscine.Database.Models;
using Coscine.Database.ReturnObjects;
using Coscine.ResourceTypes;
using Coscine.ResourceTypes.Base;
using KPIGenerator.Utils;
using Newtonsoft.Json;
using static KPIGenerator.Utils.CommandLineOptions;
namespace KPIGenerator.Reportings.Resource;
public class ResourceReporting : Reporting<ResourceReportingOptions>
{
private readonly ResourceModel _resourceModel;
private readonly ProjectModel _projectModel;
private readonly ProjectResourceModel _projectResourceModel;
public ResourceReporting(ResourceReportingOptions options) : base(options)
{
ReportingFileName = "resources.json";
_resourceModel = new ResourceModel();
_projectModel = new ProjectModel();
_projectResourceModel = new ProjectResourceModel();
}
public override IEnumerable<ReportingFileObject> GenerateReporting()
{
/*
* 1. Collect the reporting for the whole database -- General/{ReportingReportingFileName}
* 2. Append to the list the same information per organization as folders -- Organizations/{OrgRorId}/{ReportingReportingFileName}
* --> See envisioned folder structure.
*/
throw new NotImplementedException();
var resources = _resourceModel.GetAllWhere(r => r.Deleted.Equals(true) || r.Deleted.Equals(false));
var reportingFiles = new List<ReportingFileObject>();
var returnObjects = Generate(resources);
// General File
reportingFiles.Add(new ReportingFileObject
{
Path = GetReportingPathGeneral(ReportingFileName),
Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjects, Formatting.Indented))
});
// Per Organization
reportingFiles.AddRange(GeneratePerOrganization(returnObjects));
return reportingFiles;
}
private List<ReturnObject> Generate(IEnumerable<Coscine.Database.DataModel.Resource> resources)
{
var returnObjects = new List<ReturnObject>();
foreach (var resource in resources)
{
var resourceReturnObject = _resourceModel.CreateReturnObjectFromDatabaseObject(resource);
var resourceReportEntry = new ReturnObject
{
Id = resourceReturnObject.Id,
ResourceType = resourceReturnObject.Type.DisplayName,
DateCreated = resourceReturnObject.DateCreated,
Archived = resourceReturnObject.Archived,
Deleted = resourceReturnObject.Deleted,
MetadataVisibilityId = resourceReturnObject.Visibility.Id,
RelatedProjectId = GetRelatedProject(resource.Id),
Organizations = GetOrganizations(resourceReturnObject.Id),
Disciplines = resourceReturnObject.Disciplines.ToList(),
License = resourceReturnObject.License is not null ? resourceReturnObject.License.DisplayName : null,
ApplicationProfile = resourceReturnObject.ApplicationProfile,
ResourceQuota = GetResourceQuota(resource)
};
returnObjects.Add(resourceReportEntry);
}
return returnObjects;
}
private IEnumerable<ReportingFileObject> GeneratePerOrganization(List<ReturnObject> returnObjects)
{
var reportingFilesPerOrganization = new List<ReportingFileObject>();
var organizationsFromResources = returnObjects.SelectMany(ro => ro.Organizations).DistinctBy(o => o.RorUrl);
foreach (var entry in organizationsFromResources)
{
var organization = Organizations.Find(o => o.Equals(entry));
if (organization is null)
{
organization = _otherOrganization;
Console.WriteLine($"WARNING!: Organization \"{entry.RorUrl}\" could not be correctly identified. Will use \"{_otherOrganization.RorUrl}\".");
}
var returnObjectsForOrganization = returnObjects.Where(ro => ro.Organizations.Select(o => o.RorUrl).Any(e => e.Equals(entry.RorUrl)));
reportingFilesPerOrganization.Add(new ReportingFileObject
{
Path = GetReportingPathOrganization(organization.RorUrl.Replace("https://ror.org/", "").ToLower(), ReportingFileName),
Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjectsForOrganization, Formatting.Indented))
});
}
return reportingFilesPerOrganization;
}
private Guid? GetRelatedProject(Guid resourceId)
{
try
{
return _projectResourceModel.GetProjectForResource(resourceId);
}
catch
{
Console.WriteLine($"There is no project related to resource with ID \"{resourceId}\".");
return null;
}
}
private static ResourceQuotaReturnObject? GetResourceQuota(Coscine.Database.DataModel.Resource resource)
{
BaseResourceType? resourceTypeDefinition;
try
{
resourceTypeDefinition = ResourceTypeFactory.Instance.GetResourceType(resource);
}
catch
{
Console.WriteLine($"No resource type definition found for resource with ID \"{resource.Id}\".");
resourceTypeDefinition = null;
}
if (resourceTypeDefinition is not null && resourceTypeDefinition.GetResourceTypeInformation().Result.IsQuotaAdjustable)
{
return Helpers.CreateResourceQuotaReturnObject(resource, resourceTypeDefinition);
}
else
{
return null;
}
}
private List<Organization> GetOrganizations(Guid resourceId)
{
var result = new List<Organization>();
Guid? relatedProjectId;
try
{
relatedProjectId = _projectResourceModel.GetProjectForResource(resourceId);
}
catch
{
relatedProjectId = null;
}
if (relatedProjectId is not null)
{
var parentProject = _projectModel.GetByIdIncludingDeleted(relatedProjectId.Value);
var organizations = _projectModel.CreateReturnObjectFromDatabaseObject(parentProject).Organizations.ToList();
foreach (var entry in organizations)
{
result.Add(FetchOrganizationByRor(entry.Url));
}
}
return result;
}
}
\ No newline at end of file
namespace KPIGenerator.Reportings.Resource;
using Coscine.Database.ReturnObjects;
using KPIGenerator.Utils;
namespace KPIGenerator.Reportings.Resource;
/// <summary>
/// Object containing the JSON structure for the reporting
/// </summary>
public class ReturnObject
{
public Guid Id { get; set; }
public string ResourceType { get; set; } = null!;
public DateTime? DateCreated { get; set; } = null;
public bool Archived { get; set; }
public bool Deleted { get; set; }
public Guid MetadataVisibilityId { get; set; }
public Guid? RelatedProjectId { get; set; }
public List<Organization> Organizations { get; set; } = new();
public List<DisciplineObject> Disciplines { get; set; } = new();
public string? License { get; set; } = null;
public string ApplicationProfile { get; set; } = null!;
public ResourceQuotaReturnObject? ResourceQuota { get; set; } = null!;
}
using Coscine.Database.ReturnObjects;
using Newtonsoft.Json;
namespace KPIGenerator.Reportings.User;
......
......@@ -72,7 +72,7 @@ public class UserReporting : Reporting<UserReportingOptions>
private IEnumerable<ReportingFileObject> GeneratePerOrganization(List<ReturnObject> returnObjects)
{
var reportingFilesPerOrganization = new List<ReportingFileObject>();
var organizationsFromUsers = returnObjects.SelectMany(ro => ro.Organizations).Distinct();
var organizationsFromUsers = returnObjects.SelectMany(ro => ro.Organizations);
foreach (var entry in organizationsFromUsers)
{
var organization = Organizations.Find(o => o.Name.Equals(entry));
......@@ -85,7 +85,7 @@ public class UserReporting : Reporting<UserReportingOptions>
reportingFilesPerOrganization.Add(new ReportingFileObject
{
Path = GetReportingPathOrganization(organization.Ror.Replace("https://ror.org/", "").ToLower(), ReportingFileName),
Path = GetReportingPathOrganization(organization.RorUrl.Replace("https://ror.org/", "").ToLower(), ReportingFileName),
Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjectsForOrganization, Formatting.Indented))
});
}
......
......@@ -8,12 +8,8 @@ public class Organization
/// <example>RWTH Aachen University</example>
public string Name { get; set; } = null!;
/// <summary>
/// Organizaiton ROR from GitLab project's title
/// </summary>
/// <example>04xfq0f34</example>
public string Ror { get; set; } = null!;
/// <summary>
/// Organizaiton ROR URL from GitLab project's title
/// </summary>
/// <example>https://ror.org/04xfq0f34</example>
public string RorUrl { get; set; } = null!;
}
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment