Skip to content
Snippets Groups Projects
Commit 92bdf5fa authored by CoscineBot's avatar CoscineBot :gear:
Browse files

Fix: Organizations

parent 59f77510
No related branches found
No related tags found
No related merge requests found
Pipeline #872012 passed
......@@ -6,7 +6,6 @@ using GitLabApiClient.Models.Branches.Requests;
using GitLabApiClient.Models.Commits.Requests.CreateCommitRequest;
using Microsoft.Extensions.Logging;
using NLog.Extensions.Logging;
using Polly;
using System.Text;
using System.Web;
using static KPIGenerator.Utils.CommandLineOptions;
......@@ -172,7 +171,7 @@ public abstract class Reporting<O> where O : class
}
else
{
var organizationTriples = WrapRequest(() => RdfStoreConnector.GetLabelForSubject(new Uri(Uri.UnescapeDataString(rorUrl))).ToList());
var organizationTriples = Helpers.WrapRequest(() => RdfStoreConnector.GetLabelForSubject(new Uri(Uri.UnescapeDataString(rorUrl))).ToList());
if (organizationTriples is not null && organizationTriples.Any())
{
result = new Organization
......@@ -260,33 +259,4 @@ public abstract class Reporting<O> where O : class
}
Console.WriteLine();
}
/// <summary>
/// Retry Virtuoso Requests since they sometimes just fail
/// </summary>
/// <typeparam name="W"></typeparam>
/// <param name="function"></param>
/// <returns></returns>
public void WrapRequest(Action action)
{
Policy
.Handle<Exception>()
.WaitAndRetry(5, retryNumber => TimeSpan.FromMilliseconds(200))
.Execute(() => action.Invoke());
}
/// <summary>
/// Retry Virtuoso Requests since they sometimes just fail
/// </summary>
/// <typeparam name="W"></typeparam>
/// <param name="function"></param>
/// <returns></returns>
public W WrapRequest<W>(Func<W> function)
{
return Policy
.Handle<Exception>()
.WaitAndRetry(5, retryNumber => TimeSpan.FromMilliseconds(200))
.ExecuteAndCapture(() => function.Invoke()).Result;
}
}
\ No newline at end of file
......@@ -57,7 +57,7 @@ public class ApplicationProfileReporting : Reporting<ApplicationProfileReporting
OPTIONAL {{ ?{_applicationProfile} dcterms:{_license} ?{_license} . }}
}}"
};
using var result = WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(queryString.ToString()));
using var result = Helpers.WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(queryString.ToString()));
var grouped = result.GroupBy(ap => new
{
......
......@@ -164,7 +164,7 @@ public class ProjectReporting : Reporting<ProjectReportingOptions>
try
{
var usedBytes = rt.GetResourceQuotaUsed(resource.Id.ToString(), _resourceModel.GetResourceTypeOptions(resource.Id)).Result;
return Helpers.ConvertCapacityUnits(new QuotaDimObject() { Value = usedBytes, Unit = QuotaUnit.BYTE }, outputInThisUnit);
return ResourceTypes.Helpers.ConvertCapacityUnits(new QuotaDimObject() { Value = usedBytes, Unit = QuotaUnit.BYTE }, outputInThisUnit);
}
catch (Exception ex)
{
......
......@@ -135,7 +135,7 @@ public class ResourceReporting : Reporting<ResourceReportingOptions>
{
try
{
return Helpers.CreateResourceQuotaReturnObject(resource, resourceTypeDefinition);
return ResourceTypes.Helpers.CreateResourceQuotaReturnObject(resource, resourceTypeDefinition);
}
catch (Exception ex)
{
......
using Coscine.Database.DataModel;
using Coscine.Database.Models;
using Coscine.Database.ReturnObjects;
using Coscine.KpiGenerator.Utils;
using Coscine.Metadata;
using Microsoft.Extensions.Logging;
......@@ -68,11 +69,12 @@ public class UserReporting : Reporting<UserReportingOptions>
foreach (var user in users)
{
var userReturnObject = _userModel.CreateReturnObjectFromDatabaseObject(user);
var (organizations, institutes) = GetUserAffiliation(userReturnObject);
var userReportEntry = new ReturnObject
{
RelatedProjects = GetRelatedProjects(user.Id),
Organizations = GetOrganizations(user.Id, userReturnObject.Organization, "organization"),
Institutes = GetOrganizations(user.Id, userReturnObject.Institute, "institute"),
Organizations = organizations,
Institutes = institutes,
Disciplines = userReturnObject.Disciplines.ToList(),
LoginProviders = userReturnObject.ExternalAuthenticators.ToList(),
LatestActivity = GetLatestActivity(user.Id)
......@@ -131,144 +133,56 @@ public class UserReporting : Reporting<UserReportingOptions>
return result;
}
/// <summary>
/// A method that fetches the organization affiliation of a user based on a user ID
/// </summary>
/// <param name="id">Coscine User ID</param>
/// <param name="organizationLabel">Organization of the user. Will be set ONLY for ORCiD users, otherwise null.</param>
/// <returns>List of Organizations that the user belogs to. Will be empty if no organization can be found.</returns>
private List<Organization> GetOrganizations(Guid id, string organizationLabel, string searchedEntityType)
{
/* A user login has the following possibilities:
* - ORCiD only login (Organization set by the user and is found inside the SQL DB; can't confirm validity)
* - Shibboleth only login (Organization set by Shibboleth provider and is found inside Virtuoso; is regarded as valid)
* - ORCiD AND Shibboleth login (Organization set by Shibboleth provider and is found inside Virtuoso; is regarded as valid)
*
* In the case of ORCiD only login, reflect that inside report's "LoginProvides". Organization can be found only using its Display Name (not very reliable).
* In the case of Shibboleth login, one needs to find the correct organization by using the user's ExternalId and ExternalAuthenticators.
*/
var result = new List<Organization>();
private (List<Organization> organizations, List<Organization> institutes) GetUserAffiliation(UserObject user)
{
var affiliations = new List<Organization>();
var externalIds = _externalIdModel.GetAllWhere((externalId) => externalId.UserId.Equals(id));
// Bellow code taken from Organizations API
var externalIds = _externalIdModel.GetAllWhere((externalId) => externalId.UserId == user.Id);
var externalIdList = new List<string>();
foreach (var externalId in externalIds)
{
var loginProvider = _externalAuthenticatorModel.GetWhere(e => e.Id.Equals(externalId.ExternalAuthenticatorId));
if (loginProvider != null)
{
switch (loginProvider.DisplayName.ToLower())
{
case "orcid":
// Find the RoR of the organization based on its "rdfs:label".
var orgOrcid = TryGetOrganizationByLabel(organizationLabel);
if (orgOrcid is null)
{
Console.WriteLine($" No {searchedEntityType} found for user with ID \"{id}\" and login provider {loginProvider.DisplayName}");
continue;
}
result.Add(orgOrcid);
break;
case "shibboleth":
// Find the RoR first based on the user's External Id, then try based on the Organization Entity Id.
var orgShibboleth = GetOrganizationByExternalOrEntityId(externalId.ExternalId1) ?? GetOrganizationByExternalOrEntityId(externalId.Organization);
if (orgShibboleth is null)
{
Console.WriteLine($" No {searchedEntityType} found for user with ID \"{id}\" and login provider {loginProvider.DisplayName}");
continue;
}
if (orgShibboleth.RorUrl.Contains('#'))
{
var orgTopLevelRor = orgShibboleth.RorUrl[..orgShibboleth.RorUrl.IndexOf('#')];
var orgTopLevel = FetchOrganizationByRor(orgTopLevelRor);
if (!orgTopLevel.RorUrl.Equals(_otherOrganization.RorUrl))
{
result.Add(orgTopLevel);
break;
}
}
result.Add(orgShibboleth);
break;
default:
Console.WriteLine($"!! Could not verify the login provider for user with ID \"{id}\"");
break;
}
}
else
{
Console.WriteLine($"!! Could not verify the login provider for user with ID \"{id}\" and External Authenticator ID \"{externalId.ExternalAuthenticatorId}\"");
}
externalIdList.Add(externalId.ExternalId1);
}
var externalOrganizations = externalIds.Select((externalId) => externalId.Organization);
var orcidLoginProvider = _loginProviders.Single(lp => lp.DisplayName.ToLower().Equals("orcid"));
if (!string.IsNullOrWhiteSpace(organizationLabel) && !externalIds.Any(e => e.ExternalAuthenticatorId.Equals(orcidLoginProvider.Id)))
{
// Special case, a user has an organization set, but does not have an external authenticator for ODCiD. Should not be possible.
Console.WriteLine($"!! User with ID \"{id}\" has an organization set inside the Database but does not own a login provider {orcidLoginProvider.DisplayName}");
// Find the RoR of the organization based on its "rdfs:label".
var orgOrcid = TryGetOrganizationByLabel(organizationLabel);
if (orgOrcid is null)
{
Console.WriteLine($" No {searchedEntityType} found for user with ID \"{id}\" and login provider {orcidLoginProvider.DisplayName}");
}
else
var triples = Helpers.WrapRequest(() => RdfStoreConnector.GetTriples(null, null, null, 1, externalIdList).ToList());
foreach (var externalOrganization in externalOrganizations)
{
result.Add(orgOrcid);
}
triples.AddRange(Helpers.WrapRequest(() => RdfStoreConnector.GetOrganizationByEntityId(externalOrganization)));
}
triples = triples.Distinct().ToList();
// Special case until `https://ror.org/%20https://ror.org/` is moved to `https://ror.org/_other`
if (result.Any(o => o.RorUrl.Equals("https://ror.org/%20https://ror.org/")))
foreach (var triple in triples)
{
foreach (var entry in result)
{
if (entry.RorUrl.Equals("https://ror.org/%20https://ror.org/"))
affiliations.Add(new()
{
entry.Name = _otherOrganization.Name;
entry.RorUrl = _otherOrganization.RorUrl;
}
}
Name = triple.Object.ToString(),
RorUrl = triple.Subject.ToString()
});
}
return result.DistinctBy(e => e.RorUrl).ToList();
}
// Clean up
affiliations = CleanUpOther(affiliations);
private Organization? GetOrganizationByExternalOrEntityId(string externalOrEntityId)
{
var _queryString = new SparqlParameterizedString()
{
CommandText = $@"
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
PREFIX org: <http://www.w3.org/ns/org#>
// Split affiliations to "organizations" and "institutes"
var organizations = affiliations.Where(a => !a.RorUrl.Contains('#')).ToList();
var institutes = affiliations.Where(a => a.RorUrl.Contains('#')).ToList();
SELECT DISTINCT ?ror
WHERE {{
?ror rdfs:label ?name .
{{
SELECT DISTINCT ?ror
WHERE {{
?class ?p ?memberUrl ;
org:organization ?ror .
{{
SELECT DISTINCT ?memberUrl
WHERE {{
?memberUrl ?p ?value .
FILTER( ?value IN ( '{externalOrEntityId}' ))
}}
}}
}}
}}
}}
"
};
using var results = WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(_queryString.ToString()));
if (!results.IsEmpty)
{
var ror = results.Select(x => x.Value("ror").ToString()); // Get the value for ?ror
if (ror.Any())
{
return FetchOrganizationByRor(ror.ToList()[0]);
}
}
return null;
// Find the RoR of the organization based on its "rdfs:label".
var orgFromSqlDb = TryGetOrganizationByLabel(user.Organization);
if (orgFromSqlDb is not null)
organizations.Add(orgFromSqlDb);
// Find the RoR of the institute based on its "rdfs:label".
var instFromSqlDb = TryGetOrganizationByLabel(user.Institute);
if (instFromSqlDb is not null)
institutes.Add(instFromSqlDb);
return (
organizations.DistinctBy(o => o.RorUrl).ToList(),
institutes.DistinctBy(i => i.RorUrl).ToList()
);
}
private Organization? TryGetOrganizationByLabel(string rdfsLabel)
......@@ -293,7 +207,7 @@ public class UserReporting : Reporting<UserReportingOptions>
}}
"
};
using var results = WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(_queryString.ToString()));
using var results = Helpers.WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(_queryString.ToString()));
if (!results.IsEmpty)
{
var ror = results.Select(x => x.Value("ror").ToString()); // Get the value for ?ror
......@@ -307,10 +221,6 @@ public class UserReporting : Reporting<UserReportingOptions>
private DateTime? GetLatestActivity(Guid id)
{
/*
* TODO: Query may take ages to execute and can lead to timeouts. Make sure to create an index!
* CREATE INDEX log_idx_userid_loglevel_servertime ON Coscine.dbo.Log (UserId, LogLevel, ServerTimestamp);
*/
var latestLog = _logModel.GetAllWhere(l => l.LogLevel.Equals("Analytics") && l.UserId.Equals(id)).OrderByDescending(a => a.ServerTimestamp).FirstOrDefault();
if (latestLog is not null)
{
......@@ -318,4 +228,26 @@ public class UserReporting : Reporting<UserReportingOptions>
}
else return null;
}
private List<Organization> CleanUpOther(List<Organization> affiliations)
{
// Special case until `https://ror.org/%20https://ror.org/` is moved to `https://ror.org/_other`
if (affiliations.Any(o => o.RorUrl.Equals("https://ror.org/%20https://ror.org/")))
{
var cleanedUpAffiliations = affiliations;
foreach (var entry in cleanedUpAffiliations)
{
if (entry.RorUrl.Equals("https://ror.org/%20https://ror.org/"))
{
cleanedUpAffiliations[cleanedUpAffiliations.IndexOf(entry)] = new()
{
Name = _otherOrganization.Name,
RorUrl = _otherOrganization.RorUrl
};
}
}
return cleanedUpAffiliations;
}
return affiliations;
}
}
\ No newline at end of file
using Polly;
namespace Coscine.KpiGenerator.Utils
{
public static class Helpers
{
public static W WrapRequest<W>(Func<W> function)
{
return Policy
.Handle<Exception>()
.WaitAndRetry(5, retryNumber => TimeSpan.FromMilliseconds(200))
.ExecuteAndCapture(() => function.Invoke()).Result;
}
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment