Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found
Select Git revision

Target

Select target project
  • coscine/backend/scripts/kpi-generator
1 result
Select Git revision
Show changes
Commits on Source (2)
Show whitespace changes
Inline Side-by-side
Showing
with 82 additions and 165 deletions
src/KPI Generator/KPI Generator.csproj
View file @ edca70bc
......@@ -7,7 +7,7 @@
<AssemblyName>Coscine.KpiGenerator</AssemblyName>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<Version>0.1.12</Version>
<Version>0.1.13</Version>
</PropertyGroup>
<PropertyGroup>
......
src/KPI Generator/Reporting.cs
View file @ edca70bc
......@@ -6,7 +6,6 @@ using GitLabApiClient.Models.Branches.Requests;
using GitLabApiClient.Models.Commits.Requests.CreateCommitRequest;
using Microsoft.Extensions.Logging;
using NLog.Extensions.Logging;
using Polly;
using System.Text;
using System.Web;
using static KPIGenerator.Utils.CommandLineOptions;
......@@ -172,7 +171,7 @@ public abstract class Reporting<O> where O : class
}
else
{
var organizationTriples = WrapRequest(() => RdfStoreConnector.GetLabelForSubject(new Uri(Uri.UnescapeDataString(rorUrl))).ToList());
var organizationTriples = Helpers.WrapRequest(() => RdfStoreConnector.GetLabelForSubject(new Uri(Uri.UnescapeDataString(rorUrl))).ToList());
if (organizationTriples is not null && organizationTriples.Any())
{
result = new Organization
......@@ -260,33 +259,4 @@ public abstract class Reporting<O> where O : class
}
Console.WriteLine();
}
/// <summary>
/// Retry Virtuoso Requests since they sometimes just fail
/// </summary>
/// <typeparam name="W"></typeparam>
/// <param name="function"></param>
/// <returns></returns>
public void WrapRequest(Action action)
{
Policy
.Handle<Exception>()
.WaitAndRetry(5, retryNumber => TimeSpan.FromMilliseconds(200))
.Execute(() => action.Invoke());
}
/// <summary>
/// Retry Virtuoso Requests since they sometimes just fail
/// </summary>
/// <typeparam name="W"></typeparam>
/// <param name="function"></param>
/// <returns></returns>
public W WrapRequest<W>(Func<W> function)
{
return Policy
.Handle<Exception>()
.WaitAndRetry(5, retryNumber => TimeSpan.FromMilliseconds(200))
.ExecuteAndCapture(() => function.Invoke()).Result;
}
}
\ No newline at end of file
src/KPI Generator/Reportings/ApplicationProfile/ApplicationProfileReporting.cs
View file @ edca70bc
......@@ -57,7 +57,7 @@ public class ApplicationProfileReporting : Reporting<ApplicationProfileReporting
OPTIONAL {{ ?{_applicationProfile} dcterms:{_license} ?{_license} . }}
}}"
};
using var result = WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(queryString.ToString()));
using var result = Helpers.WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(queryString.ToString()));
var grouped = result.GroupBy(ap => new
{
......
src/KPI Generator/Reportings/Project/ProjectReporting.cs
View file @ edca70bc
......@@ -164,7 +164,7 @@ public class ProjectReporting : Reporting<ProjectReportingOptions>
try
{
var usedBytes = rt.GetResourceQuotaUsed(resource.Id.ToString(), _resourceModel.GetResourceTypeOptions(resource.Id)).Result;
return Helpers.ConvertCapacityUnits(new QuotaDimObject() { Value = usedBytes, Unit = QuotaUnit.BYTE }, outputInThisUnit);
return ResourceTypes.Helpers.ConvertCapacityUnits(new QuotaDimObject() { Value = usedBytes, Unit = QuotaUnit.BYTE }, outputInThisUnit);
}
catch (Exception ex)
{
......
src/KPI Generator/Reportings/Resource/ResourceReporting.cs
View file @ edca70bc
......@@ -135,7 +135,7 @@ public class ResourceReporting : Reporting<ResourceReportingOptions>
{
try
{
return Helpers.CreateResourceQuotaReturnObject(resource, resourceTypeDefinition);
return ResourceTypes.Helpers.CreateResourceQuotaReturnObject(resource, resourceTypeDefinition);
}
catch (Exception ex)
{
......
src/KPI Generator/Reportings/User/UserReporting.cs
View file @ edca70bc
using Coscine.Database.DataModel;
using Coscine.Database.Models;
using Coscine.Database.ReturnObjects;
using Coscine.KpiGenerator.Utils;
using Coscine.Metadata;
using Microsoft.Extensions.Logging;
......@@ -68,11 +69,12 @@ public class UserReporting : Reporting<UserReportingOptions>
foreach (var user in users)
{
var userReturnObject = _userModel.CreateReturnObjectFromDatabaseObject(user);
var (organizations, institutes) = GetUserAffiliation(userReturnObject);
var userReportEntry = new ReturnObject
{
RelatedProjects = GetRelatedProjects(user.Id),
Organizations = GetOrganizations(user.Id, userReturnObject.Organization, "organization"),
Institutes = GetOrganizations(user.Id, userReturnObject.Institute, "institute"),
Organizations = organizations,
Institutes = institutes,
Disciplines = userReturnObject.Disciplines.ToList(),
LoginProviders = userReturnObject.ExternalAuthenticators.ToList(),
LatestActivity = GetLatestActivity(user.Id)
......@@ -131,144 +133,56 @@ public class UserReporting : Reporting<UserReportingOptions>
return result;
}
/// <summary>
/// A method that fetches the organization affiliation of a user based on a user ID
/// </summary>
/// <param name="id">Coscine User ID</param>
/// <param name="organizationLabel">Organization of the user. Will be set ONLY for ORCiD users, otherwise null.</param>
/// <returns>List of Organizations that the user belogs to. Will be empty if no organization can be found.</returns>
private List<Organization> GetOrganizations(Guid id, string organizationLabel, string searchedEntityType)
{
/* A user login has the following possibilities:
* - ORCiD only login (Organization set by the user and is found inside the SQL DB; can't confirm validity)
* - Shibboleth only login (Organization set by Shibboleth provider and is found inside Virtuoso; is regarded as valid)
* - ORCiD AND Shibboleth login (Organization set by Shibboleth provider and is found inside Virtuoso; is regarded as valid)
*
* In the case of ORCiD only login, reflect that inside report's "LoginProvides". Organization can be found only using its Display Name (not very reliable).
* In the case of Shibboleth login, one needs to find the correct organization by using the user's ExternalId and ExternalAuthenticators.
*/
var result = new List<Organization>();
private (List<Organization> organizations, List<Organization> institutes) GetUserAffiliation(UserObject user)
{
var affiliations = new List<Organization>();
var externalIds = _externalIdModel.GetAllWhere((externalId) => externalId.UserId.Equals(id));
// Bellow code taken from Organizations API
var externalIds = _externalIdModel.GetAllWhere((externalId) => externalId.UserId == user.Id);
var externalIdList = new List<string>();
foreach (var externalId in externalIds)
{
var loginProvider = _externalAuthenticatorModel.GetWhere(e => e.Id.Equals(externalId.ExternalAuthenticatorId));
if (loginProvider != null)
{
switch (loginProvider.DisplayName.ToLower())
{
case "orcid":
// Find the RoR of the organization based on its "rdfs:label".
var orgOrcid = TryGetOrganizationByLabel(organizationLabel);
if (orgOrcid is null)
{
Console.WriteLine($" No {searchedEntityType} found for user with ID \"{id}\" and login provider {loginProvider.DisplayName}");
continue;
}
result.Add(orgOrcid);
break;
case "shibboleth":
// Find the RoR first based on the user's External Id, then try based on the Organization Entity Id.
var orgShibboleth = GetOrganizationByExternalOrEntityId(externalId.ExternalId1) ?? GetOrganizationByExternalOrEntityId(externalId.Organization);
if (orgShibboleth is null)
{
Console.WriteLine($" No {searchedEntityType} found for user with ID \"{id}\" and login provider {loginProvider.DisplayName}");
continue;
}
if (orgShibboleth.RorUrl.Contains('#'))
{
var orgTopLevelRor = orgShibboleth.RorUrl[..orgShibboleth.RorUrl.IndexOf('#')];
var orgTopLevel = FetchOrganizationByRor(orgTopLevelRor);
if (!orgTopLevel.RorUrl.Equals(_otherOrganization.RorUrl))
{
result.Add(orgTopLevel);
break;
}
}
result.Add(orgShibboleth);
break;
default:
Console.WriteLine($"!! Could not verify the login provider for user with ID \"{id}\"");
break;
}
}
else
{
Console.WriteLine($"!! Could not verify the login provider for user with ID \"{id}\" and External Authenticator ID \"{externalId.ExternalAuthenticatorId}\"");
}
externalIdList.Add(externalId.ExternalId1);
}
var externalOrganizations = externalIds.Select((externalId) => externalId.Organization);
var orcidLoginProvider = _loginProviders.Single(lp => lp.DisplayName.ToLower().Equals("orcid"));
if (!string.IsNullOrWhiteSpace(organizationLabel) && !externalIds.Any(e => e.ExternalAuthenticatorId.Equals(orcidLoginProvider.Id)))
{
// Special case, a user has an organization set, but does not have an external authenticator for ODCiD. Should not be possible.
Console.WriteLine($"!! User with ID \"{id}\" has an organization set inside the Database but does not own a login provider {orcidLoginProvider.DisplayName}");
// Find the RoR of the organization based on its "rdfs:label".
var orgOrcid = TryGetOrganizationByLabel(organizationLabel);
if (orgOrcid is null)
{
Console.WriteLine($" No {searchedEntityType} found for user with ID \"{id}\" and login provider {orcidLoginProvider.DisplayName}");
}
else
var triples = Helpers.WrapRequest(() => RdfStoreConnector.GetTriples(null, null, null, 1, externalIdList).ToList());
foreach (var externalOrganization in externalOrganizations)
{
result.Add(orgOrcid);
}
triples.AddRange(Helpers.WrapRequest(() => RdfStoreConnector.GetOrganizationByEntityId(externalOrganization)));
}
triples = triples.Distinct().ToList();
// Special case until `https://ror.org/%20https://ror.org/` is moved to `https://ror.org/_other`
if (result.Any(o => o.RorUrl.Equals("https://ror.org/%20https://ror.org/")))
foreach (var triple in triples)
{
foreach (var entry in result)
{
if (entry.RorUrl.Equals("https://ror.org/%20https://ror.org/"))
affiliations.Add(new()
{
entry.Name = _otherOrganization.Name;
entry.RorUrl = _otherOrganization.RorUrl;
}
}
Name = triple.Object.ToString(),
RorUrl = triple.Subject.ToString()
});
}
return result.DistinctBy(e => e.RorUrl).ToList();
}
// Clean up
affiliations = CleanUpOther(affiliations);
private Organization? GetOrganizationByExternalOrEntityId(string externalOrEntityId)
{
var _queryString = new SparqlParameterizedString()
{
CommandText = $@"
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
PREFIX org: <http://www.w3.org/ns/org#>
// Split affiliations to "organizations" and "institutes"
var organizations = affiliations.Where(a => !a.RorUrl.Contains('#')).ToList();
var institutes = affiliations.Where(a => a.RorUrl.Contains('#')).ToList();
SELECT DISTINCT ?ror
WHERE {{
?ror rdfs:label ?name .
{{
SELECT DISTINCT ?ror
WHERE {{
?class ?p ?memberUrl ;
org:organization ?ror .
{{
SELECT DISTINCT ?memberUrl
WHERE {{
?memberUrl ?p ?value .
FILTER( ?value IN ( '{externalOrEntityId}' ))
}}
}}
}}
}}
}}
"
};
using var results = WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(_queryString.ToString()));
if (!results.IsEmpty)
{
var ror = results.Select(x => x.Value("ror").ToString()); // Get the value for ?ror
if (ror.Any())
{
return FetchOrganizationByRor(ror.ToList()[0]);
}
}
return null;
// Find the RoR of the organization based on its "rdfs:label".
var orgFromSqlDb = TryGetOrganizationByLabel(user.Organization);
if (orgFromSqlDb is not null)
organizations.Add(orgFromSqlDb);
// Find the RoR of the institute based on its "rdfs:label".
var instFromSqlDb = TryGetOrganizationByLabel(user.Institute);
if (instFromSqlDb is not null)
institutes.Add(instFromSqlDb);
return (
organizations.DistinctBy(o => o.RorUrl).ToList(),
institutes.DistinctBy(i => i.RorUrl).ToList()
);
}
private Organization? TryGetOrganizationByLabel(string rdfsLabel)
......@@ -293,7 +207,7 @@ public class UserReporting : Reporting<UserReportingOptions>
}}
"
};
using var results = WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(_queryString.ToString()));
using var results = Helpers.WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(_queryString.ToString()));
if (!results.IsEmpty)
{
var ror = results.Select(x => x.Value("ror").ToString()); // Get the value for ?ror
......@@ -307,10 +221,6 @@ public class UserReporting : Reporting<UserReportingOptions>
private DateTime? GetLatestActivity(Guid id)
{
/*
* TODO: Query may take ages to execute and can lead to timeouts. Make sure to create an index!
* CREATE INDEX log_idx_userid_loglevel_servertime ON Coscine.dbo.Log (UserId, LogLevel, ServerTimestamp);
*/
var latestLog = _logModel.GetAllWhere(l => l.LogLevel.Equals("Analytics") && l.UserId.Equals(id)).OrderByDescending(a => a.ServerTimestamp).FirstOrDefault();
if (latestLog is not null)
{
......@@ -318,4 +228,26 @@ public class UserReporting : Reporting<UserReportingOptions>
}
else return null;
}
private List<Organization> CleanUpOther(List<Organization> affiliations)
{
// Special case until `https://ror.org/%20https://ror.org/` is moved to `https://ror.org/_other`
if (affiliations.Any(o => o.RorUrl.Equals("https://ror.org/%20https://ror.org/")))
{
var cleanedUpAffiliations = affiliations;
foreach (var entry in cleanedUpAffiliations)
{
if (entry.RorUrl.Equals("https://ror.org/%20https://ror.org/"))
{
cleanedUpAffiliations[cleanedUpAffiliations.IndexOf(entry)] = new()
{
Name = _otherOrganization.Name,
RorUrl = _otherOrganization.RorUrl
};
}
}
return cleanedUpAffiliations;
}
return affiliations;
}
}
\ No newline at end of file
src/KPI Generator/Utils/Helpers.cs 0 → 100644
View file @ edca70bc
using Polly;
namespace Coscine.KpiGenerator.Utils
{
public static class Helpers
{
public static W WrapRequest<W>(Func<W> function)
{
return Policy
.Handle<Exception>()
.WaitAndRetry(5, retryNumber => TimeSpan.FromMilliseconds(200))
.ExecuteAndCapture(() => function.Invoke()).Result;
}
}
}