Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found
Select Git revision
  • Fix/xxxx-indexOutOfRange
  • Fix/xxxx-minorFixes
  • Fix/xxxx-organization
  • Fix/xxxx-wrap
  • Hotfix/2332-userInstitutesInReporting
  • Hotfix/2388-sensitive
  • Hotfix/3115-userReportingEmpty
  • Hotfix/3115-userReportingEmpty2
  • Hotfix/xxxx-rors
  • Issue/2181-kpiGeneratorBase
  • Issue/2182-kpiGeneratorUser
  • Issue/2183-kpiGeneratorResource
  • Issue/2184-kpiGeneratorProject
  • Issue/2185-kpiGeneratorAP
  • Issue/2186-systemStatusReporting
  • Issue/2283-activityFix
  • Issue/2304-virtuosoRoars
  • Issue/2330-fixNaNQuotainAdmin
  • Issue/2432-publicationKpi
  • Issue/2492-respOrg
  • Issue/2518-docs
  • Issue/2568-betterLogging
  • Issue/2666-adminCronjobs
  • Issue/2666-adminCronjobs-theSequal
  • Issue/2847-reporting
  • Issue/2850-removeGrantId
  • Issue/2982-kpiDataPub
  • Issue/3005-kpiReportingBroken
  • Issue/3073-kpi
  • Issue/3142-kpiGenerator
  • dev
  • gitkeep
  • main
  • v0.1.0
  • v0.1.1
  • v0.1.10
  • v0.1.11
  • v0.1.12
  • v0.1.13
  • v0.1.14
  • v0.1.15
  • v0.1.16
  • v0.1.17
  • v0.1.18
  • v0.1.19
  • v0.1.2
  • v0.1.20
  • v0.1.21
  • v0.1.22
  • v0.1.23
  • v0.1.3
  • v0.1.4
  • v0.1.5
  • v0.1.6
  • v0.1.7
  • v0.1.8
  • v0.1.9
  • v1.0.1
  • v1.0.2
  • v1.0.3
  • v1.0.4
  • v1.0.5
  • v1.0.6
  • v1.0.7
  • v1.0.8
  • v1.0.9
  • v1.1.0
  • v1.1.1
  • v1.2.0
  • v1.2.1
  • v1.2.10
  • v1.2.2
  • v1.2.3
  • v1.2.4
  • v1.2.5
  • v1.2.6
  • v1.2.7
  • v1.2.8
  • v1.2.9
79 results

Target

Select target project
  • coscine/backend/scripts/kpi-generator
1 result
Select Git revision
  • Fix/xxxx-indexOutOfRange
  • Fix/xxxx-minorFixes
  • Fix/xxxx-organization
  • Fix/xxxx-wrap
  • Hotfix/2332-userInstitutesInReporting
  • Hotfix/2388-sensitive
  • Hotfix/3115-userReportingEmpty
  • Hotfix/3115-userReportingEmpty2
  • Hotfix/xxxx-rors
  • Issue/2181-kpiGeneratorBase
  • Issue/2182-kpiGeneratorUser
  • Issue/2183-kpiGeneratorResource
  • Issue/2184-kpiGeneratorProject
  • Issue/2185-kpiGeneratorAP
  • Issue/2186-systemStatusReporting
  • Issue/2283-activityFix
  • Issue/2304-virtuosoRoars
  • Issue/2330-fixNaNQuotainAdmin
  • Issue/2432-publicationKpi
  • Issue/2492-respOrg
  • Issue/2518-docs
  • Issue/2568-betterLogging
  • Issue/2666-adminCronjobs
  • Issue/2666-adminCronjobs-theSequal
  • Issue/2847-reporting
  • Issue/2850-removeGrantId
  • Issue/2982-kpiDataPub
  • Issue/3005-kpiReportingBroken
  • Issue/3073-kpi
  • Issue/3142-kpiGenerator
  • dev
  • gitkeep
  • main
  • v0.1.0
  • v0.1.1
  • v0.1.10
  • v0.1.11
  • v0.1.12
  • v0.1.13
  • v0.1.14
  • v0.1.15
  • v0.1.16
  • v0.1.17
  • v0.1.18
  • v0.1.19
  • v0.1.2
  • v0.1.20
  • v0.1.21
  • v0.1.22
  • v0.1.23
  • v0.1.3
  • v0.1.4
  • v0.1.5
  • v0.1.6
  • v0.1.7
  • v0.1.8
  • v0.1.9
  • v1.0.1
  • v1.0.2
  • v1.0.3
  • v1.0.4
  • v1.0.5
  • v1.0.6
  • v1.0.7
  • v1.0.8
  • v1.0.9
  • v1.1.0
  • v1.1.1
  • v1.2.0
  • v1.2.1
  • v1.2.10
  • v1.2.2
  • v1.2.3
  • v1.2.4
  • v1.2.5
  • v1.2.6
  • v1.2.7
  • v1.2.8
  • v1.2.9
79 results
Show changes

Commits on Source 2

...@@ -7,7 +7,7 @@ ...@@ -7,7 +7,7 @@
<AssemblyName>Coscine.KpiGenerator</AssemblyName> <AssemblyName>Coscine.KpiGenerator</AssemblyName>
<ImplicitUsings>enable</ImplicitUsings> <ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable> <Nullable>enable</Nullable>
<Version>0.1.12</Version> <Version>0.1.13</Version>
</PropertyGroup> </PropertyGroup>
<PropertyGroup> <PropertyGroup>
......
...@@ -6,7 +6,6 @@ using GitLabApiClient.Models.Branches.Requests; ...@@ -6,7 +6,6 @@ using GitLabApiClient.Models.Branches.Requests;
using GitLabApiClient.Models.Commits.Requests.CreateCommitRequest; using GitLabApiClient.Models.Commits.Requests.CreateCommitRequest;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using NLog.Extensions.Logging; using NLog.Extensions.Logging;
using Polly;
using System.Text; using System.Text;
using System.Web; using System.Web;
using static KPIGenerator.Utils.CommandLineOptions; using static KPIGenerator.Utils.CommandLineOptions;
...@@ -172,7 +171,7 @@ public abstract class Reporting<O> where O : class ...@@ -172,7 +171,7 @@ public abstract class Reporting<O> where O : class
} }
else else
{ {
var organizationTriples = WrapRequest(() => RdfStoreConnector.GetLabelForSubject(new Uri(Uri.UnescapeDataString(rorUrl))).ToList()); var organizationTriples = Helpers.WrapRequest(() => RdfStoreConnector.GetLabelForSubject(new Uri(Uri.UnescapeDataString(rorUrl))).ToList());
if (organizationTriples is not null && organizationTriples.Any()) if (organizationTriples is not null && organizationTriples.Any())
{ {
result = new Organization result = new Organization
...@@ -260,33 +259,4 @@ public abstract class Reporting<O> where O : class ...@@ -260,33 +259,4 @@ public abstract class Reporting<O> where O : class
} }
Console.WriteLine(); Console.WriteLine();
} }
/// <summary>
/// Retry Virtuoso Requests since they sometimes just fail
/// </summary>
/// <typeparam name="W"></typeparam>
/// <param name="function"></param>
/// <returns></returns>
public void WrapRequest(Action action)
{
Policy
.Handle<Exception>()
.WaitAndRetry(5, retryNumber => TimeSpan.FromMilliseconds(200))
.Execute(() => action.Invoke());
}
/// <summary>
/// Retry Virtuoso Requests since they sometimes just fail
/// </summary>
/// <typeparam name="W"></typeparam>
/// <param name="function"></param>
/// <returns></returns>
public W WrapRequest<W>(Func<W> function)
{
return Policy
.Handle<Exception>()
.WaitAndRetry(5, retryNumber => TimeSpan.FromMilliseconds(200))
.ExecuteAndCapture(() => function.Invoke()).Result;
}
} }
\ No newline at end of file
...@@ -57,7 +57,7 @@ public class ApplicationProfileReporting : Reporting<ApplicationProfileReporting ...@@ -57,7 +57,7 @@ public class ApplicationProfileReporting : Reporting<ApplicationProfileReporting
OPTIONAL {{ ?{_applicationProfile} dcterms:{_license} ?{_license} . }} OPTIONAL {{ ?{_applicationProfile} dcterms:{_license} ?{_license} . }}
}}" }}"
}; };
using var result = WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(queryString.ToString())); using var result = Helpers.WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(queryString.ToString()));
var grouped = result.GroupBy(ap => new var grouped = result.GroupBy(ap => new
{ {
......
...@@ -164,7 +164,7 @@ public class ProjectReporting : Reporting<ProjectReportingOptions> ...@@ -164,7 +164,7 @@ public class ProjectReporting : Reporting<ProjectReportingOptions>
try try
{ {
var usedBytes = rt.GetResourceQuotaUsed(resource.Id.ToString(), _resourceModel.GetResourceTypeOptions(resource.Id)).Result; var usedBytes = rt.GetResourceQuotaUsed(resource.Id.ToString(), _resourceModel.GetResourceTypeOptions(resource.Id)).Result;
return Helpers.ConvertCapacityUnits(new QuotaDimObject() { Value = usedBytes, Unit = QuotaUnit.BYTE }, outputInThisUnit); return ResourceTypes.Helpers.ConvertCapacityUnits(new QuotaDimObject() { Value = usedBytes, Unit = QuotaUnit.BYTE }, outputInThisUnit);
} }
catch (Exception ex) catch (Exception ex)
{ {
......
...@@ -135,7 +135,7 @@ public class ResourceReporting : Reporting<ResourceReportingOptions> ...@@ -135,7 +135,7 @@ public class ResourceReporting : Reporting<ResourceReportingOptions>
{ {
try try
{ {
return Helpers.CreateResourceQuotaReturnObject(resource, resourceTypeDefinition); return ResourceTypes.Helpers.CreateResourceQuotaReturnObject(resource, resourceTypeDefinition);
} }
catch (Exception ex) catch (Exception ex)
{ {
......
using Coscine.Database.DataModel; using Coscine.Database.DataModel;
using Coscine.Database.Models; using Coscine.Database.Models;
using Coscine.Database.ReturnObjects;
using Coscine.KpiGenerator.Utils; using Coscine.KpiGenerator.Utils;
using Coscine.Metadata; using Coscine.Metadata;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
...@@ -68,11 +69,12 @@ public class UserReporting : Reporting<UserReportingOptions> ...@@ -68,11 +69,12 @@ public class UserReporting : Reporting<UserReportingOptions>
foreach (var user in users) foreach (var user in users)
{ {
var userReturnObject = _userModel.CreateReturnObjectFromDatabaseObject(user); var userReturnObject = _userModel.CreateReturnObjectFromDatabaseObject(user);
var (organizations, institutes) = GetUserAffiliation(userReturnObject);
var userReportEntry = new ReturnObject var userReportEntry = new ReturnObject
{ {
RelatedProjects = GetRelatedProjects(user.Id), RelatedProjects = GetRelatedProjects(user.Id),
Organizations = GetOrganizations(user.Id, userReturnObject.Organization, "organization"), Organizations = organizations,
Institutes = GetOrganizations(user.Id, userReturnObject.Institute, "institute"), Institutes = institutes,
Disciplines = userReturnObject.Disciplines.ToList(), Disciplines = userReturnObject.Disciplines.ToList(),
LoginProviders = userReturnObject.ExternalAuthenticators.ToList(), LoginProviders = userReturnObject.ExternalAuthenticators.ToList(),
LatestActivity = GetLatestActivity(user.Id) LatestActivity = GetLatestActivity(user.Id)
...@@ -131,144 +133,56 @@ public class UserReporting : Reporting<UserReportingOptions> ...@@ -131,144 +133,56 @@ public class UserReporting : Reporting<UserReportingOptions>
return result; return result;
} }
/// <summary> private (List<Organization> organizations, List<Organization> institutes) GetUserAffiliation(UserObject user)
/// A method that fetches the organization affiliation of a user based on a user ID {
/// </summary> var affiliations = new List<Organization>();
/// <param name="id">Coscine User ID</param>
/// <param name="organizationLabel">Organization of the user. Will be set ONLY for ORCiD users, otherwise null.</param>
/// <returns>List of Organizations that the user belogs to. Will be empty if no organization can be found.</returns>
private List<Organization> GetOrganizations(Guid id, string organizationLabel, string searchedEntityType)
{
/* A user login has the following possibilities:
* - ORCiD only login (Organization set by the user and is found inside the SQL DB; can't confirm validity)
* - Shibboleth only login (Organization set by Shibboleth provider and is found inside Virtuoso; is regarded as valid)
* - ORCiD AND Shibboleth login (Organization set by Shibboleth provider and is found inside Virtuoso; is regarded as valid)
*
* In the case of ORCiD only login, reflect that inside report's "LoginProvides". Organization can be found only using its Display Name (not very reliable).
* In the case of Shibboleth login, one needs to find the correct organization by using the user's ExternalId and ExternalAuthenticators.
*/
var result = new List<Organization>();
var externalIds = _externalIdModel.GetAllWhere((externalId) => externalId.UserId.Equals(id)); // Bellow code taken from Organizations API
var externalIds = _externalIdModel.GetAllWhere((externalId) => externalId.UserId == user.Id);
var externalIdList = new List<string>();
foreach (var externalId in externalIds) foreach (var externalId in externalIds)
{ {
var loginProvider = _externalAuthenticatorModel.GetWhere(e => e.Id.Equals(externalId.ExternalAuthenticatorId)); externalIdList.Add(externalId.ExternalId1);
if (loginProvider != null)
{
switch (loginProvider.DisplayName.ToLower())
{
case "orcid":
// Find the RoR of the organization based on its "rdfs:label".
var orgOrcid = TryGetOrganizationByLabel(organizationLabel);
if (orgOrcid is null)
{
Console.WriteLine($" No {searchedEntityType} found for user with ID \"{id}\" and login provider {loginProvider.DisplayName}");
continue;
}
result.Add(orgOrcid);
break;
case "shibboleth":
// Find the RoR first based on the user's External Id, then try based on the Organization Entity Id.
var orgShibboleth = GetOrganizationByExternalOrEntityId(externalId.ExternalId1) ?? GetOrganizationByExternalOrEntityId(externalId.Organization);
if (orgShibboleth is null)
{
Console.WriteLine($" No {searchedEntityType} found for user with ID \"{id}\" and login provider {loginProvider.DisplayName}");
continue;
}
if (orgShibboleth.RorUrl.Contains('#'))
{
var orgTopLevelRor = orgShibboleth.RorUrl[..orgShibboleth.RorUrl.IndexOf('#')];
var orgTopLevel = FetchOrganizationByRor(orgTopLevelRor);
if (!orgTopLevel.RorUrl.Equals(_otherOrganization.RorUrl))
{
result.Add(orgTopLevel);
break;
}
}
result.Add(orgShibboleth);
break;
default:
Console.WriteLine($"!! Could not verify the login provider for user with ID \"{id}\"");
break;
}
}
else
{
Console.WriteLine($"!! Could not verify the login provider for user with ID \"{id}\" and External Authenticator ID \"{externalId.ExternalAuthenticatorId}\"");
}
} }
var externalOrganizations = externalIds.Select((externalId) => externalId.Organization);
var orcidLoginProvider = _loginProviders.Single(lp => lp.DisplayName.ToLower().Equals("orcid")); var triples = Helpers.WrapRequest(() => RdfStoreConnector.GetTriples(null, null, null, 1, externalIdList).ToList());
if (!string.IsNullOrWhiteSpace(organizationLabel) && !externalIds.Any(e => e.ExternalAuthenticatorId.Equals(orcidLoginProvider.Id))) foreach (var externalOrganization in externalOrganizations)
{
// Special case, a user has an organization set, but does not have an external authenticator for ODCiD. Should not be possible.
Console.WriteLine($"!! User with ID \"{id}\" has an organization set inside the Database but does not own a login provider {orcidLoginProvider.DisplayName}");
// Find the RoR of the organization based on its "rdfs:label".
var orgOrcid = TryGetOrganizationByLabel(organizationLabel);
if (orgOrcid is null)
{
Console.WriteLine($" No {searchedEntityType} found for user with ID \"{id}\" and login provider {orcidLoginProvider.DisplayName}");
}
else
{ {
result.Add(orgOrcid); triples.AddRange(Helpers.WrapRequest(() => RdfStoreConnector.GetOrganizationByEntityId(externalOrganization)));
}
} }
triples = triples.Distinct().ToList();
// Special case until `https://ror.org/%20https://ror.org/` is moved to `https://ror.org/_other` foreach (var triple in triples)
if (result.Any(o => o.RorUrl.Equals("https://ror.org/%20https://ror.org/")))
{ {
foreach (var entry in result) affiliations.Add(new()
{
if (entry.RorUrl.Equals("https://ror.org/%20https://ror.org/"))
{ {
entry.Name = _otherOrganization.Name; Name = triple.Object.ToString(),
entry.RorUrl = _otherOrganization.RorUrl; RorUrl = triple.Subject.ToString()
} });
}
} }
return result.DistinctBy(e => e.RorUrl).ToList(); // Clean up
} affiliations = CleanUpOther(affiliations);
private Organization? GetOrganizationByExternalOrEntityId(string externalOrEntityId) // Split affiliations to "organizations" and "institutes"
{ var organizations = affiliations.Where(a => !a.RorUrl.Contains('#')).ToList();
var _queryString = new SparqlParameterizedString() var institutes = affiliations.Where(a => a.RorUrl.Contains('#')).ToList();
{
CommandText = $@"
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
PREFIX org: <http://www.w3.org/ns/org#>
SELECT DISTINCT ?ror // Find the RoR of the organization based on its "rdfs:label".
WHERE {{ var orgFromSqlDb = TryGetOrganizationByLabel(user.Organization);
?ror rdfs:label ?name . if (orgFromSqlDb is not null)
{{ organizations.Add(orgFromSqlDb);
SELECT DISTINCT ?ror
WHERE {{ // Find the RoR of the institute based on its "rdfs:label".
?class ?p ?memberUrl ; var instFromSqlDb = TryGetOrganizationByLabel(user.Institute);
org:organization ?ror . if (instFromSqlDb is not null)
{{ institutes.Add(instFromSqlDb);
SELECT DISTINCT ?memberUrl
WHERE {{ return (
?memberUrl ?p ?value . organizations.DistinctBy(o => o.RorUrl).ToList(),
FILTER( ?value IN ( '{externalOrEntityId}' )) institutes.DistinctBy(i => i.RorUrl).ToList()
}} );
}}
}}
}}
}}
"
};
using var results = WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(_queryString.ToString()));
if (!results.IsEmpty)
{
var ror = results.Select(x => x.Value("ror").ToString()); // Get the value for ?ror
if (ror.Any())
{
return FetchOrganizationByRor(ror.ToList()[0]);
}
}
return null;
} }
private Organization? TryGetOrganizationByLabel(string rdfsLabel) private Organization? TryGetOrganizationByLabel(string rdfsLabel)
...@@ -293,7 +207,7 @@ public class UserReporting : Reporting<UserReportingOptions> ...@@ -293,7 +207,7 @@ public class UserReporting : Reporting<UserReportingOptions>
}} }}
" "
}; };
using var results = WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(_queryString.ToString())); using var results = Helpers.WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(_queryString.ToString()));
if (!results.IsEmpty) if (!results.IsEmpty)
{ {
var ror = results.Select(x => x.Value("ror").ToString()); // Get the value for ?ror var ror = results.Select(x => x.Value("ror").ToString()); // Get the value for ?ror
...@@ -307,10 +221,6 @@ public class UserReporting : Reporting<UserReportingOptions> ...@@ -307,10 +221,6 @@ public class UserReporting : Reporting<UserReportingOptions>
private DateTime? GetLatestActivity(Guid id) private DateTime? GetLatestActivity(Guid id)
{ {
/*
* TODO: Query may take ages to execute and can lead to timeouts. Make sure to create an index!
* CREATE INDEX log_idx_userid_loglevel_servertime ON Coscine.dbo.Log (UserId, LogLevel, ServerTimestamp);
*/
var latestLog = _logModel.GetAllWhere(l => l.LogLevel.Equals("Analytics") && l.UserId.Equals(id)).OrderByDescending(a => a.ServerTimestamp).FirstOrDefault(); var latestLog = _logModel.GetAllWhere(l => l.LogLevel.Equals("Analytics") && l.UserId.Equals(id)).OrderByDescending(a => a.ServerTimestamp).FirstOrDefault();
if (latestLog is not null) if (latestLog is not null)
{ {
...@@ -318,4 +228,26 @@ public class UserReporting : Reporting<UserReportingOptions> ...@@ -318,4 +228,26 @@ public class UserReporting : Reporting<UserReportingOptions>
} }
else return null; else return null;
} }
private List<Organization> CleanUpOther(List<Organization> affiliations)
{
// Special case until `https://ror.org/%20https://ror.org/` is moved to `https://ror.org/_other`
if (affiliations.Any(o => o.RorUrl.Equals("https://ror.org/%20https://ror.org/")))
{
var cleanedUpAffiliations = affiliations;
foreach (var entry in cleanedUpAffiliations)
{
if (entry.RorUrl.Equals("https://ror.org/%20https://ror.org/"))
{
cleanedUpAffiliations[cleanedUpAffiliations.IndexOf(entry)] = new()
{
Name = _otherOrganization.Name,
RorUrl = _otherOrganization.RorUrl
};
}
}
return cleanedUpAffiliations;
}
return affiliations;
}
} }
\ No newline at end of file
using Polly;
namespace Coscine.KpiGenerator.Utils
{
public static class Helpers
{
public static W WrapRequest<W>(Func<W> function)
{
return Policy
.Handle<Exception>()
.WaitAndRetry(5, retryNumber => TimeSpan.FromMilliseconds(200))
.ExecuteAndCapture(() => function.Invoke()).Result;
}
}
}