Fix: Organizations

src/KPI Generator/Reporting.cs

@@ -6,7 +6,6 @@ using GitLabApiClient.Models.Branches.Requests;
 using GitLabApiClient.Models.Commits.Requests.CreateCommitRequest;
 using Microsoft.Extensions.Logging;
 using NLog.Extensions.Logging;
-using Polly;
 using System.Text;
 using System.Web;
 using static KPIGenerator.Utils.CommandLineOptions;
@@ -172,7 +171,7 @@ public abstract class Reporting<O> where O : class
         }
         else
         {
-            var organizationTriples = WrapRequest(() => RdfStoreConnector.GetLabelForSubject(new Uri(Uri.UnescapeDataString(rorUrl))).ToList());
+            var organizationTriples = Helpers.WrapRequest(() => RdfStoreConnector.GetLabelForSubject(new Uri(Uri.UnescapeDataString(rorUrl))).ToList());
             if (organizationTriples is not null && organizationTriples.Any())
             {
                 result = new Organization
@@ -260,33 +259,4 @@ public abstract class Reporting<O> where O : class
         }
         Console.WriteLine();
     }
-
-    /// <summary>
-    /// Retry Virtuoso Requests since they sometimes just fail
-    /// </summary>
-    /// <typeparam name="W"></typeparam>
-    /// <param name="function"></param>
-    /// <returns></returns>
-    public void WrapRequest(Action action)
-    {
-        Policy
-            .Handle<Exception>()
-            .WaitAndRetry(5, retryNumber => TimeSpan.FromMilliseconds(200))
-            .Execute(() => action.Invoke());
-    }
-
-    /// <summary>
-    /// Retry Virtuoso Requests since they sometimes just fail
-    /// </summary>
-    /// <typeparam name="W"></typeparam>
-    /// <param name="function"></param>
-    /// <returns></returns>
-    public W WrapRequest<W>(Func<W> function)
-    {
-        return Policy
-            .Handle<Exception>()
-            .WaitAndRetry(5, retryNumber => TimeSpan.FromMilliseconds(200))
-            .ExecuteAndCapture(() => function.Invoke()).Result;
-    }
-
 }
\ No newline at end of file

src/KPI Generator/Reportings/ApplicationProfile/ApplicationProfileReporting.cs

@@ -57,7 +57,7 @@ public class ApplicationProfileReporting : Reporting<ApplicationProfileReporting
                               OPTIONAL {{ ?{_applicationProfile} dcterms:{_license} ?{_license} .  }}
                             }}"
         };
-        using var result = WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(queryString.ToString()));
+        using var result = Helpers.WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(queryString.ToString()));
 
         var grouped = result.GroupBy(ap => new
         {

src/KPI Generator/Reportings/Project/ProjectReporting.cs

@@ -164,7 +164,7 @@ public class ProjectReporting : Reporting<ProjectReportingOptions>
                 try
                 {
                     var usedBytes = rt.GetResourceQuotaUsed(resource.Id.ToString(), _resourceModel.GetResourceTypeOptions(resource.Id)).Result;
-                    return Helpers.ConvertCapacityUnits(new QuotaDimObject() { Value = usedBytes, Unit = QuotaUnit.BYTE }, outputInThisUnit);
+                    return ResourceTypes.Helpers.ConvertCapacityUnits(new QuotaDimObject() { Value = usedBytes, Unit = QuotaUnit.BYTE }, outputInThisUnit);
                 }
                 catch (Exception ex)
                 {

src/KPI Generator/Reportings/Resource/ResourceReporting.cs

@@ -135,7 +135,7 @@ public class ResourceReporting : Reporting<ResourceReportingOptions>
         {
             try
             {
-                return Helpers.CreateResourceQuotaReturnObject(resource, resourceTypeDefinition);
+                return ResourceTypes.Helpers.CreateResourceQuotaReturnObject(resource, resourceTypeDefinition);
             }
             catch (Exception ex)
             {

src/KPI Generator/Reportings/User/UserReporting.cs

 using Coscine.Database.DataModel;
 using Coscine.Database.Models;
+using Coscine.Database.ReturnObjects;
 using Coscine.KpiGenerator.Utils;
 using Coscine.Metadata;
 using Microsoft.Extensions.Logging;
@@ -68,11 +69,12 @@ public class UserReporting : Reporting<UserReportingOptions>
         foreach (var user in users)
         {
             var userReturnObject = _userModel.CreateReturnObjectFromDatabaseObject(user);
+            var (organizations, institutes) = GetUserAffiliation(userReturnObject);
             var userReportEntry = new ReturnObject
             {
                 RelatedProjects = GetRelatedProjects(user.Id),
-                Organizations = GetOrganizations(user.Id, userReturnObject.Organization, "organization"),
-                Institutes = GetOrganizations(user.Id, userReturnObject.Institute, "institute"),
+                Organizations = organizations,
+                Institutes = institutes,
                 Disciplines = userReturnObject.Disciplines.ToList(),
                 LoginProviders = userReturnObject.ExternalAuthenticators.ToList(),
                 LatestActivity = GetLatestActivity(user.Id)
@@ -131,144 +133,56 @@ public class UserReporting : Reporting<UserReportingOptions>
         return result;
     }
 
-    /// <summary>
-    /// A method that fetches the organization affiliation of a user based on a user ID
-    /// </summary>
-    /// <param name="id">Coscine User ID</param>
-    /// <param name="organizationLabel">Organization of the user. Will be set ONLY for ORCiD users, otherwise null.</param>
-    /// <returns>List of Organizations that the user belogs to. Will be empty if no organization can be found.</returns>
-    private List<Organization> GetOrganizations(Guid id, string organizationLabel, string searchedEntityType)
-    {
-        /* A user login has the following possibilities: 
-         *  - ORCiD only login              (Organization set by the user and is found inside the SQL DB; can't confirm validity)
-         *  - Shibboleth only login         (Organization set by Shibboleth provider and is found inside Virtuoso; is regarded as valid)
-         *  - ORCiD AND Shibboleth login    (Organization set by Shibboleth provider and is found inside Virtuoso; is regarded as valid)
-         *  
-         * In the case of ORCiD only login, reflect that inside report's "LoginProvides". Organization can be found only using its Display Name (not very reliable).
-         * In the case of Shibboleth login, one needs to find the correct organization by using the user's ExternalId and ExternalAuthenticators.
-         */
-        var result = new List<Organization>();
+    private (List<Organization> organizations, List<Organization> institutes) GetUserAffiliation(UserObject user)
+    {
+        var affiliations = new List<Organization>();
 
-        var externalIds = _externalIdModel.GetAllWhere((externalId) => externalId.UserId.Equals(id));
+        // Bellow code taken from Organizations API
+        var externalIds = _externalIdModel.GetAllWhere((externalId) => externalId.UserId == user.Id);
+        var externalIdList = new List<string>();
         foreach (var externalId in externalIds)
         {
-            var loginProvider = _externalAuthenticatorModel.GetWhere(e => e.Id.Equals(externalId.ExternalAuthenticatorId));
-            if (loginProvider != null)
-            {
-                switch (loginProvider.DisplayName.ToLower())
-                {
-                    case "orcid":
-                        // Find the RoR of the organization based on its "rdfs:label".
-                        var orgOrcid = TryGetOrganizationByLabel(organizationLabel);
-                        if (orgOrcid is null)
-                        {
-                            Console.WriteLine($"   No {searchedEntityType} found for user with ID \"{id}\" and login provider {loginProvider.DisplayName}");
-                            continue;
-                        }
-                        result.Add(orgOrcid);
-                        break;
-                    case "shibboleth":
-                        // Find the RoR first based on the user's External Id, then try based on the Organization Entity Id.
-                        var orgShibboleth = GetOrganizationByExternalOrEntityId(externalId.ExternalId1) ?? GetOrganizationByExternalOrEntityId(externalId.Organization);
-                        if (orgShibboleth is null)
-                        {
-                            Console.WriteLine($"   No {searchedEntityType} found for user with ID \"{id}\" and login provider {loginProvider.DisplayName}");
-                            continue;
-                        }
-                        if (orgShibboleth.RorUrl.Contains('#'))
-                        {
-                            var orgTopLevelRor = orgShibboleth.RorUrl[..orgShibboleth.RorUrl.IndexOf('#')];
-                            var orgTopLevel = FetchOrganizationByRor(orgTopLevelRor);
-                            if (!orgTopLevel.RorUrl.Equals(_otherOrganization.RorUrl))
-                            {
-                                result.Add(orgTopLevel);
-                                break;
-                            }
-                        }
-                        result.Add(orgShibboleth);
-                        break;
-                    default:
-                        Console.WriteLine($"!! Could not verify the login provider for user with ID \"{id}\"");
-                        break;
-                }
-            }
-            else
-            {
-                Console.WriteLine($"!! Could not verify the login provider for user with ID \"{id}\" and External Authenticator ID \"{externalId.ExternalAuthenticatorId}\"");
-            }
+            externalIdList.Add(externalId.ExternalId1);
         }
+        var externalOrganizations = externalIds.Select((externalId) => externalId.Organization);
 
-        var orcidLoginProvider = _loginProviders.Single(lp => lp.DisplayName.ToLower().Equals("orcid"));
-        if (!string.IsNullOrWhiteSpace(organizationLabel) && !externalIds.Any(e => e.ExternalAuthenticatorId.Equals(orcidLoginProvider.Id)))
-        {
-            // Special case, a user has an organization set, but does not have an external authenticator for ODCiD. Should not be possible.
-            Console.WriteLine($"!! User with ID \"{id}\" has an organization set inside the Database but does not own a login provider {orcidLoginProvider.DisplayName}");
-            // Find the RoR of the organization based on its "rdfs:label".
-            var orgOrcid = TryGetOrganizationByLabel(organizationLabel);
-            if (orgOrcid is null)
-            {
-                Console.WriteLine($"   No {searchedEntityType} found for user with ID \"{id}\" and login provider {orcidLoginProvider.DisplayName}");
-            }
-            else
+        var triples = Helpers.WrapRequest(() => RdfStoreConnector.GetTriples(null, null, null, 1, externalIdList).ToList());
+        foreach (var externalOrganization in externalOrganizations)
         {
-                result.Add(orgOrcid);
-            }
+            triples.AddRange(Helpers.WrapRequest(() => RdfStoreConnector.GetOrganizationByEntityId(externalOrganization)));
         }
+        triples = triples.Distinct().ToList();
 
-        // Special case until `https://ror.org/%20https://ror.org/` is moved to `https://ror.org/_other`
-        if (result.Any(o => o.RorUrl.Equals("https://ror.org/%20https://ror.org/")))
+        foreach (var triple in triples)
         {
-            foreach (var entry in result)
-            {
-                if (entry.RorUrl.Equals("https://ror.org/%20https://ror.org/"))
+            affiliations.Add(new()
             {
-                    entry.Name = _otherOrganization.Name;
-                    entry.RorUrl = _otherOrganization.RorUrl;
-                }
-            }
+                Name = triple.Object.ToString(),
+                RorUrl = triple.Subject.ToString()
+            });
         }
 
-        return result.DistinctBy(e => e.RorUrl).ToList();
-    }
+        // Clean up
+        affiliations = CleanUpOther(affiliations);
 
-    private Organization? GetOrganizationByExternalOrEntityId(string externalOrEntityId)
-    {
-        var _queryString = new SparqlParameterizedString()
-        {
-            CommandText = $@"
-                PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
-                PREFIX org: <http://www.w3.org/ns/org#>
+        // Split affiliations to "organizations" and "institutes"
+        var organizations = affiliations.Where(a => !a.RorUrl.Contains('#')).ToList();
+        var institutes = affiliations.Where(a => a.RorUrl.Contains('#')).ToList();
 
-                SELECT DISTINCT ?ror
-                WHERE {{
-                    ?ror rdfs:label ?name . 
-                    {{
-                        SELECT DISTINCT ?ror
-                        WHERE {{
-                            ?class ?p ?memberUrl ;
-                            org:organization ?ror .
-                            {{
-                                SELECT DISTINCT ?memberUrl 
-                                WHERE {{
-                                    ?memberUrl ?p ?value . 
-                                    FILTER( ?value IN ( '{externalOrEntityId}' ))
-                                }}
-                            }}
-                        }}
-                    }}
-                }}
-            "
-        };
-        using var results = WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(_queryString.ToString()));
-        if (!results.IsEmpty)
-        {
-            var ror = results.Select(x => x.Value("ror").ToString());   // Get the value for ?ror
-            if (ror.Any())
-            {
-                return FetchOrganizationByRor(ror.ToList()[0]);
-            }
-        }
-        return null;
+        // Find the RoR of the organization based on its "rdfs:label".
+        var orgFromSqlDb = TryGetOrganizationByLabel(user.Organization);
+        if (orgFromSqlDb is not null)
+            organizations.Add(orgFromSqlDb);
+
+        // Find the RoR of the institute based on its "rdfs:label".
+        var instFromSqlDb = TryGetOrganizationByLabel(user.Institute);
+        if (instFromSqlDb is not null)
+            institutes.Add(instFromSqlDb);
+
+        return (
+            organizations.DistinctBy(o => o.RorUrl).ToList(),
+            institutes.DistinctBy(i => i.RorUrl).ToList()
+        );
     }
 
     private Organization? TryGetOrganizationByLabel(string rdfsLabel)
@@ -293,7 +207,7 @@ public class UserReporting : Reporting<UserReportingOptions>
                 }}
             "
         };
-        using var results = WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(_queryString.ToString()));
+        using var results = Helpers.WrapRequest(() => RdfStoreConnector.QueryEndpoint.QueryWithResultSet(_queryString.ToString()));
         if (!results.IsEmpty)
         {
             var ror = results.Select(x => x.Value("ror").ToString());   // Get the value for ?ror
@@ -307,10 +221,6 @@ public class UserReporting : Reporting<UserReportingOptions>
 
     private DateTime? GetLatestActivity(Guid id)
     {
-        /*
-         * TODO: Query may take ages to execute and can lead to timeouts. Make sure to create an index!
-         *   CREATE INDEX log_idx_userid_loglevel_servertime ON Coscine.dbo.Log (UserId, LogLevel, ServerTimestamp);
-         */
         var latestLog = _logModel.GetAllWhere(l => l.LogLevel.Equals("Analytics") && l.UserId.Equals(id)).OrderByDescending(a => a.ServerTimestamp).FirstOrDefault();
         if (latestLog is not null)
         {
@@ -318,4 +228,26 @@ public class UserReporting : Reporting<UserReportingOptions>
         }
         else return null;
     }
+
+    private List<Organization> CleanUpOther(List<Organization> affiliations)
+    {
+        // Special case until `https://ror.org/%20https://ror.org/` is moved to `https://ror.org/_other`
+        if (affiliations.Any(o => o.RorUrl.Equals("https://ror.org/%20https://ror.org/")))
+        {
+            var cleanedUpAffiliations = affiliations;
+            foreach (var entry in cleanedUpAffiliations)
+            {
+                if (entry.RorUrl.Equals("https://ror.org/%20https://ror.org/"))
+                {
+                    cleanedUpAffiliations[cleanedUpAffiliations.IndexOf(entry)] = new()
+                    {
+                        Name = _otherOrganization.Name,
+                        RorUrl = _otherOrganization.RorUrl
+                    };
+                }
+            }
+            return cleanedUpAffiliations;
+        }
+        return affiliations;
+    }
 }
\ No newline at end of file

src/KPI Generator/Utils/Helpers.cs

+using Polly;
+
+namespace Coscine.KpiGenerator.Utils
+{
+    public static class Helpers
+    {
+        public static W WrapRequest<W>(Func<W> function)
+        {
+            return Policy
+                .Handle<Exception>()
+                .WaitAndRetry(5, retryNumber => TimeSpan.FromMilliseconds(200))
+                .ExecuteAndCapture(() => function.Invoke()).Result;
+        }
+    }
+}