src/KPI Generator/KPI Generator.csproj

@@ -7,7 +7,7 @@
 		<AssemblyName>Coscine.KpiGenerator</AssemblyName>
 		<ImplicitUsings>enable</ImplicitUsings>
 		<Nullable>enable</Nullable>
-		<Version>0.1.0</Version>
+		<Version>0.1.1</Version>
 	</PropertyGroup>
 
 	<PropertyGroup>
@@ -19,6 +19,7 @@
 
 	<ItemGroup>
 		<PackageReference Include="CommandLineParser" Version="2.9.1" />
+		<PackageReference Include="Coscine.ApiCommons" Version="2.*-*" />
 		<PackageReference Include="Coscine.Database" Version="2.*-*" />
 		<PackageReference Include="Coscine.Metadata" Version="2.*-*" />
 		<PackageReference Include="GitLabApiClient" Version="1.8.1-beta.5" />

src/KPI Generator/Reporting.cs

@@ -3,6 +3,10 @@ using Coscine.Metadata;
 using GitLabApiClient;
 using GitLabApiClient.Models.Commits.Requests.CreateCommitRequest;
 using KPIGenerator.Utils;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using System.Text;
+using VDS.RDF;
 using VDS.RDF.Query;
 using static KPIGenerator.Utils.CommandLineOptions;
 
@@ -15,7 +19,7 @@ public abstract class Reporting<O> where O : class
     public List<Organization> Organizations { get; set; }
     public RdfStoreConnector RdfStoreConnector { get; init; }
     public SparqlRemoteEndpoint QueryEndpoint { get; init; }
-    private ConsulConfiguration Configuration { get; }
+    public ConsulConfiguration Configuration { get; }
     private static string HostUrl { get; } = "https://git.rwth-aachen.de/";
     private static string InstanceName { get; set; } = null!;
     public virtual string ReportingFileName { get; init; } = null!;
@@ -42,19 +46,20 @@ public abstract class Reporting<O> where O : class
     public bool Run()
     {
         // Console text output
-        Console.Write($"{new string('-', 35)}\n{InstanceName}");
+        Console.Write($"{new string('=', 60)}\n{InstanceName}");
         var baseOptions = Options as BaseOptions;
         if (baseOptions is not null && baseOptions.DummyMode)
         {
             Console.Write(" : DUMMY MODE");
         }
-        Console.WriteLine($"\n{new string('-', 35)}");
+        Console.WriteLine($"\n{new string('-', 60)}");
         // Generate Reporting based on CLI input
         var reportingFiles = GenerateReporting();
+        Console.WriteLine($"\n{new string('=', 60)}");
         Console.WriteLine("Reporting generated successfully. Publishing...");
         // Publish Report
         var success = PublishAsync(reportingFiles).Result;
-        Console.WriteLine(success ? "Reporting published successfully." : "Reporting publishing FAILED!");
+        Console.WriteLine(success ? "Published successfully." : "Publishing FAILED!");
         return success;
     }
 
@@ -65,14 +70,32 @@ public abstract class Reporting<O> where O : class
         {
             // Retrieve Reporting Database project
             var reportingDatabaseProject = await GitLabClient.Projects.GetAsync(ReportingDatabaseProjectId);
+            var commitBranch = reportingDatabaseProject.DefaultBranch;
             var commitMessage = $"{InstanceName} Generated - {DateTime.Now:dd.MM.yyyy HH:mm}"; // CompleteReporting Generated - 31.08.2022 10:25
 
+            var projectTree = await GitLabClient.Trees.GetAsync(reportingDatabaseProject, o =>
+            {
+                o.Recursive = true;
+                o.Reference = commitBranch;
+            });
+
             // Define commit actions
             var actions = new List<CreateCommitRequestAction>();
 
+            // Delete files or organizations that are not valid anymore
+            foreach (var fileInProject in projectTree.Where(file => file.Type.Equals("blob")))
+            {
+                if (!files.Any(f => f.Path.Equals(fileInProject.Path)) && !fileInProject.Path.Equals("README.md"))
+                {
+                    // Add Action
+                    actions.Add(new CreateCommitRequestAction(CreateCommitRequestActionType.Delete, fileInProject.Path));
+                }
+            }
+
             // Create a commit per file with its contents
             foreach (var file in files)
             {
+                // Write file contents to bytes
                 byte[] bytes;
                 using (var ms = new MemoryStream())
                 {
@@ -80,7 +103,15 @@ public abstract class Reporting<O> where O : class
                     bytes = ms.ToArray();
                 }
 
-                actions.Add(new CreateCommitRequestAction(CreateCommitRequestActionType.Create, file.Path)
+                // Distinguish between Creating or Updating a file
+                var actionType = CreateCommitRequestActionType.Create;
+                if (projectTree.Any(f => f.Path.Equals(file.Path)))
+                {
+                    actionType = CreateCommitRequestActionType.Update;
+                }
+
+                // Add Action
+                actions.Add(new CreateCommitRequestAction(actionType, file.Path)
                 {
                     Content = Convert.ToBase64String(bytes),
                     Encoding = CreateCommitRequestActionEncoding.Base64
@@ -90,7 +121,7 @@ public abstract class Reporting<O> where O : class
             var baseOptions = Options as BaseOptions;
             if (baseOptions is not null && !baseOptions.DummyMode)
             {
-                await GitLabClient.Commits.CreateAsync(reportingDatabaseProject, new CreateCommitRequest(reportingDatabaseProject.DefaultBranch, commitMessage, actions));
+                await GitLabClient.Commits.CreateAsync(reportingDatabaseProject, new CreateCommitRequest(commitBranch, commitMessage, actions));
             }
             return true;
         }
@@ -104,14 +135,58 @@ public abstract class Reporting<O> where O : class
     private List<Organization> FetchOrganizations()
     {
         var organizations = new List<Organization>();
-        var organizationsGroupId = 23782; // Organisations GitLab Group URL: https://git.rwth-aachen.de/coscine/graphs/organisations
-        var organizationsGroup = GitLabClient.Groups.GetAsync(organizationsGroupId).Result;
-        foreach (var project in organizationsGroup.Projects)
+        var organizationsToFind = JsonConvert.DeserializeObject<IEnumerable<Uri>>(
+                Configuration.GetStringAndWait("coscine/local/organizations/list",
+                    "['https://ror.org/', 'https://ror.org/04xfq0f34']")
+            );
+
+        var resultSet = new List<Triple>();
+        foreach (var orgGraph in organizationsToFind)
+        {
+            resultSet.AddRange(RdfStoreConnector.GetTriples(orgGraph, null));
+        }
+        var organizationTriples = resultSet.Where(r => !r.Subject.ToString().Contains('#')).Distinct().ToList();
+
+        foreach (var triple in organizationTriples)
+        {
+            organizations.Add(new Organization
             {
-            var name = project.Description[..project.Description.IndexOf(':')];
-            var ror = project.Name;
-            organizations.Add(new Organization(name, ror));
+                Name = triple.Object.ToString(),
+                Ror = triple.Subject.ToString(),
+            });
+        }
+
+        // Organization "Other"
+        var organizationOther = organizations.Find(o => o.Name.Equals("Other"));
+        var organizationOtherRor = "https://ror.org/_other";
+        if (organizationOther is null)
+        {
+            organizations.Add(new Organization
+            {
+                Name = "Other",
+                Ror = organizationOtherRor
+            });
+        }
+        else
+        {
+            var index = organizations.IndexOf(organizationOther);
+            organizations[index].Ror = organizationOtherRor;
         }
+
         return organizations;
     }
+    public static string GetReportingPathGeneral(string fileName)
+    {
+        return string.Format("General/{0}", fileName);
+    }
+    public static string GetReportingPathOrganization(string organizationRor, string fileName)
+    {
+        return string.Format("Organizations/{0}/{1}", organizationRor, fileName);
+    }
+
+    public static Stream ConvertStringContentsToStream(string contents)
+    {
+        byte[] byteArray = Encoding.UTF8.GetBytes(contents);
+        return new MemoryStream(byteArray);
+    }
 }

src/KPI Generator/Reportings/ApplicationProfile/ApplicationProfileReporting.cs

 using KPIGenerator.Utils;
+using Newtonsoft.Json;
+using VDS.RDF.Query;
 using static KPIGenerator.Utils.CommandLineOptions;
 
 namespace KPIGenerator.Reportings.ApplicationProfile;
@@ -12,10 +14,67 @@ public class ApplicationProfileReporting : Reporting<ApplicationProfileReporting
 
     public override IEnumerable<ReportingFileObject> GenerateReporting()
     {
-        /* 
-         * 1. Collect the reporting for the whole database -- General/{ReportingReportingFileName}
-         * --> See envisioned folder structure. 
-         */
-        throw new NotImplementedException();
+        var reportingFiles = new List<ReportingFileObject>();
+        var returnObjects = GetApplicationProfiles();
+
+        // General File
+        reportingFiles.Add(new ReportingFileObject
+        {
+            Path = GetReportingPathGeneral(ReportingFileName),
+            Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjects, Formatting.Indented))
+        });
+
+        return reportingFiles;
+    }
+
+    private List<ReturnObject> GetApplicationProfiles()
+    {
+        var _applicationProfile = "applicationProfile";
+        var _title = "title";
+        var _publisher = "publisher";
+        var _rights = "rights";
+        var _license = "license";
+
+        var returnObjects = new List<ReturnObject>();
+        var queryString = new SparqlParameterizedString
+        {
+            CommandText = $@"PREFIX dcterms: <http://purl.org/dc/terms/>
+                            SELECT DISTINCT * WHERE {{ 
+                              ?{_applicationProfile} a <http://www.w3.org/ns/shacl#NodeShape> . 
+                              OPTIONAL {{ ?{_applicationProfile} dcterms:{_title} ?{_title}.  }}
+                              OPTIONAL {{ ?{_applicationProfile} dcterms:{_publisher} ?{_publisher} . }}
+                              OPTIONAL {{ ?{_applicationProfile} dcterms:{_rights} ?{_rights} .  }}
+                              OPTIONAL {{ ?{_applicationProfile} dcterms:{_license} ?{_license} .  }}
+                            }}"
+        };
+        using var result = RdfStoreConnector.QueryEndpoint.QueryWithResultSet(queryString.ToString());
+
+        var grouped = result.GroupBy(ap => new
+        {
+            Uri = ap.Value(_applicationProfile).ToString(),
+            Publisher = ap.HasValue(_publisher) && ap.Value(_publisher) is not null ? ap.Value(_publisher).ToString() : null,
+            Rights = ap.HasValue(_rights) && ap.Value(_rights) is not null ? ap.Value(_rights).ToString() : null,
+            License = ap.HasValue(_license) && ap.Value(_license) is not null ? ap.Value(_license).ToString() : null
+        },
+        t => t.HasValue(_title) && t.Value(_title) is not null ? t.Value(_title).ToString() : null);
+        foreach (var ap in grouped)
+        {
+            returnObjects.Add(new ReturnObject
+            {
+                Uri = ap.Key.Uri,
+                Publisher = ap.Key.Publisher,
+                Rights = ap.Key.Rights,
+                License = ap.Key.License,
+                Titles = ap.Select(t =>
+                {
+                    if (t is not null)
+                    {
+                        return t[..t.IndexOf('@')];
+                    }
+                    return t;
+                }).ToList()
+            });
+        }
+        return returnObjects;
     }
 }

src/KPI Generator/Reportings/ApplicationProfile/ApplicationProfileReportingOptions.cs

@@ -4,7 +4,7 @@ namespace KPIGenerator.Utils;
 
 public static partial class CommandLineOptions
 {
-    [Verb("applprofiles", HelpText = "Generate application profile KPIs")]
+    [Verb("applicationprofiles", HelpText = "Generate application profile KPIs")]
     public class ApplicationProfileReportingOptions : BaseOptions
     {
         // Add verb specific options here

src/KPI Generator/Reportings/ApplicationProfile/ReturnObject.cs

@@ -5,5 +5,10 @@
 /// </summary>
 public class ReturnObject
 {
+    public List<string?> Titles { get; set; } = new();
+    public string Uri { get; set; } = null!;
+    public string? Publisher { get; set; } = null!;
+    public string? Rights { get; set; } = null!;
+    public string? License { get; set; } = null!;
 
 }

src/KPI Generator/Reportings/User/ReturnObject.cs

-namespace KPIGenerator.Reportings.User;
+using Coscine.Database.ReturnObjects;
+using Newtonsoft.Json;
+
+namespace KPIGenerator.Reportings.User;
 
 /// <summary>
 /// Object containing the JSON structure for the reporting
 /// </summary>
 public class ReturnObject
 {
+    public List<RelatedProject> RelatedProjects { get; set; } = new();
+    public List<string> Organizations { get; set; } = new();
+    public List<string> Institutes { get; set; } = new();
+    public List<DisciplineObject> Disciplines { get; set; } = new();
+    public List<ExternalAuthenticatorsObject> LoginProviders { get; set; } = new();
+    public DateTime? LatestActivity { get; set; } = null;
+
 
+    public class RelatedProject
+    {
+        public Guid ProjectId { get; set; }
+        public string Role { get; set; } = null!;
+    }
 }

src/KPI Generator/Reportings/User/UserReporting.cs

-using KPIGenerator.Utils;
+using Coscine.ApiCommons;
+using Coscine.Database.Models;
+using Coscine.Metadata;
+using KPIGenerator.Utils;
+using Newtonsoft.Json;
 using static KPIGenerator.Utils.CommandLineOptions;
 
 namespace KPIGenerator.Reportings.User;
 
 public class UserReporting : Reporting<UserReportingOptions>
 {
+    private readonly Authenticator _authenticator;
+    private readonly ExternalAuthenticatorModel _externalAuthenticatorModel;
+    private readonly ExternalIdModel _externalIdModel;
+    private readonly ProjectRoleModel _projectRoleModel;
+    private readonly ProjectModel _projectModel;
+    private readonly RoleModel _roleModel;
+    private readonly UserModel _userModel;
+    private readonly LogModel _logModel;
+
     public UserReporting(UserReportingOptions options) : base(options)
     {
         ReportingFileName = "users.json";
+        _authenticator = new Authenticator(null, Configuration);
+        _externalAuthenticatorModel = new ExternalAuthenticatorModel();
+        _externalIdModel = new ExternalIdModel();
+        _projectRoleModel = new ProjectRoleModel();
+        _projectModel = new ProjectModel();
+        _roleModel = new RoleModel();
+        _userModel = new UserModel();
+        _logModel = new LogModel();
     }
 
     public override IEnumerable<ReportingFileObject> GenerateReporting()
     {
-        /* 
-         * 1. Collect the reporting for the whole database -- General/{ReportingReportingFileName}
-         * 2. Append to the list the same information per organization as folders -- Organizations/{OrgRorId}/{ReportingReportingFileName}
-         * --> See envisioned folder structure. 
-         */
-        throw new NotImplementedException();
+        var users = _userModel.GetAllWhere((user) => user.Tosaccepteds.Any());
+        var reportingFiles = new List<ReportingFileObject>();
+        var returnObjects = Generate(users);
+
+        // General File
+        reportingFiles.Add(new ReportingFileObject
+        {
+            Path = GetReportingPathGeneral(ReportingFileName),
+            Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjects, Formatting.Indented))
+        });
+        // Per Organization
+        reportingFiles.AddRange(GeneratePerOrganization(returnObjects));
+
+        return reportingFiles;
+    }
+
+    private List<ReturnObject> Generate(IEnumerable<Coscine.Database.DataModel.User> users)
+    {
+        var returnObjects = new List<ReturnObject>();
+        foreach (var user in users)
+        {
+            var userReturnObject = _userModel.CreateReturnObjectFromDatabaseObject(user);
+            var userReportEntry = new ReturnObject
+            {
+                RelatedProjects = GetRelatedProjects(user.Id),
+                Organizations = GetOrganizations(user.Id, userReturnObject.Organization),
+                Institutes = GetInstitutes(user.Id, userReturnObject.Institute),
+                Disciplines = userReturnObject.Disciplines.ToList(),
+                LoginProviders = userReturnObject.ExternalAuthenticators.ToList(),
+                LatestActivity = GetLatestActivity(user.Id)
+            };
+            returnObjects.Add(userReportEntry);
+        }
+        return returnObjects;
+    }
+
+    private IEnumerable<ReportingFileObject> GeneratePerOrganization(List<ReturnObject> returnObjects)
+    {
+        var reportingFilesPerOrganization = new List<ReportingFileObject>();
+        var organizationsFromUsers = returnObjects.SelectMany(ro => ro.Organizations).Distinct();
+        foreach (var entry in organizationsFromUsers)
+        {
+            var organization = Organizations.Find(o => o.Name.Equals(entry));
+            if (organization is null)
+            {
+                organization = Organizations.Find(o => o.Name.Equals("Other"));
+                Console.WriteLine($"WARNING!: Organization \"{entry}\" could not be correctly identified. Will use \"{organization!.Name}\".");
+            }
+            var returnObjectsForOrganization = returnObjects.Where(ro => ro.Organizations.Contains(entry));
+
+            reportingFilesPerOrganization.Add(new ReportingFileObject
+            {
+                Path = GetReportingPathOrganization(organization.Ror.Replace("https://ror.org/", "").ToLower(), ReportingFileName),
+                Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjectsForOrganization, Formatting.Indented))
+            });
+        }
+        return reportingFilesPerOrganization;
+    }
+
+    private List<ReturnObject.RelatedProject> GetRelatedProjects(Guid id)
+    {
+        var result = new List<ReturnObject.RelatedProject>();
+        var projectRoles = _projectRoleModel.GetAllWhere(role => role.UserId.Equals(id));
+        if (projectRoles.Any())
+        {
+            foreach (var projectRole in projectRoles)
+            {
+                if (_projectModel.GetById(projectRole.ProjectId) is not null) // null if project has been deleted
+                {
+                    result.Add(new ReturnObject.RelatedProject
+                    {
+                        ProjectId = projectRole.ProjectId,
+                        Role = _roleModel.GetById(projectRole.RoleId).DisplayName
+                    });
+                }
+            }
+        }
+        return result;
+    }
+
+    private List<string> GetOrganizations(Guid id, string organization)
+    {
+        var externalIdModel = new ExternalIdModel();
+        var result = new List<string>();
+
+        if (!string.IsNullOrWhiteSpace(organization))
+        {
+            result.Add(organization);
+        }
+
+        var externalIds = externalIdModel.GetAllWhere((externalId) => externalId.UserId.Equals(id));
+        var externalIdList = new List<string>();
+        foreach (var externalId in externalIds)
+        {
+            externalIdList.Add(externalId.ExternalId1);
+        }
+        var resultSet = RdfStoreConnector.GetTriples(null, null, null, 1, externalIdList);
+
+        var organizationTriples = resultSet.Where(r => !r.Subject.ToString().Contains('#')).Distinct().ToList();
+
+        foreach (var triple in organizationTriples)
+        {
+            result.Add(triple.Object.ToString());
+        }
+
+        return result;
+    }
+
+    private List<string> GetInstitutes(Guid id, string institute)
+    {
+        var result = new List<string>();
+
+        if (!string.IsNullOrWhiteSpace(institute))
+        {
+            result.Add(institute);
+        }
+
+        var externalIds = _externalIdModel.GetAllWhere((externalId) => externalId.UserId.Equals(id));
+        var externalIdList = new List<string>();
+        foreach (var externalId in externalIds)
+        {
+            externalIdList.Add(externalId.ExternalId1);
+        }
+        var resultSet = RdfStoreConnector.GetTriples(null, null, null, 1, externalIdList);
+
+        var instituteTriples = resultSet.Where(r => r.Subject.ToString().Contains('#')).Distinct().ToList();
+
+        foreach (var triple in instituteTriples)
+        {
+            result.Add(triple.Object.ToString());
+        }
+
+        return result;
+    }
+
+    private DateTime? GetLatestActivity(Guid id)
+    {
+        var latestLog = _logModel.GetAllWhere(l => l.LogLevel.Equals("Analytics") && l.UserId.Equals(id)).OrderByDescending(a => a.ServerTimestamp).FirstOrDefault();
+        if (latestLog is not null)
+        {
+            return latestLog.ServerTimestamp;
+        }
+        else return null;
     }
 }

src/KPI Generator/Utils/Organization.cs

@@ -16,12 +16,4 @@ public class Organization
     /// Organizaiton ROR URL from GitLab project's title
     /// </summary>
     /// <example>https://ror.org/04xfq0f34</example>
-    public Uri RorUri { get; set; } = null!;
-
-    public Organization(string name, string ror)
-    {
-        Name = name;
-        Ror = ror;
-        RorUri = new Uri($"https://ror.org/{ror}");
-    }
 }
\ No newline at end of file