src/KPI Generator/KPI Generator.csproj

@@ -7,7 +7,7 @@
 		<AssemblyName>Coscine.KpiGenerator</AssemblyName>
 		<ImplicitUsings>enable</ImplicitUsings>
 		<Nullable>enable</Nullable>
-		<Version>0.1.2</Version>
+		<Version>0.1.3</Version>
 	</PropertyGroup>
 
 	<PropertyGroup>

src/KPI Generator/Program.cs

@@ -33,7 +33,7 @@ public static class Program
               _ => false);
             if (result)
             {
-                Console.WriteLine("Finished.\n");
+                Console.WriteLine("\nFinished.\n");
                 return 0; // Exit Code 0 for Success
             }
             else

src/KPI Generator/Reporting.cs

@@ -3,10 +3,7 @@ using Coscine.Metadata;
 using GitLabApiClient;
 using GitLabApiClient.Models.Commits.Requests.CreateCommitRequest;
 using KPIGenerator.Utils;
-using Newtonsoft.Json;
-using Newtonsoft.Json.Linq;
 using System.Text;
-using VDS.RDF;
 using VDS.RDF.Query;
 using static KPIGenerator.Utils.CommandLineOptions;
 
@@ -52,20 +49,20 @@ public abstract class Reporting<O> where O : class
     public bool Run()
     {
         // Console text output
-        Console.Write($"{new string('=', 60)}\n{InstanceName}");
+        Console.Write($"{new string('=', 80)}\n{InstanceName}");
         var baseOptions = Options as BaseOptions;
         if (baseOptions is not null && baseOptions.DummyMode)
         {
             Console.Write(" : DUMMY MODE");
         }
-        Console.WriteLine($"\n{new string('-', 60)}");
+        Console.WriteLine($"\n{new string('-', 80)}");
         // Generate Reporting based on CLI input
         var reportingFiles = GenerateReporting();
-        Console.WriteLine($"\n{new string('=', 60)}");
-        Console.WriteLine("Reporting generated successfully. Publishing...");
+        Console.WriteLine($"\n{new string('=', 80)}");
+        Console.WriteLine(" - Reporting generated successfully. Publishing...");
         // Publish Report
         var success = PublishAsync(reportingFiles).Result;
-        Console.WriteLine(success ? "Published successfully." : "Publishing FAILED!");
+        Console.WriteLine(success ? " - Published successfully." : " - Publishing FAILED!");
         return success;
     }
 
@@ -78,6 +75,8 @@ public abstract class Reporting<O> where O : class
             var commitBranch = reportingDatabaseProject.DefaultBranch;
             var commitMessage = $"{InstanceName} Generated - {DateTime.Now:dd.MM.yyyy HH:mm}"; // CompleteReporting Generated - 31.08.2022 10:25
 
+            Console.WriteLine($" - Commit: \"{commitMessage}\"");
+
             var projectTree = await GitLabClient.Trees.GetAsync(reportingDatabaseProject, o =>
             {
                 o.Recursive = true;

src/KPI Generator/Reportings/ApplicationProfile/ApplicationProfileReporting.cs

@@ -14,6 +14,7 @@ public class ApplicationProfileReporting : Reporting<ApplicationProfileReporting
 
     public override IEnumerable<ReportingFileObject> GenerateReporting()
     {
+        Console.WriteLine($" - {GetType().Name}: Begin reporting generation");
         var reportingFiles = new List<ReportingFileObject>();
         var returnObjects = GetApplicationProfiles();
 
@@ -23,7 +24,9 @@ public class ApplicationProfileReporting : Reporting<ApplicationProfileReporting
             Path = GetReportingPathGeneral(ReportingFileName),
             Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjects, Formatting.Indented))
         });
+        Console.WriteLine($" - {GetType().Name}: \"{GetReportingPathGeneral(ReportingFileName)}\" generated successfully");
         
+        Console.WriteLine();
         return reportingFiles;
     }
 

src/KPI Generator/Reportings/Complete/CompleteReporting.cs

@@ -18,38 +18,69 @@ public class CompleteReporting : Reporting<CompleteReportingOptions>
         var result = new List<ReportingFileObject>();
 
         // Project Reporting
+        try
+        {
             result.AddRange(new ProjectReporting(new ProjectReportingOptions
             {
                 DummyMode = Options.DummyMode
             }).GenerateReporting());
-
+        }
+        catch (Exception e)
+        {
+            Console.WriteLine($"!! Skipping ProjectReporting: {e.Message}");
+        }
 
         // Resource Reporting
+        try
+        {
             result.AddRange(new ResourceReporting(new ResourceReportingOptions
             {
                 DummyMode = Options.DummyMode
             }).GenerateReporting());
-
+        }
+        catch (Exception e)
+        {
+            Console.WriteLine($"!! Skipping ResourceReporting: {e.Message}");
+        }
 
         // User Reporting
+        try
+        {
             result.AddRange(new UserReporting(new UserReportingOptions
             {
                 DummyMode = Options.DummyMode
             }).GenerateReporting());
-
+        }
+        catch (Exception e)
+        {
+            Console.WriteLine($"!! Skipping UserReporting: {e.Message}");
+        }
 
         // Application Profile Reporting
+        try
+        {
             result.AddRange(new ApplicationProfileReporting(new ApplicationProfileReportingOptions
             {
                 DummyMode = Options.DummyMode
             }).GenerateReporting());
-
+        }
+        catch (Exception e)
+        {
+            Console.WriteLine($"!! Skipping ApplicationProfileReporting: {e.Message}");
+        }
 
         // System Status Reporting
+        try
+        {
             result.AddRange(new SystemReporting(new SystemReportingOptions
             {
                 DummyMode = Options.DummyMode
             }).GenerateReporting());
+        }
+        catch (Exception e)
+        {
+            Console.WriteLine($"!! Skipping SystemReporting: {e.Message}");
+        }
 
         return result;
     }

src/KPI Generator/Reportings/Project/ProjectReporting.cs

-using KPIGenerator.Utils;
+using Coscine.Database.DataModel;
+using Coscine.Database.Models;
+using Coscine.Database.ReturnObjects;
+using Coscine.ResourceTypes;
+using KPIGenerator.Utils;
+using Newtonsoft.Json;
 using static KPIGenerator.Utils.CommandLineOptions;
 
 namespace KPIGenerator.Reportings.Project;
 
 public class ProjectReporting : Reporting<ProjectReportingOptions>
 {
+    private readonly ProjectModel _projectModel;
+    private readonly ResourceModel _resourceModel;
+    private readonly ProjectRoleModel _projectRoleModel;
+    private readonly ProjectQuotaModel _projectQuotaModel;
+    private readonly ResourceTypeModel _resourceTypeModel;
+
     public ProjectReporting(ProjectReportingOptions options) : base(options)
     {
         ReportingFileName = "projects.json";
+        _projectModel = new ProjectModel();
+        _projectRoleModel = new ProjectRoleModel();
+        _projectQuotaModel = new ProjectQuotaModel();
+        _resourceModel = new ResourceModel();
+        _resourceTypeModel = new ResourceTypeModel();
     }
 
     public override IEnumerable<ReportingFileObject> GenerateReporting()
     {
-        /* 
-         * 1. Collect the reporting for the whole database -- General/{ReportingReportingFileName}
-         * 2. Append to the list the same information per organization as folders -- Organizations/{OrgRorId}/{ReportingReportingFileName}
-         * --> See envisioned folder structure. 
-         */
-        throw new NotImplementedException();
+        Console.WriteLine($" - {GetType().Name}: Begin reporting generation");
+        var projects = _projectModel.GetAllWhere(r => r.Deleted.Equals(true) || r.Deleted.Equals(false));
+        var reportingFiles = new List<ReportingFileObject>();
+        var returnObjects = Generate(projects);
+
+        //General File
+        reportingFiles.Add(new ReportingFileObject
+        {
+            Path = GetReportingPathGeneral(ReportingFileName),
+            Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjects, Formatting.Indented)),
+        });
+        Console.WriteLine($" - {GetType().Name}: \"{GetReportingPathGeneral(ReportingFileName)}\" generated successfully");
+
+        // Per Organization
+        reportingFiles.AddRange(GeneratePerOrganization(returnObjects));
+
+        Console.WriteLine();
+        return reportingFiles;
+    }
+
+    private List<ReturnObject> Generate(IEnumerable<Coscine.Database.DataModel.Project> projects)
+    {
+        var returnObjects = new List<ReturnObject>();
+        foreach (var project in projects)
+        {
+            var projectReturnObject = _projectModel.CreateReturnObjectFromDatabaseObject(project);
+
+            var projectReportEntry = new ReturnObject
+            {
+                Id = projectReturnObject.Id,
+                DateCreated = projectReturnObject.DateCreated,
+                Organizations = GetOrganizations(projectReturnObject),
+                Disciplines = projectReturnObject.Disciplines.ToList(),
+                Deleted = projectReturnObject.Deleted,
+                ProjectVisibilityId = projectReturnObject.Visibility.Id,
+                GrantId = projectReturnObject.GrantId,
+                Users = _projectRoleModel.GetAllWhere(x => x.ProjectId == projectReturnObject.Id).Count(),
+                ResourceTypeQuota = GetResourceTypeQuota(projectReturnObject.Id)
+            };
+            returnObjects.Add(projectReportEntry);
+        }
+        return returnObjects;
+    }
+
+    private IEnumerable<ReportingFileObject> GeneratePerOrganization(List<ReturnObject> returnObjects)
+    {
+        var reportingFilesPerOrganization = new List<ReportingFileObject>();
+        var organizationsFromProjects = returnObjects.SelectMany(ro => ro.Organizations).DistinctBy(o => o.RorUrl);
+        foreach (var entry in organizationsFromProjects)
+        {
+            var organization = Organizations.Find(o => o.Equals(entry));
+            if (organization is null)
+            {
+                organization = _otherOrganization;
+                Console.WriteLine($"   WARNING!: Organization \"{entry.RorUrl}\" could not be correctly identified. Will use \"{_otherOrganization.RorUrl}\".");
+            }
+            var returnObjectsForOrganization = returnObjects.Where(ro => ro.Organizations.Select(o => o.RorUrl).Any(e => e.Equals(entry.RorUrl)));
+            reportingFilesPerOrganization.Add(new ReportingFileObject
+            {
+                Path = GetReportingPathOrganization(organization.RorUrl.Replace("https://ror.org/", "").ToLower(), ReportingFileName),
+                Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjectsForOrganization, Formatting.Indented))
+            });
+            Console.WriteLine($" - {GetType().Name}: \"{GetReportingPathOrganization(organization.RorUrl.Replace("https://ror.org/", "").ToLower(), ReportingFileName)}\" generated successfully");
+        }
+
+        return reportingFilesPerOrganization;
+    }
+
+    private List<ResourceTypeQuotaReturnObject> GetResourceTypeQuota(Guid projectId)
+    {
+        var result = new List<ResourceTypeQuotaReturnObject>();
+        var types = ResourceTypeFactory.Instance.GetResourceTypes();
+        var resourceTypes = _resourceTypeModel.GetAllWhere(x => types.Any(t => x.SpecificType.Equals(t)));
+        var quotas = _projectQuotaModel.GetAllWhere(x => x.ProjectId == projectId);
+        var resourceQuotas = quotas.Where(x => resourceTypes.Any(e => x.ResourceTypeId.Equals(e.Id)));
+        result.AddRange(resourceQuotas.Select(x => CreateQuotaReturnObject(x, projectId)).ToList());
+        return result;
+    }
+
+    private ResourceTypeQuotaReturnObject CreateQuotaReturnObject(ProjectQuota projectQuota, Guid projectId)
+    {
+        return new ResourceTypeQuotaReturnObject
+        {
+            ResourceType = _resourceTypeModel.GetById(projectQuota.ResourceTypeId).SpecificType,
+            Allocated = new QuotaDimObject()
+            {
+                Value = projectQuota.Quota,
+                Unit = QuotaUnit.GibiBYTE,
+            },
+            Maximum = new QuotaDimObject()
+            {
+                Value = projectQuota.MaxQuota,
+                Unit = QuotaUnit.GibiBYTE,
+            },
+            TotalReserved = new QuotaDimObject()
+            {
+                Value = CalculateAllocatedForAll(_resourceTypeModel.GetById(projectQuota.ResourceTypeId), projectId),
+                Unit = QuotaUnit.GibiBYTE,
+            },
+            TotalUsed = new QuotaDimObject()
+            {
+                Value = CalculateUsedForAll(_resourceTypeModel.GetById(projectQuota.ResourceTypeId), projectId),
+                Unit = QuotaUnit.BYTE,
+            },
+        };
+    }
+
+    private int CalculateUsedForAll(ResourceType resourceType, Guid projectId)
+    {
+        var resources = _resourceModel.GetAllWhere((resource) =>
+                    (from projectResource in resource.ProjectResources
+                     where projectResource.ProjectId == projectId
+                     select projectResource).Any() &&
+                     resource.TypeId == resourceType.Id);
+
+        var used = resources.Sum(resource =>
+        {
+            // Linked has no quota.
+            var rt = ResourceTypeFactory.Instance.GetResourceType(resource);
+            if (rt.GetResourceTypeInformation().Result.IsQuotaAvailable)
+            {
+                return rt.GetResourceQuotaUsed(resource.Id.ToString(), _resourceModel.GetResourceTypeOptions(resource.Id)).Result;
+            }
+            else
+            {
+                return 0;
+            }
+        }
+        );
+
+        return (int)used;
+    }
+
+    private int CalculateAllocatedForAll(ResourceType resourceType, Guid projectId)
+    {
+        var resources = _resourceModel.GetAllWhere((resource) =>
+                    (from projectResource in resource.ProjectResources
+                     where projectResource.ProjectId == projectId
+                     select projectResource).Any() &&
+                     resource.TypeId == resourceType.Id);
+
+        var allocated = resources.Sum(resource =>
+        {
+            // Linked has no quota.
+            var rt = ResourceTypeFactory.Instance.GetResourceType(resource);
+            if (rt.GetResourceTypeInformation().Result.IsQuotaAvailable)
+            {
+                return rt.GetResourceQuotaAvailable(resource.Id.ToString(), _resourceModel.GetResourceTypeOptions(resource.Id)).Result;
+            }
+            else
+            {
+                return 0;
+            }
+        });
+
+        return (int)allocated;
+    }
+
+    private List<Organization> GetOrganizations(ProjectObject project)
+    {
+        var result = new List<Organization>();
+        foreach (var entry in project.Organizations)
+        {
+            result.Add(FetchOrganizationByRor(entry.Url));
+        }
+        return result;
     }
 }
\ No newline at end of file

src/KPI Generator/Reportings/Project/ResourceTypeQuotaReturnObject.cs

+using Coscine.Database.ReturnObjects;
+
+namespace KPIGenerator.Reportings.Project;
+
+public class ResourceTypeQuotaReturnObject
+{
+    public string ResourceType { get; set; } = null!;
+
+    public QuotaDimObject TotalUsed { get; set; } = null!;
+
+    public QuotaDimObject TotalReserved { get; set; } = null!;
+
+    public QuotaDimObject Allocated { get; set; } = null!;
+
+    public QuotaDimObject Maximum { get; set; } = null!;
+}
\ No newline at end of file

src/KPI Generator/Reportings/Project/ReturnObject.cs

-namespace KPIGenerator.Reportings.Project;
+using Coscine.Database.ReturnObjects;
+using KPIGenerator.Utils;
+
+namespace KPIGenerator.Reportings.Project;
 
 /// <summary>
 /// Object containing the JSON structure for the reporting
 /// </summary>
 public class ReturnObject
 {
-
+    public Guid Id { get; set; }
+    public DateTime? DateCreated { get; set; } = null;
+    public List<Organization>? Organizations { get; set; } = new();
+    public List<DisciplineObject> Disciplines { get; set; } = new();
+    public bool Deleted { get; set; }
+    public Guid ProjectVisibilityId { get; set; }
+    public string? GrantId { get; set; }
+    public int? Users { get; set; } = null;
+    public List<ResourceTypeQuotaReturnObject> ResourceTypeQuota { get; set; } = new();
 }
\ No newline at end of file

src/KPI Generator/Reportings/Resource/ResourceReporting.cs

@@ -24,6 +24,7 @@ public class ResourceReporting : Reporting<ResourceReportingOptions>
 
     public override IEnumerable<ReportingFileObject> GenerateReporting()
     {
+        Console.WriteLine($" - {GetType().Name}: Begin reporting generation");
         var resources = _resourceModel.GetAllWhere(r => r.Deleted.Equals(true) || r.Deleted.Equals(false));
         var reportingFiles = new List<ReportingFileObject>();
         var returnObjects = Generate(resources);
@@ -34,9 +35,12 @@ public class ResourceReporting : Reporting<ResourceReportingOptions>
             Path = GetReportingPathGeneral(ReportingFileName),
             Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjects, Formatting.Indented))
         });
+        Console.WriteLine($" - {GetType().Name}: \"{GetReportingPathGeneral(ReportingFileName)}\" generated successfully");
+
         // Per Organization
         reportingFiles.AddRange(GeneratePerOrganization(returnObjects));
 
+        Console.WriteLine();
         return reportingFiles;
     }
 
@@ -86,6 +90,7 @@ public class ResourceReporting : Reporting<ResourceReportingOptions>
                 Path = GetReportingPathOrganization(organization.RorUrl.Replace("https://ror.org/", "").ToLower(), ReportingFileName),
                 Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjectsForOrganization, Formatting.Indented))
             });
+            Console.WriteLine($" - {GetType().Name}: \"{GetReportingPathOrganization(organization.RorUrl.Replace("https://ror.org/", "").ToLower(), ReportingFileName)}\" generated successfully");
         }
         return reportingFilesPerOrganization;
     }
@@ -98,7 +103,7 @@ public class ResourceReporting : Reporting<ResourceReportingOptions>
         }
         catch
         {
-            Console.WriteLine($"There is no project related to resource with ID \"{resourceId}\".");
+            Console.WriteLine($"   There is no project related to resource with ID \"{resourceId}\"");
             return null;
         }
     }
@@ -112,7 +117,7 @@ public class ResourceReporting : Reporting<ResourceReportingOptions>
         }
         catch
         {
-            Console.WriteLine($"No resource type definition found for resource with ID \"{resource.Id}\".");
+            Console.WriteLine($"   No resource type definition found for resource with ID \"{resource.Id}\"");
             resourceTypeDefinition = null;
         }
 

src/KPI Generator/Reportings/System/MaintenanceReturnObject.cs

+using Newtonsoft.Json;
+
+namespace KPIGenerator.Reportings.System;
+
+public class MaintenanceBannerObject
+{
+    [JsonProperty("id")]
+    public int Id { get; set; }
+    [JsonProperty("title")]
+    public string Title { get; set; } = null!;
+    [JsonProperty("short_message")]
+    public string ShortMessage { get; set; } = null!;
+    [JsonProperty("message")]
+    public string Message { get; set; } = null!;
+    [JsonProperty("status")]
+    public string Status { get; set; } = null!;
+    [JsonProperty("start_at")]
+    public DateTime StartAt { get; set; }
+    [JsonProperty("end_at")]
+    public DateTime EndAt { get; set; }
+}
+

src/KPI Generator/Reportings/System/ReturnObject.cs

@@ -5,5 +5,5 @@
 /// </summary>
 public class ReturnObject
 {
-
+    public List<MaintenanceBannerObject> Banners { get; set; } = new();
 }
\ No newline at end of file

src/KPI Generator/Reportings/System/SystemReporting.cs

-using KPIGenerator.Utils;
+using Coscine.Configuration;
+using KPIGenerator.Utils;
+using Newtonsoft.Json;
+using System.Net.Http.Headers;
+using System.Text;
 using static KPIGenerator.Utils.CommandLineOptions;
 
 namespace KPIGenerator.Reportings.System;
 
 public class SystemReporting : Reporting<SystemReportingOptions>
 {
+    private readonly ConsulConfiguration _configuration;
+
     public SystemReporting(SystemReportingOptions options) : base(options)
     {
         ReportingFileName = "system_status.json";
+        _configuration = new ConsulConfiguration();
     }
 
     public override IEnumerable<ReportingFileObject> GenerateReporting()
     {
-        /* 
-         * 1. Collect the reporting for the whole database -- General/{ReportingReportingFileName}
-         * --> See envisioned folder structure. 
-         */
-        throw new NotImplementedException();
+        Console.WriteLine($" - {GetType().Name}: Begin reporting generation");
+        var reportingFiles = new List<ReportingFileObject>();
+        var returnObject = GetSystemStatus();
+
+        // General File
+        reportingFiles.Add(new ReportingFileObject
+        {
+            Path = GetReportingPathGeneral(ReportingFileName),
+            Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObject, Formatting.Indented))
+        });
+        Console.WriteLine($" - {GetType().Name}: \"{GetReportingPathGeneral(ReportingFileName)}\" generated successfully");
+
+        Console.WriteLine();
+        return reportingFiles;
     }
+
+    private ReturnObject GetSystemStatus()
+    {
+        return new ReturnObject()
+        {
+            Banners = GetMaintenanceBannersAsync().Result
+            // Extend here with other specific props in the future
+        };
     }
+
+    private async Task<List<MaintenanceBannerObject>> GetMaintenanceBannersAsync()
+    {
+        var httpClient = new HttpClient();
+        var requestMessage = new HttpRequestMessage()
+        {
+            RequestUri = new Uri("https://noc-portal.rz.rwth-aachen.de/ticket/api/coscine/tickets"),
+            Method = HttpMethod.Get
+        };
+
+        // Add Basic Authentication Headers
+        var authenticationString = $"{_configuration.GetStringAndWait("coscine/global/maintenance/user")}:{_configuration.GetStringAndWait("coscine/global/maintenance/password")}";
+        var base64EncodedAuthenticationString = Convert.ToBase64String(Encoding.ASCII.GetBytes(authenticationString));
+        requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Basic", base64EncodedAuthenticationString);
+
+        var result = await httpClient.SendAsync(requestMessage);
+        var responseBody = JsonConvert.DeserializeObject<List<MaintenanceBannerObject>>(result.Content.ReadAsStringAsync().Result);
+        return responseBody.OrderBy(e => e.StartAt).ToList();
+    }
+}
+

src/KPI Generator/Reportings/User/UserReporting.cs

@@ -33,6 +33,7 @@ public class UserReporting : Reporting<UserReportingOptions>
 
     public override IEnumerable<ReportingFileObject> GenerateReporting()
     {
+        Console.WriteLine($" - {GetType().Name}: Begin reporting generation");
         var users = _userModel.GetAllWhere((user) => user.Tosaccepteds.Any());
         var reportingFiles = new List<ReportingFileObject>();
         var returnObjects = Generate(users);
@@ -43,9 +44,12 @@ public class UserReporting : Reporting<UserReportingOptions>
             Path = GetReportingPathGeneral(ReportingFileName),
             Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjects, Formatting.Indented))
         });
+        Console.WriteLine($" - {GetType().Name}: \"{GetReportingPathGeneral(ReportingFileName)}\" generated successfully");
+
         // Per Organization
         reportingFiles.AddRange(GeneratePerOrganization(returnObjects));
 
+        Console.WriteLine();
         return reportingFiles;
     }
 
@@ -88,6 +92,7 @@ public class UserReporting : Reporting<UserReportingOptions>
                 Path = GetReportingPathOrganization(organization.RorUrl.Replace("https://ror.org/", "").ToLower(), ReportingFileName),
                 Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjectsForOrganization, Formatting.Indented))
             });
+            Console.WriteLine($" - {GetType().Name}: \"{GetReportingPathOrganization(organization.RorUrl.Replace("https://ror.org/", "").ToLower(), ReportingFileName)}\" generated successfully");
         }
         return reportingFilesPerOrganization;
     }