From 471ab3a5bc25bfd34926b30e2a1d3d0592cf3931 Mon Sep 17 00:00:00 2001
From: Petar Hristov <hristov@itc.rwth-aachen.de>
Date: Fri, 14 Oct 2022 12:09:08 +0200
Subject: [PATCH] Fix: Generator Workflow
---
src/KPI Generator/Reporting.cs | 58 ++++++++++++++++---
.../Reportings/Project/ProjectReporting.cs | 4 +-
.../Reportings/Resource/ResourceReporting.cs | 4 +-
.../Reportings/User/UserReporting.cs | 4 +-
4 files changed, 55 insertions(+), 15 deletions(-)
diff --git a/src/KPI Generator/Reporting.cs b/src/KPI Generator/Reporting.cs
index 7c2ec31..d533dce 100644
--- a/src/KPI Generator/Reporting.cs
+++ b/src/KPI Generator/Reporting.cs
@@ -1,9 +1,11 @@
using Coscine.Configuration;
using Coscine.Metadata;
using GitLabApiClient;
+using GitLabApiClient.Models.Branches.Requests;
using GitLabApiClient.Models.Commits.Requests.CreateCommitRequest;
using KPIGenerator.Utils;
using System.Text;
+using System.Web;
using VDS.RDF.Query;
using static KPIGenerator.Utils.CommandLineOptions;
@@ -21,18 +23,18 @@ public abstract class Reporting<O> where O : class
private static string InstanceName { get; set; } = null!;
public virtual string ReportingFileName { get; init; } = null!;
+ private string Domain { get; init; }
+ private bool ReportingEnabled { get; init; }
+ private string ReportingDatabaseProjectId { get; init; }
+ private string ReportingBranch { get; init; }
+
+
public readonly Organization _otherOrganization = new()
{
Name = "Other",
RorUrl = "https://ror.org/_other",
};
- /// <summary>
- /// Reporting Database GitLab Project URL
- /// </summary>
- /// <remarks>https://git.rwth-aachen.de/coscine/reporting/reporting-database</remarks>
- private static readonly int ReportingDatabaseProjectId = 75304;
-
public Reporting(O options)
{
InstanceName = this.GetType().Name;
@@ -42,6 +44,11 @@ public abstract class Reporting<O> where O : class
QueryEndpoint = new SparqlRemoteEndpoint(new Uri(Configuration.GetStringAndWait("coscine/local/virtuoso/additional/url")));
GitLabClient = new GitLabClient(HostUrl, Configuration.GetStringAndWait("coscine/global/gitlabtoken"));
Organizations = new List<Organization>() { _otherOrganization };
+
+ ReportingEnabled = Configuration.GetStringAndWait("coscine/local/reporting/enabled", "false") == "true";
+ Domain = Configuration.GetStringAndWait("coscine/local/profilesync/domain");
+ ReportingDatabaseProjectId = Configuration.GetStringAndWait("coscine/local/reporting/gitlab_project_id");
+ ReportingBranch = Configuration.GetStringAndWait("coscine/local/reporting/branch");
}
public abstract IEnumerable<ReportingFileObject> GenerateReporting();
@@ -49,13 +56,14 @@ public abstract class Reporting<O> where O : class
public bool Run()
{
// Console text output
- Console.Write($"{new string('=', 80)}\n{InstanceName}");
+ Console.Write($"{new string('=', 80)}\n {Domain} | {InstanceName}");
var baseOptions = Options as BaseOptions;
if (baseOptions is not null && baseOptions.DummyMode)
{
Console.Write(" : DUMMY MODE");
}
Console.WriteLine($"\n{new string('-', 80)}");
+ EnsureGitLabInformationIsSetAndCorrect();
// Generate Reporting based on CLI input
var reportingFiles = GenerateReporting();
Console.WriteLine($"\n{new string('=', 80)}");
@@ -72,7 +80,7 @@ public abstract class Reporting<O> where O : class
{
// Retrieve Reporting Database project
var reportingDatabaseProject = await GitLabClient.Projects.GetAsync(ReportingDatabaseProjectId);
- var commitBranch = reportingDatabaseProject.DefaultBranch;
+ var commitBranch = ReportingBranch;
var commitMessage = $"{InstanceName} Generated - {DateTime.Now:dd.MM.yyyy HH:mm}"; // CompleteReporting Generated - 31.08.2022 10:25
Console.WriteLine($" - Commit: \"{commitMessage}\"");
@@ -172,6 +180,11 @@ public abstract class Reporting<O> where O : class
return result;
}
+ public static string SanitizeOrganizationRor(string organizationRor)
+ {
+ return HttpUtility.UrlEncode(organizationRor.Replace("https://ror.org/", "").ToLower());
+ }
+
public static string GetReportingPathGeneral(string fileName)
{
return string.Format("General/{0}", fileName);
@@ -179,7 +192,7 @@ public abstract class Reporting<O> where O : class
public static string GetReportingPathOrganization(string organizationRor, string fileName)
{
- return string.Format("Organizations/{0}/{1}", organizationRor, fileName);
+ return string.Format("Organizations/{0}/{1}", SanitizeOrganizationRor(organizationRor), fileName);
}
public static Stream ConvertStringContentsToStream(string contents)
@@ -187,4 +200,31 @@ public abstract class Reporting<O> where O : class
byte[] byteArray = Encoding.UTF8.GetBytes(contents);
return new MemoryStream(byteArray);
}
+
+ public void EnsureGitLabInformationIsSetAndCorrect()
+ {
+ if (!ReportingEnabled)
+ {
+ throw new ApplicationException($"\nReporting is deactivated on this machine! \nTo enable it, set the Consul Key \"coscine/local/reporting/enabled\" to \"true\".");
+ }
+ if (string.IsNullOrWhiteSpace(ReportingDatabaseProjectId) || string.IsNullOrWhiteSpace(ReportingBranch))
+ {
+ throw new ArgumentNullException($"\nNo valid Reporting Project ID or Branch were provided!");
+ }
+ var project = GitLabClient.Projects.GetAsync(ReportingDatabaseProjectId).Result;
+ Console.WriteLine($" - Report Generation to be uploaded to GitLab Project \"{project.Name}\" on branch \"{ReportingBranch}\"");
+ var branch = GitLabClient.Branches.GetAsync(project.Id, o => o.Search = ReportingBranch).Result;
+
+ if (!branch.Any(b => b.Name.Equals(ReportingBranch)) && Domain.Equals("DEVLEF") && !project.DefaultBranch.Equals(ReportingBranch))
+ {
+ Console.WriteLine($" - Branch \"{ReportingBranch}\" does not exist. Working on Domain {Domain}. Creating branch...");
+ GitLabClient.Branches.CreateAsync(ReportingDatabaseProjectId, new CreateBranchRequest(ReportingBranch, project.DefaultBranch)).Wait();
+ Console.WriteLine($" - Branch \"{ReportingBranch}\" successfully created");
+ }
+ else if (!branch.Any(b => b.Name.Equals(ReportingBranch)))
+ {
+ throw new ArgumentNullException($"\nBranch \"{ReportingBranch}\" does not exist!");
+ }
+ Console.WriteLine();
+ }
}
\ No newline at end of file
diff --git a/src/KPI Generator/Reportings/Project/ProjectReporting.cs b/src/KPI Generator/Reportings/Project/ProjectReporting.cs
index 5f8ad9f..aab3eef 100644
--- a/src/KPI Generator/Reportings/Project/ProjectReporting.cs
+++ b/src/KPI Generator/Reportings/Project/ProjectReporting.cs
@@ -87,10 +87,10 @@ public class ProjectReporting : Reporting<ProjectReportingOptions>
var returnObjectsForOrganization = returnObjects.Where(ro => ro.Organizations.Select(o => o.RorUrl).Any(e => e.Equals(entry.RorUrl)));
reportingFilesPerOrganization.Add(new ReportingFileObject
{
- Path = GetReportingPathOrganization(organization.RorUrl.Replace("https://ror.org/", "").ToLower(), ReportingFileName),
+ Path = GetReportingPathOrganization(organization.RorUrl, ReportingFileName),
Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjectsForOrganization, Formatting.Indented))
});
- Console.WriteLine($" - {GetType().Name}: \"{GetReportingPathOrganization(organization.RorUrl.Replace("https://ror.org/", "").ToLower(), ReportingFileName)}\" generated successfully");
+ Console.WriteLine($" - {GetType().Name}: \"{GetReportingPathOrganization(organization.RorUrl, ReportingFileName)}\" generated successfully");
}
return reportingFilesPerOrganization;
diff --git a/src/KPI Generator/Reportings/Resource/ResourceReporting.cs b/src/KPI Generator/Reportings/Resource/ResourceReporting.cs
index aefc846..32f00ff 100644
--- a/src/KPI Generator/Reportings/Resource/ResourceReporting.cs
+++ b/src/KPI Generator/Reportings/Resource/ResourceReporting.cs
@@ -87,10 +87,10 @@ public class ResourceReporting : Reporting<ResourceReportingOptions>
reportingFilesPerOrganization.Add(new ReportingFileObject
{
- Path = GetReportingPathOrganization(organization.RorUrl.Replace("https://ror.org/", "").ToLower(), ReportingFileName),
+ Path = GetReportingPathOrganization(organization.RorUrl, ReportingFileName),
Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjectsForOrganization, Formatting.Indented))
});
- Console.WriteLine($" - {GetType().Name}: \"{GetReportingPathOrganization(organization.RorUrl.Replace("https://ror.org/", "").ToLower(), ReportingFileName)}\" generated successfully");
+ Console.WriteLine($" - {GetType().Name}: \"{GetReportingPathOrganization(organization.RorUrl, ReportingFileName)}\" generated successfully");
}
return reportingFilesPerOrganization;
}
diff --git a/src/KPI Generator/Reportings/User/UserReporting.cs b/src/KPI Generator/Reportings/User/UserReporting.cs
index 7854e6c..dfd6799 100644
--- a/src/KPI Generator/Reportings/User/UserReporting.cs
+++ b/src/KPI Generator/Reportings/User/UserReporting.cs
@@ -89,10 +89,10 @@ public class UserReporting : Reporting<UserReportingOptions>
reportingFilesPerOrganization.Add(new ReportingFileObject
{
- Path = GetReportingPathOrganization(organization.RorUrl.Replace("https://ror.org/", "").ToLower(), ReportingFileName),
+ Path = GetReportingPathOrganization(organization.RorUrl, ReportingFileName),
Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjectsForOrganization, Formatting.Indented))
});
- Console.WriteLine($" - {GetType().Name}: \"{GetReportingPathOrganization(organization.RorUrl.Replace("https://ror.org/", "").ToLower(), ReportingFileName)}\" generated successfully");
+ Console.WriteLine($" - {GetType().Name}: \"{GetReportingPathOrganization(organization.RorUrl, ReportingFileName)}\" generated successfully");
}
return reportingFilesPerOrganization;
}
--
GitLab