Select Git revision
PQLoadDP.cpp
Code owners
Assign users and groups as approvers for specific file changes. Learn more.
Reporting.cs 12.42 KiB
using Coscine.Configuration;
using Coscine.Metadata;
using GitLabApiClient;
using GitLabApiClient.Models.Branches.Requests;
using GitLabApiClient.Models.Commits.Requests.CreateCommitRequest;
using KPIGenerator.Utils;
using System.Text;
using System.Web;
using VDS.RDF.Query;
using static KPIGenerator.Utils.CommandLineOptions;
namespace KPIGenerator;
public abstract class Reporting<O> where O : class
{
public O Options { get; init; }
public GitLabClient GitLabClient { get; set; }
public List<Organization> Organizations { get; set; }
public RdfStoreConnector RdfStoreConnector { get; init; }
public SparqlRemoteEndpoint QueryEndpoint { get; init; }
public ConsulConfiguration Configuration { get; }
private static string HostUrl { get; } = "https://git.rwth-aachen.de/";
private static string InstanceName { get; set; } = null!;
public virtual string ReportingFileName { get; init; } = null!;
private string Domain { get; init; }
private bool ReportingEnabled { get; init; }
private string ReportingDatabaseProjectId { get; init; }
private string ReportingBranch { get; init; }
public string RwthRor { get; init; }
public readonly Organization _otherOrganization = new()
{
Name = "Other",
RorUrl = "https://ror.org/_other",
};
public Reporting(O options)
{
InstanceName = this.GetType().Name;
Options = options;
Configuration = new ConsulConfiguration();
RdfStoreConnector = new RdfStoreConnector(Configuration.GetStringAndWait("coscine/local/virtuoso/additional/url"));
QueryEndpoint = new SparqlRemoteEndpoint(new Uri(Configuration.GetStringAndWait("coscine/local/virtuoso/additional/url")));
GitLabClient = new GitLabClient(HostUrl, Configuration.GetStringAndWait("coscine/global/gitlabtoken"));
Organizations = new List<Organization>() { _otherOrganization };
ReportingEnabled = Configuration.GetStringAndWait("coscine/local/reporting/enabled", "false") == "true";
Domain = Configuration.GetStringAndWait("coscine/local/profilesync/domain");
ReportingDatabaseProjectId = Configuration.GetStringAndWait("coscine/local/reporting/gitlab_project_id");
ReportingBranch = Configuration.GetStringAndWait("coscine/local/reporting/branch");
RwthRor = Configuration.GetStringAndWait("coscine/global/organizations/rwth/ror_url");
}
public abstract IEnumerable<ReportingFileObject> GenerateReporting();
public bool Run()
{
// Console text output
Console.Write($"{new string('=', 80)}\n {Domain} | {InstanceName}");
var baseOptions = Options as BaseOptions;
if (baseOptions is not null && baseOptions.DummyMode)
{
Console.Write(" : DUMMY MODE");
}
Console.WriteLine($"\n{new string('-', 80)}");
EnsureGitLabInformationIsSetAndCorrect();
// Generate Reporting based on CLI input
var reportingFiles = GenerateReporting();
Console.WriteLine($"\n{new string('=', 80)}");
Console.WriteLine(" - Reporting generated successfully. Publishing...");
// Publish Report
var success = PublishAsync(reportingFiles).Result;
Console.WriteLine(success ? " - Published successfully." : " - Publishing FAILED!");
return success;
}
private async Task<bool> PublishAsync(IEnumerable<ReportingFileObject> files)
{
try
{
// Retrieve Reporting Database project
var reportingDatabaseProject = await GitLabClient.Projects.GetAsync(ReportingDatabaseProjectId);
var commitBranch = ReportingBranch;
var commitMessage = $"{InstanceName} Generated - {DateTime.Now:dd.MM.yyyy HH:mm}"; // CompleteReporting Generated - 31.08.2022 10:25
Console.WriteLine($" - Commit: \"{commitMessage}\"");
var projectTree = await GitLabClient.Trees.GetAsync(reportingDatabaseProject, o =>
{
o.Recursive = true;
o.Reference = commitBranch;
});
// Define commit actions
var actions = new List<CreateCommitRequestAction>();
// Delete files or organizations that are not valid anymore
foreach (var fileInProject in projectTree.Where(file => file.Type.Equals("blob")))
{
if (!files.Any(f => f.Path.Equals(fileInProject.Path)) && !fileInProject.Path.Equals("README.md"))
{
// Add Action
actions.Add(new CreateCommitRequestAction(CreateCommitRequestActionType.Delete, fileInProject.Path));
}
}
// Create a commit per file with its contents
foreach (var file in files)
{
// Write file contents to bytes
byte[] bytes;
using (var ms = new MemoryStream())
{
file.Content.CopyTo(ms);
bytes = ms.ToArray();
}
// Distinguish between Creating or Updating a file
var actionType = CreateCommitRequestActionType.Create;
if (projectTree.Any(f => f.Path.Equals(file.Path)))
{
actionType = CreateCommitRequestActionType.Update;
}
// Add Action
actions.Add(new CreateCommitRequestAction(actionType, file.Path)
{
Content = Convert.ToBase64String(bytes),
Encoding = CreateCommitRequestActionEncoding.Base64
});
}
// Push Commit
var baseOptions = Options as BaseOptions;
if (baseOptions is not null && !baseOptions.DummyMode)
{
await GitLabClient.Commits.CreateAsync(reportingDatabaseProject, new CreateCommitRequest(commitBranch, commitMessage, actions));
}
return true;
}
catch (Exception e)
{
Console.WriteLine(e.Message);
return false;
}
}
public Organization FetchOrganizationByRor(string rorUrl)
{
var result = new Organization();
var organizationFound = Organizations.Find(o => o.RorUrl.Equals(rorUrl));
if (organizationFound is not null)
{
result = new Organization
{
Name = organizationFound.Name,
RorUrl = organizationFound.RorUrl,
};
}
else
{
var organizationTriples = RdfStoreConnector.GetLabelForSubject(new Uri(Uri.UnescapeDataString(rorUrl))).ToList();
if (organizationTriples.Any())
{
result = new Organization
{
// Only one entry possible per organization, take 0th element
Name = organizationTriples[0].Object.ToString(),
RorUrl = organizationTriples[0].Subject.ToString(),
};
Organizations.Add(result); // Cache the fetched organization
}
else
{
result = _otherOrganization;
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine($" WARNING!: Organization with ROR \"{rorUrl}\" could not be correctly identified. Will use \"{result.RorUrl}\" and \"{result.Name}\".");
Console.ResetColor();
}
}
return result;
}
public IEnumerable<Organization> GetTopLevelOrganizationsFromEntries(IEnumerable<Organization> organizations)
{
var result = new List<Organization>();
foreach (var org in organizations)
{
var ror = org.RorUrl;
if (ror.Contains("www.rwth-aachen.de"))
{
ror = ConvertOldRwthOrganizationToRor(ror); // e.g. <https://www.rwth-aachen.de/22000> turns into <https://ror.org/04xfq0f34#ORG-42NHW>
}
result.Add(new Organization
{
RorUrl = ror.Contains('#') ? ror[..ror.IndexOf('#')] : ror, // e.g. <https://ror.org/04xfq0f34#ORG-42NHW> turns into <https://ror.org/04xfq0f34>
Name = org.Name
});
}
return result.DistinctBy(r => r.RorUrl);
}
public static string SanitizeOrganizationRor(string organizationRor)
{
return HttpUtility.UrlEncode(organizationRor.Replace("https://ror.org/", "").ToLower());
}
public static string GetReportingPathGeneral(string fileName)
{
return string.Format("General/{0}", fileName);
}
public static string GetReportingPathOrganization(string organizationRor, string fileName)
{
return string.Format("Organizations/{0}/{1}", SanitizeOrganizationRor(organizationRor), fileName);
}
public static Stream ConvertStringContentsToStream(string contents)
{
byte[] byteArray = Encoding.UTF8.GetBytes(contents);
return new MemoryStream(byteArray);
}
public void EnsureGitLabInformationIsSetAndCorrect()
{
if (!ReportingEnabled)
{
throw new ApplicationException($"\nReporting is deactivated on this machine! \nTo enable it, set the Consul Key \"coscine/local/reporting/enabled\" to \"true\".");
}
if (string.IsNullOrWhiteSpace(ReportingDatabaseProjectId) || string.IsNullOrWhiteSpace(ReportingBranch))
{
throw new ArgumentNullException($"\nNo valid Reporting Project ID or Branch were provided!");
}
var project = GitLabClient.Projects.GetAsync(ReportingDatabaseProjectId).Result;
Console.WriteLine($" - Report Generation to be uploaded to GitLab Project \"{project.Name}\" on branch \"{ReportingBranch}\"");
var branch = GitLabClient.Branches.GetAsync(project.Id, o => o.Search = ReportingBranch).Result;
if (!branch.Any(b => b.Name.Equals(ReportingBranch)) && Domain.Equals("DEVLEF") && !project.DefaultBranch.Equals(ReportingBranch))
{
Console.WriteLine($" - Branch \"{ReportingBranch}\" does not exist. Working on Domain {Domain}. Creating branch...");
GitLabClient.Branches.CreateAsync(ReportingDatabaseProjectId, new CreateBranchRequest(ReportingBranch, project.DefaultBranch)).Wait();
Console.WriteLine($" - Branch \"{ReportingBranch}\" successfully created");
}
else if (!branch.Any(b => b.Name.Equals(ReportingBranch)))
{
throw new ArgumentNullException($"\nBranch \"{ReportingBranch}\" does not exist!");
}
Console.WriteLine();
}
public string ConvertOldRwthOrganizationToRor(string organization)
{
// Converts values like https://www.rwth-aachen.de/22000 to its corresponding RoR (here: https://ror.org/04xfq0f34#ORG-42NHW)
var _queryString = new SparqlParameterizedString
{
CommandText = $@"
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
PREFIX org: <http://www.w3.org/ns/org#>
SELECT DISTINCT ?source ?inst
WHERE {{
# Define ?source URI
VALUES ?source
{{
<{organization}>
}} .
# Get Display Name of the institution
?source rdfs:label ?name .
# Get Institute Identifier in ?ikzid
?source org:identifier ?ikzid .
# IKZ ID is in the form 'ikz:<ID>' or 'ikz:0<ID>'
BIND(concat('ikz:', ?ikzid) AS ?ikz) .
BIND(concat('ikz:0', ?ikzid) AS ?ikz0) .
# Fetch all institutes from RWTH
<{RwthRor}> org:hasUnit ?inst .
# OR statement to search by ikz variations and name
{{ ?inst org:identifier ?ikz .}} UNION {{ ?inst org:identifier ?ikz0 .}} UNION {{ ?inst rdfs:label ?name .}}
}}
GROUP BY ?inst
"
};
using var results = RdfStoreConnector.QueryEndpoint.QueryWithResultSet(_queryString.ToString());
if (!results.IsEmpty)
{
var inst = results.Select(x => x.Value("inst").ToString()).ToList()[0]; // Get the value for ?inst
if (!string.IsNullOrWhiteSpace(inst))
{
Console.WriteLine($" Organization {organization} found to match {inst}");
return inst;
}
}
Console.WriteLine($" Could not find match for {organization}");
return organization;
}
}