Skip to content
Snippets Groups Projects
Commit 339c169d authored by Petar Hristov's avatar Petar Hristov :speech_balloon: Committed by Sirieam Marie Hunke
Browse files

New: Extended with User Reporting

parent a6316dc2
Branches
Tags
2 merge requests!5Release: Sprint/2022 18 :robot:,!3New: Extended with User Reporting
Show whitespace changes
Inline Side-by-side
src/KPI Generator/KPI Generator.csproj
+ 1
− 0
View file @ 339c169d
......@@ -19,6 +19,7 @@
<ItemGroup>
<PackageReference Include="CommandLineParser" Version="2.9.1" />
<PackageReference Include="Coscine.ApiCommons" Version="2.*-*" />
<PackageReference Include="Coscine.Database" Version="2.*-*" />
<PackageReference Include="Coscine.Metadata" Version="2.*-*" />
<PackageReference Include="GitLabApiClient" Version="1.8.1-beta.5" />
......
src/KPI Generator/Reporting.cs
+ 80
− 13
View file @ 339c169d
......@@ -3,6 +3,10 @@ using Coscine.Metadata;
using GitLabApiClient;
using GitLabApiClient.Models.Commits.Requests.CreateCommitRequest;
using KPIGenerator.Utils;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System.Text;
using VDS.RDF;
using VDS.RDF.Query;
using static KPIGenerator.Utils.CommandLineOptions;
......@@ -15,7 +19,7 @@ public abstract class Reporting<O> where O : class
public List<Organization> Organizations { get; set; }
public RdfStoreConnector RdfStoreConnector { get; init; }
public SparqlRemoteEndpoint QueryEndpoint { get; init; }
private ConsulConfiguration Configuration { get; }
public ConsulConfiguration Configuration { get; }
private static string HostUrl { get; } = "https://git.rwth-aachen.de/";
private static string InstanceName { get; set; } = null!;
public virtual string ReportingFileName { get; init; } = null!;
......@@ -42,19 +46,20 @@ public abstract class Reporting<O> where O : class
public bool Run()
{
// Console text output
Console.Write($"{new string('-', 35)}\n{InstanceName}");
Console.Write($"{new string('=', 60)}\n{InstanceName}");
var baseOptions = Options as BaseOptions;
if (baseOptions is not null && baseOptions.DummyMode)
{
Console.Write(" : DUMMY MODE");
}
Console.WriteLine($"\n{new string('-', 35)}");
Console.WriteLine($"\n{new string('-', 60)}");
// Generate Reporting based on CLI input
var reportingFiles = GenerateReporting();
Console.WriteLine($"\n{new string('=', 60)}");
Console.WriteLine("Reporting generated successfully. Publishing...");
// Publish Report
var success = PublishAsync(reportingFiles).Result;
Console.WriteLine(success ? "Reporting published successfully." : "Reporting publishing FAILED!");
Console.WriteLine(success ? "Published successfully." : "Publishing FAILED!");
return success;
}
......@@ -65,14 +70,22 @@ public abstract class Reporting<O> where O : class
{
// Retrieve Reporting Database project
var reportingDatabaseProject = await GitLabClient.Projects.GetAsync(ReportingDatabaseProjectId);
var commitBranch = reportingDatabaseProject.DefaultBranch;
var commitMessage = $"{InstanceName} Generated - {DateTime.Now:dd.MM.yyyy HH:mm}"; // CompleteReporting Generated - 31.08.2022 10:25
var projectTree = await GitLabClient.Trees.GetAsync(reportingDatabaseProject, o =>
{
o.Recursive = true;
o.Reference = commitBranch;
});
// Define commit actions
var actions = new List<CreateCommitRequestAction>();
// Create a commit per file with its contents
foreach (var file in files)
{
// Write file contents to bytes
byte[] bytes;
using (var ms = new MemoryStream())
{
......@@ -80,7 +93,17 @@ public abstract class Reporting<O> where O : class
bytes = ms.ToArray();
}
actions.Add(new CreateCommitRequestAction(CreateCommitRequestActionType.Create, file.Path)
// Distinguish between Creating or Updating a file
var actionType = CreateCommitRequestActionType.Create;
if (projectTree.Any(f => f.Path.Equals(file.Path)))
{
actionType = CreateCommitRequestActionType.Update;
}
// TODO: Add Mechanism to Delete files or organizations that are not valid anymore
// Add Action
actions.Add(new CreateCommitRequestAction(actionType, file.Path)
{
Content = Convert.ToBase64String(bytes),
Encoding = CreateCommitRequestActionEncoding.Base64
......@@ -90,7 +113,7 @@ public abstract class Reporting<O> where O : class
var baseOptions = Options as BaseOptions;
if (baseOptions is not null && !baseOptions.DummyMode)
{
await GitLabClient.Commits.CreateAsync(reportingDatabaseProject, new CreateCommitRequest(reportingDatabaseProject.DefaultBranch, commitMessage, actions));
await GitLabClient.Commits.CreateAsync(reportingDatabaseProject, new CreateCommitRequest(commitBranch, commitMessage, actions));
}
return true;
}
......@@ -104,14 +127,58 @@ public abstract class Reporting<O> where O : class
private List<Organization> FetchOrganizations()
{
var organizations = new List<Organization>();
var organizationsGroupId = 23782; // Organisations GitLab Group URL: https://git.rwth-aachen.de/coscine/graphs/organisations
var organizationsGroup = GitLabClient.Groups.GetAsync(organizationsGroupId).Result;
foreach (var project in organizationsGroup.Projects)
var organizationsToFind = JsonConvert.DeserializeObject<IEnumerable<Uri>>(
Configuration.GetStringAndWait("coscine/local/organizations/list",
"['https://ror.org/', 'https://ror.org/04xfq0f34']")
);
var resultSet = new List<Triple>();
foreach (var orgGraph in organizationsToFind)
{
resultSet.AddRange(RdfStoreConnector.GetTriples(orgGraph, null));
}
var organizationTriples = resultSet.Where(r => !r.Subject.ToString().Contains('#')).Distinct().ToList();
foreach (var triple in organizationTriples)
{
organizations.Add(new Organization
{
Name = triple.Object.ToString(),
Ror = triple.Subject.ToString(),
});
}
// Organization "Other"
var organizationOther = organizations.Find(o => o.Name.Equals("Other"));
var organizationOtherRor = "https://ror.org/_other";
if (organizationOther is null)
{
var name = project.Description[..project.Description.IndexOf(':')];
var ror = project.Name;
organizations.Add(new Organization(name, ror));
organizations.Add(new Organization
{
Name = "Other",
Ror = organizationOtherRor
});
}
else
{
var index = organizations.IndexOf(organizationOther);
organizations[index].Ror = organizationOtherRor;
}
return organizations;
}
public static string GetReportingPathGeneral(string fileName)
{
return string.Format("General/{0}", fileName);
}
public static string GetReportingPathOrganization(string organizationRor, string fileName)
{
return string.Format("Organizations/{0}/{1}", organizationRor, fileName);
}
public static Stream ConvertStringContentsToStream(string contents)
{
byte[] byteArray = Encoding.UTF8.GetBytes(contents);
return new MemoryStream(byteArray);
}
}
src/KPI Generator/Reportings/User/ReturnObject.cs
+ 22
− 1
View file @ 339c169d
namespace KPIGenerator.Reportings.User;
using Coscine.Database.ReturnObjects;
using Newtonsoft.Json;
namespace KPIGenerator.Reportings.User;
/// <summary>
/// Object containing the JSON structure for the reporting
/// </summary>
public class ReturnObject
{
[JsonProperty("related_projects")]
public List<RelatedProject> RelatedProjects { get; set; } = new();
[JsonProperty("organizations")]
public List<string> Organizations { get; set; } = new();
[JsonProperty("institutes")]
public List<string> Institutes { get; set; } = new();
[JsonProperty("disciplines")]
public List<DisciplineObject> Disciplines { get; set; } = new();
[JsonProperty("login_providers")]
public List<ExternalAuthenticatorsObject> LoginProviders { get; set; } = new();
[JsonProperty("latest_activity")]
public DateTime? LatestActivity { get; set; } = null;
public class RelatedProject
{
public Guid ProjectId { get; set; }
public string Role { get; set; } = null!;
}
}
src/KPI Generator/Reportings/User/UserReporting.cs
+ 165
− 7
View file @ 339c169d
using KPIGenerator.Utils;
using Coscine.ApiCommons;
using Coscine.Database.Models;
using Coscine.Metadata;
using KPIGenerator.Utils;
using Newtonsoft.Json;
using static KPIGenerator.Utils.CommandLineOptions;
namespace KPIGenerator.Reportings.User;
public class UserReporting : Reporting<UserReportingOptions>
{
private readonly Authenticator _authenticator;
private readonly ExternalAuthenticatorModel _externalAuthenticatorModel;
private readonly ExternalIdModel _externalIdModel;
private readonly ProjectRoleModel _projectRoleModel;
private readonly ProjectModel _projectModel;
private readonly RoleModel _roleModel;
private readonly UserModel _userModel;
private readonly LogModel _logModel;
public UserReporting(UserReportingOptions options) : base(options)
{
ReportingFileName = "users.json";
_authenticator = new Authenticator(null, Configuration);
_externalAuthenticatorModel = new ExternalAuthenticatorModel();
_externalIdModel = new ExternalIdModel();
_projectRoleModel = new ProjectRoleModel();
_projectModel = new ProjectModel();
_roleModel = new RoleModel();
_userModel = new UserModel();
_logModel = new LogModel();
}
public override IEnumerable<ReportingFileObject> GenerateReporting()
{
/*
* 1. Collect the reporting for the whole database -- General/{ReportingReportingFileName}
* 2. Append to the list the same information per organization as folders -- Organizations/{OrgRorId}/{ReportingReportingFileName}
* --> See envisioned folder structure.
*/
throw new NotImplementedException();
var users = _userModel.GetAllWhere((user) => user.Tosaccepteds.Any());
var reportingFiles = new List<ReportingFileObject>();
var returnObjects = Generate(users);
// General File
reportingFiles.Add(new ReportingFileObject
{
Path = GetReportingPathGeneral(ReportingFileName),
Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjects, Formatting.Indented))
});
// Per Organization
reportingFiles.AddRange(GeneratePerOrganization(returnObjects));
return reportingFiles;
}
private List<ReturnObject> Generate(IEnumerable<Coscine.Database.DataModel.User> users)
{
var returnObjects = new List<ReturnObject>();
foreach (var user in users)
{
var userReturnObject = _userModel.CreateReturnObjectFromDatabaseObject(user);
var userReportEntry = new ReturnObject
{
RelatedProjects = GetRelatedProjects(user.Id),
Organizations = GetOrganizations(user.Id, userReturnObject.Organization),
Institutes = GetInstitutes(user.Id, userReturnObject.Institute),
Disciplines = userReturnObject.Disciplines.ToList(),
LoginProviders = userReturnObject.ExternalAuthenticators.ToList(),
LatestActivity = GetLatestActivity(user.Id)
};
returnObjects.Add(userReportEntry);
}
return returnObjects;
}
private IEnumerable<ReportingFileObject> GeneratePerOrganization(List<ReturnObject> returnObjects)
{
var reportingFilesPerOrganization = new List<ReportingFileObject>();
var organizationsFromUsers = returnObjects.SelectMany(ro => ro.Organizations).Distinct();
foreach (var entry in organizationsFromUsers)
{
var organization = Organizations.Find(o => o.Name.Equals(entry));
if (organization is null)
{
organization = Organizations.Find(o => o.Name.Equals("Other"));
Console.WriteLine($"WARNING!: Organization \"{entry}\" could not be correctly identified. Will use \"{organization!.Name}\".");
}
var returnObjectsForOrganization = returnObjects.Where(ro => ro.Organizations.Contains(entry));
reportingFilesPerOrganization.Add(new ReportingFileObject
{
Path = GetReportingPathOrganization(organization.Ror.Replace("https://ror.org/", "").ToLower(), ReportingFileName),
Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjectsForOrganization, Formatting.Indented))
});
}
return reportingFilesPerOrganization;
}
private List<ReturnObject.RelatedProject> GetRelatedProjects(Guid id)
{
var result = new List<ReturnObject.RelatedProject>();
var projectRoles = _projectRoleModel.GetAllWhere(role => role.UserId.Equals(id));
if (projectRoles.Any())
{
foreach (var projectRole in projectRoles)
{
if (_projectModel.GetById(projectRole.ProjectId) is not null) // null if project has been deleted
{
result.Add(new ReturnObject.RelatedProject
{
ProjectId = projectRole.ProjectId,
Role = _roleModel.GetById(projectRole.RoleId).DisplayName
});
}
}
}
return result;
}
private List<string> GetOrganizations(Guid id, string organization)
{
var externalIdModel = new ExternalIdModel();
var result = new List<string>();
if (!string.IsNullOrWhiteSpace(organization))
{
result.Add(organization);
}
var externalIds = externalIdModel.GetAllWhere((externalId) => externalId.UserId.Equals(id));
var externalIdList = new List<string>();
foreach (var externalId in externalIds)
{
externalIdList.Add(externalId.ExternalId1);
}
var resultSet = RdfStoreConnector.GetTriples(null, null, null, 1, externalIdList);
var organizationTriples = resultSet.Where(r => !r.Subject.ToString().Contains('#')).Distinct().ToList();
foreach (var triple in organizationTriples)
{
result.Add(triple.Object.ToString());
}
return result;
}
private List<string> GetInstitutes(Guid id, string institute)
{
var result = new List<string>();
if (!string.IsNullOrWhiteSpace(institute))
{
result.Add(institute);
}
var externalIds = _externalIdModel.GetAllWhere((externalId) => externalId.UserId.Equals(id));
var externalIdList = new List<string>();
foreach (var externalId in externalIds)
{
externalIdList.Add(externalId.ExternalId1);
}
var resultSet = RdfStoreConnector.GetTriples(null, null, null, 1, externalIdList);
var instituteTriples = resultSet.Where(r => r.Subject.ToString().Contains('#')).Distinct().ToList();
foreach (var triple in instituteTriples)
{
result.Add(triple.Object.ToString());
}
return result;
}
private DateTime? GetLatestActivity(Guid id)
{
var latestLog = _logModel.GetAllWhere(l => l.LogLevel.Equals("Analytics") && l.UserId.Equals(id)).OrderByDescending(a => a.ServerTimestamp).FirstOrDefault();
if (latestLog is not null)
{
return latestLog.ServerTimestamp;
}
else return null;
}
}
src/KPI Generator/Utils/Organization.cs
+ 1
− 9
View file @ 339c169d
......@@ -16,12 +16,4 @@ public class Organization
/// Organizaiton ROR URL from GitLab project's title
/// </summary>
/// <example>https://ror.org/04xfq0f34</example>
public Uri RorUri { get; set; } = null!;
public Organization(string name, string ror)
{
Name = name;
Ror = ror;
RorUri = new Uri($"https://ror.org/{ror}");
}
}
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment