Skip to content
Snippets Groups Projects
Commit 8585508c authored by Petar Hristov's avatar Petar Hristov :speech_balloon:
Browse files

Finished

parent 523fc518
No related branches found
No related tags found
2 merge requests!4New: Extended with Application Profile Reporting,!3New: Extended with User Reporting
Pipeline #803632 passed
Stage: build
Show whitespace changes
Inline Side-by-side
src/KPI Generator/Reporting.cs
+ 57
− 5
View file @ 8585508c
...@@ -5,6 +5,7 @@ using GitLabApiClient.Models.Commits.Requests.CreateCommitRequest; ...@@ -5,6 +5,7 @@ using GitLabApiClient.Models.Commits.Requests.CreateCommitRequest;
using KPIGenerator.Utils; using KPIGenerator.Utils;
using Newtonsoft.Json; using Newtonsoft.Json;
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
using System.Text;
using VDS.RDF; using VDS.RDF;
using VDS.RDF.Query; using VDS.RDF.Query;
using static KPIGenerator.Utils.CommandLineOptions; using static KPIGenerator.Utils.CommandLineOptions;
...@@ -45,19 +46,20 @@ public abstract class Reporting<O> where O : class ...@@ -45,19 +46,20 @@ public abstract class Reporting<O> where O : class
public bool Run() public bool Run()
{ {
// Console text output // Console text output
Console.Write($"{new string('-', 35)}\n{InstanceName}"); Console.Write($"{new string('=', 60)}\n{InstanceName}");
var baseOptions = Options as BaseOptions; var baseOptions = Options as BaseOptions;
if (baseOptions is not null && baseOptions.DummyMode) if (baseOptions is not null && baseOptions.DummyMode)
{ {
Console.Write(" : DUMMY MODE"); Console.Write(" : DUMMY MODE");
} }
Console.WriteLine($"\n{new string('-', 35)}"); Console.WriteLine($"\n{new string('-', 60)}");
// Generate Reporting based on CLI input // Generate Reporting based on CLI input
var reportingFiles = GenerateReporting(); var reportingFiles = GenerateReporting();
Console.WriteLine($"\n{new string('=', 60)}");
Console.WriteLine("Reporting generated successfully. Publishing..."); Console.WriteLine("Reporting generated successfully. Publishing...");
// Publish Report // Publish Report
var success = PublishAsync(reportingFiles).Result; var success = PublishAsync(reportingFiles).Result;
Console.WriteLine(success ? "Reporting published successfully." : "Reporting publishing FAILED!"); Console.WriteLine(success ? "Published successfully." : "Publishing FAILED!");
return success; return success;
} }
...@@ -68,14 +70,22 @@ public abstract class Reporting<O> where O : class ...@@ -68,14 +70,22 @@ public abstract class Reporting<O> where O : class
{ {
// Retrieve Reporting Database project // Retrieve Reporting Database project
var reportingDatabaseProject = await GitLabClient.Projects.GetAsync(ReportingDatabaseProjectId); var reportingDatabaseProject = await GitLabClient.Projects.GetAsync(ReportingDatabaseProjectId);
var commitBranch = "Issue/2182-kpiGeneratorUser"; // reportingDatabaseProject.DefaultBranch;
var commitMessage = $"{InstanceName} Generated - {DateTime.Now:dd.MM.yyyy HH:mm}"; // CompleteReporting Generated - 31.08.2022 10:25 var commitMessage = $"{InstanceName} Generated - {DateTime.Now:dd.MM.yyyy HH:mm}"; // CompleteReporting Generated - 31.08.2022 10:25
var projectTree = await GitLabClient.Trees.GetAsync(reportingDatabaseProject, o =>
{
o.Recursive = true;
o.Reference = commitBranch;
});
// Define commit actions // Define commit actions
var actions = new List<CreateCommitRequestAction>(); var actions = new List<CreateCommitRequestAction>();
// Create a commit per file with its contents // Create a commit per file with its contents
foreach (var file in files) foreach (var file in files)
{ {
// Write file contents to bytes
byte[] bytes; byte[] bytes;
using (var ms = new MemoryStream()) using (var ms = new MemoryStream())
{ {
...@@ -83,7 +93,17 @@ public abstract class Reporting<O> where O : class ...@@ -83,7 +93,17 @@ public abstract class Reporting<O> where O : class
bytes = ms.ToArray(); bytes = ms.ToArray();
} }
actions.Add(new CreateCommitRequestAction(CreateCommitRequestActionType.Create, file.Path) // Distinguish between Creating or Updating a file
var actionType = CreateCommitRequestActionType.Create;
if (projectTree.Any(f => f.Path.Equals(file.Path)))
{
actionType = CreateCommitRequestActionType.Update;
}
// TODO: Add Mechanism to Delete files or organizations that are not valid anymore
// Add Action
actions.Add(new CreateCommitRequestAction(actionType, file.Path)
{ {
Content = Convert.ToBase64String(bytes), Content = Convert.ToBase64String(bytes),
Encoding = CreateCommitRequestActionEncoding.Base64 Encoding = CreateCommitRequestActionEncoding.Base64
...@@ -93,7 +113,7 @@ public abstract class Reporting<O> where O : class ...@@ -93,7 +113,7 @@ public abstract class Reporting<O> where O : class
var baseOptions = Options as BaseOptions; var baseOptions = Options as BaseOptions;
if (baseOptions is not null && !baseOptions.DummyMode) if (baseOptions is not null && !baseOptions.DummyMode)
{ {
await GitLabClient.Commits.CreateAsync(reportingDatabaseProject, new CreateCommitRequest(reportingDatabaseProject.DefaultBranch, commitMessage, actions)); await GitLabClient.Commits.CreateAsync(reportingDatabaseProject, new CreateCommitRequest(commitBranch, commitMessage, actions));
} }
return true; return true;
} }
...@@ -127,6 +147,38 @@ public abstract class Reporting<O> where O : class ...@@ -127,6 +147,38 @@ public abstract class Reporting<O> where O : class
Ror = triple.Subject.ToString(), Ror = triple.Subject.ToString(),
}); });
} }
// Organization "Other"
var organizationOther = organizations.Find(o => o.Name.Equals("Other"));
var organizationOtherRor = "https://ror.org/_other";
if (organizationOther is null)
{
organizations.Add(new Organization
{
Name = "Other",
Ror = organizationOtherRor
});
}
else
{
var index = organizations.IndexOf(organizationOther);
organizations[index].Ror = organizationOtherRor;
}
return organizations; return organizations;
} }
public static string GetReportingPathGeneral(string fileName)
{
return string.Format("General/{0}", fileName);
}
public static string GetReportingPathOrganization(string organizationRor, string fileName)
{
return string.Format("Organizations/{0}/{1}", organizationRor, fileName);
}
public static Stream ConvertStringContentsToStream(string contents)
{
byte[] byteArray = Encoding.UTF8.GetBytes(contents);
return new MemoryStream(byteArray);
}
} }
src/KPI Generator/Reportings/User/ReturnObject.cs
+ 4
− 3
View file @ 8585508c
using Newtonsoft.Json; using Coscine.Database.ReturnObjects;
using Newtonsoft.Json;
namespace KPIGenerator.Reportings.User; namespace KPIGenerator.Reportings.User;
...@@ -14,9 +15,9 @@ public class ReturnObject ...@@ -14,9 +15,9 @@ public class ReturnObject
[JsonProperty("institutes")] [JsonProperty("institutes")]
public List<string> Institutes { get; set; } = new(); public List<string> Institutes { get; set; } = new();
[JsonProperty("disciplines")] [JsonProperty("disciplines")]
public List<string> Disciplines { get; set; } = new(); public List<DisciplineObject> Disciplines { get; set; } = new();
[JsonProperty("login_providers")] [JsonProperty("login_providers")]
public List<string> LoginProviders { get; set; } = new(); public List<ExternalAuthenticatorsObject> LoginProviders { get; set; } = new();
[JsonProperty("latest_activity")] [JsonProperty("latest_activity")]
public DateTime? LatestActivity { get; set; } = null; public DateTime? LatestActivity { get; set; } = null;
......
src/KPI Generator/Reportings/User/UserReporting.cs
+ 101
− 64
View file @ 8585508c
using Coscine.ApiCommons; using Coscine.ApiCommons;
using Coscine.Configuration;
using Coscine.Database.Models; using Coscine.Database.Models;
using Coscine.Metadata; using Coscine.Metadata;
using KPIGenerator.Utils; using KPIGenerator.Utils;
using Newtonsoft.Json;
using static KPIGenerator.Utils.CommandLineOptions; using static KPIGenerator.Utils.CommandLineOptions;
namespace KPIGenerator.Reportings.User; namespace KPIGenerator.Reportings.User;
...@@ -10,58 +10,102 @@ namespace KPIGenerator.Reportings.User; ...@@ -10,58 +10,102 @@ namespace KPIGenerator.Reportings.User;
public class UserReporting : Reporting<UserReportingOptions> public class UserReporting : Reporting<UserReportingOptions>
{ {
private readonly Authenticator _authenticator; private readonly Authenticator _authenticator;
private readonly ExternalAuthenticatorModel _externalAuthenticatorModel;
private readonly ExternalIdModel _externalIdModel;
private readonly ProjectRoleModel _projectRoleModel;
private readonly ProjectModel _projectModel;
private readonly RoleModel _roleModel;
private readonly UserModel _userModel;
private readonly LogModel _logModel;
public UserReporting(UserReportingOptions options) : base(options) public UserReporting(UserReportingOptions options) : base(options)
{ {
ReportingFileName = "users.json"; ReportingFileName = "users.json";
_authenticator = new Authenticator(null, Configuration); _authenticator = new Authenticator(null, Configuration);
_externalAuthenticatorModel = new ExternalAuthenticatorModel();
_externalIdModel = new ExternalIdModel();
_projectRoleModel = new ProjectRoleModel();
_projectModel = new ProjectModel();
_roleModel = new RoleModel();
_userModel = new UserModel();
_logModel = new LogModel();
} }
public override IEnumerable<ReportingFileObject> GenerateReporting() public override IEnumerable<ReportingFileObject> GenerateReporting()
{ {
/* var users = _userModel.GetAllWhere((user) => user.Tosaccepteds.Any());
* 1. Collect the reporting for the whole database -- General/{ReportingReportingFileName} var reportingFiles = new List<ReportingFileObject>();
* 2. Append to the list the same information per organization as folders -- Organizations/{OrgRorId}/{ReportingReportingFileName} var returnObjects = Generate(users);
* --> See envisioned folder structure.
*/ // General File
var userModel = new UserModel(); reportingFiles.Add(new ReportingFileObject
var users = userModel.GetAll(); {
var result = new List<ReportingFileObject>(); Path = GetReportingPathGeneral(ReportingFileName),
var generalReportingFileContents = new List<ReturnObject>(); Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjects, Formatting.Indented))
//users = users.Where(u => u.DisplayName.Contains("Petar")); });
// Per Organization
reportingFiles.AddRange(GeneratePerOrganization(returnObjects));
return reportingFiles;
}
private List<ReturnObject> Generate(IEnumerable<Coscine.Database.DataModel.User> users)
{
var returnObjects = new List<ReturnObject>();
foreach (var user in users) foreach (var user in users)
{ {
var userReturnObject = _userModel.CreateReturnObjectFromDatabaseObject(user);
var userReportEntry = new ReturnObject var userReportEntry = new ReturnObject
{ {
RelatedProjecs = GetRelatedProjects(user.Id), RelatedProjecs = GetRelatedProjects(user.Id),
Organizations = GetOrganizations(user.Id), Organizations = GetOrganizations(user.Id, userReturnObject.Organization),
Institutes = GetInstitutes(user.Id), Institutes = GetInstitutes(user.Id, userReturnObject.Institute),
LoginProviders = GetLoginProviders(user.Id), Disciplines = userReturnObject.Disciplines.ToList(),
LoginProviders = userReturnObject.ExternalAuthenticators.ToList(),
LatestActivity = GetLatestActivity(user.Id) LatestActivity = GetLatestActivity(user.Id)
}; };
userReportEntry.Disciplines = GetDisciplines(user.Id, userReportEntry.RelatedProjecs); returnObjects.Add(userReportEntry);
generalReportingFileContents.Add(userReportEntry);
} }
return result; return returnObjects;
} }
private static List<ReturnObject.RelatedProject> GetRelatedProjects(Guid id) private IEnumerable<ReportingFileObject> GeneratePerOrganization(List<ReturnObject> returnObjects)
{
var reportingFilesPerOrganization = new List<ReportingFileObject>();
var organizationsFromUsers = returnObjects.SelectMany(ro => ro.Organizations).Distinct();
foreach (var entry in organizationsFromUsers)
{
var organization = Organizations.Find(o => o.Name.Equals(entry));
if (organization is null)
{
organization = Organizations.Find(o => o.Name.Equals("Other"));
Console.WriteLine($"WARNING!: Organization \"{entry}\" could not be correctly identified. Will use \"{organization!.Name}\".");
}
var returnObjectsForOrganization = returnObjects.Where(ro => ro.Organizations.Contains(entry));
reportingFilesPerOrganization.Add(new ReportingFileObject
{
Path = GetReportingPathOrganization(organization.Ror.Replace("https://ror.org/", "").ToLower(), ReportingFileName),
Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjectsForOrganization, Formatting.Indented))
});
}
return reportingFilesPerOrganization;
}
private List<ReturnObject.RelatedProject> GetRelatedProjects(Guid id)
{ {
var projectRoleModel = new ProjectRoleModel();
var projectModel = new ProjectModel();
var roleModel = new RoleModel();
var result = new List<ReturnObject.RelatedProject>(); var result = new List<ReturnObject.RelatedProject>();
var projectRoles = projectRoleModel.GetAllWhere(role => role.UserId.Equals(id)); var projectRoles = _projectRoleModel.GetAllWhere(role => role.UserId.Equals(id));
if (projectRoles.Any()) if (projectRoles.Any())
{ {
foreach (var projectRole in projectRoles) foreach (var projectRole in projectRoles)
{ {
if (projectModel.GetById(projectRole.ProjectId) is not null) // null if project has been deleted if (_projectModel.GetById(projectRole.ProjectId) is not null) // null if project has been deleted
{ {
result.Add(new ReturnObject.RelatedProject result.Add(new ReturnObject.RelatedProject
{ {
ProjectId = projectRole.ProjectId, ProjectId = projectRole.ProjectId,
Role = roleModel.GetById(projectRole.RoleId).DisplayName Role = _roleModel.GetById(projectRole.RoleId).DisplayName
}); });
} }
} }
...@@ -69,71 +113,64 @@ public class UserReporting : Reporting<UserReportingOptions> ...@@ -69,71 +113,64 @@ public class UserReporting : Reporting<UserReportingOptions>
return result; return result;
} }
private List<string> GetOrganizations(Guid id) private List<string> GetOrganizations(Guid id, string organization)
{ {
var externalIdModel = new ExternalIdModel(); var externalIdModel = new ExternalIdModel();
var result = new List<string>(); var result = new List<string>();
if (!string.IsNullOrWhiteSpace(organization))
{
result.Add(organization);
}
var externalIds = externalIdModel.GetAllWhere((externalId) => externalId.UserId.Equals(id)); var externalIds = externalIdModel.GetAllWhere((externalId) => externalId.UserId.Equals(id));
var externalIdList = new List<string>(); var externalIdList = new List<string>();
foreach (var externalId in externalIds) foreach (var externalId in externalIds)
{ {
externalIdList.Add(externalId.ExternalId1); externalIdList.Add(externalId.ExternalId1);
} }
var test = RdfStoreConnector.GetTriples(null, null, null, 1, externalIdList); var resultSet = RdfStoreConnector.GetTriples(null, null, null, 1, externalIdList);
return result; var organizationTriples = resultSet.Where(r => !r.Subject.ToString().Contains('#')).Distinct().ToList();
}
private static List<string> GetInstitutes(Guid id) foreach (var triple in organizationTriples)
{ {
var result = new List<string>(); result.Add(triple.Object.ToString());
}
return result; return result;
} }
private List<string> GetDisciplines(Guid id, List<ReturnObject.RelatedProject> relatedProjects) private List<string> GetInstitutes(Guid id, string institute)
{ {
var stuff = _authenticator.GetUser(id.ToString());
var userModel = new UserModel();
var idkman = userModel.CreateReturnObjectFromDatabaseObject(stuff);
var projectDisciplineModel = new ProjectDisciplineModel();
var disciplineModel = new DisciplineModel();
var result = new List<string>(); var result = new List<string>();
foreach (var relatedProject in relatedProjects)
{ if (!string.IsNullOrWhiteSpace(institute))
var projectDisciplines = projectDisciplineModel.GetAllWhere(d => d.ProjectId.Equals(relatedProject.ProjectId));
if (projectDisciplines.Any())
{
foreach (var projectDiscipline in projectDisciplines)
{ {
result.Add(disciplineModel.GetById(projectDiscipline.DisciplineId).DisplayNameEn); result.Add(institute);
}
}
}
return result.Distinct().ToList();
} }
private static List<string> GetLoginProviders(Guid id) var externalIds = _externalIdModel.GetAllWhere((externalId) => externalId.UserId.Equals(id));
{ var externalIdList = new List<string>();
var externalAuthenticatorModel = new ExternalAuthenticatorModel(); foreach (var externalId in externalIds)
var externalIdModel = new ExternalIdModel();
var result = new List<string>();
var userExternalLogins = externalIdModel.GetAllWhere(login => login.UserId.Equals(id));
if (userExternalLogins.Any())
{
foreach (var externalLogin in userExternalLogins)
{ {
result.Add(externalAuthenticatorModel.GetById(externalLogin.ExternalAuthenticatorId).DisplayName); externalIdList.Add(externalId.ExternalId1);
} }
var resultSet = RdfStoreConnector.GetTriples(null, null, null, 1, externalIdList);
var instituteTriples = resultSet.Where(r => r.Subject.ToString().Contains('#')).Distinct().ToList();
foreach (var triple in instituteTriples)
{
result.Add(triple.Object.ToString());
} }
return result; return result;
} }
private static DateTime? GetLatestActivity(Guid id) private DateTime? GetLatestActivity(Guid id)
{ {
var logModel = new LogModel(); var latestLog = _logModel.GetAllWhere(l => l.LogLevel.Equals("Analytics") && l.UserId.Equals(id)).OrderByDescending(a => a.ServerTimestamp).FirstOrDefault();
var latestLog = logModel.GetAllWhere(l => l.LogLevel.Equals("Analytics") && l.UserId.Equals(id)).OrderByDescending(a => a.ServerTimestamp).FirstOrDefault();
if (latestLog is not null) if (latestLog is not null)
{ {
return latestLog.ServerTimestamp; return latestLog.ServerTimestamp;
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment