Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found
Select Git revision
  • Fix/xxxx-indexOutOfRange
  • Fix/xxxx-minorFixes
  • Fix/xxxx-organization
  • Fix/xxxx-wrap
  • Hotfix/2332-userInstitutesInReporting
  • Hotfix/2388-sensitive
  • Hotfix/3115-userReportingEmpty
  • Hotfix/3115-userReportingEmpty2
  • Hotfix/xxxx-rors
  • Issue/2181-kpiGeneratorBase
  • Issue/2182-kpiGeneratorUser
  • Issue/2183-kpiGeneratorResource
  • Issue/2184-kpiGeneratorProject
  • Issue/2185-kpiGeneratorAP
  • Issue/2186-systemStatusReporting
  • Issue/2283-activityFix
  • Issue/2304-virtuosoRoars
  • Issue/2330-fixNaNQuotainAdmin
  • Issue/2432-publicationKpi
  • Issue/2492-respOrg
  • Issue/2518-docs
  • Issue/2568-betterLogging
  • Issue/2666-adminCronjobs
  • Issue/2666-adminCronjobs-theSequal
  • Issue/2847-reporting
  • Issue/2850-removeGrantId
  • Issue/2982-kpiDataPub
  • Issue/3005-kpiReportingBroken
  • Issue/3073-kpi
  • Issue/3142-kpiGenerator
  • dev
  • gitkeep
  • main
  • v0.1.0
  • v0.1.1
  • v0.1.10
  • v0.1.11
  • v0.1.12
  • v0.1.13
  • v0.1.14
  • v0.1.15
  • v0.1.16
  • v0.1.17
  • v0.1.18
  • v0.1.19
  • v0.1.2
  • v0.1.20
  • v0.1.21
  • v0.1.22
  • v0.1.23
  • v0.1.3
  • v0.1.4
  • v0.1.5
  • v0.1.6
  • v0.1.7
  • v0.1.8
  • v0.1.9
  • v1.0.1
  • v1.0.2
  • v1.0.3
  • v1.0.4
  • v1.0.5
  • v1.0.6
  • v1.0.7
  • v1.0.8
  • v1.0.9
  • v1.1.0
  • v1.1.1
  • v1.2.0
  • v1.2.1
  • v1.2.10
  • v1.2.2
  • v1.2.3
  • v1.2.4
  • v1.2.5
  • v1.2.6
  • v1.2.7
  • v1.2.8
  • v1.2.9
79 results

Target

Select target project
  • coscine/backend/scripts/kpi-generator
1 result
Select Git revision
  • Fix/xxxx-indexOutOfRange
  • Fix/xxxx-minorFixes
  • Fix/xxxx-organization
  • Fix/xxxx-wrap
  • Hotfix/2332-userInstitutesInReporting
  • Hotfix/2388-sensitive
  • Hotfix/3115-userReportingEmpty
  • Hotfix/3115-userReportingEmpty2
  • Hotfix/xxxx-rors
  • Issue/2181-kpiGeneratorBase
  • Issue/2182-kpiGeneratorUser
  • Issue/2183-kpiGeneratorResource
  • Issue/2184-kpiGeneratorProject
  • Issue/2185-kpiGeneratorAP
  • Issue/2186-systemStatusReporting
  • Issue/2283-activityFix
  • Issue/2304-virtuosoRoars
  • Issue/2330-fixNaNQuotainAdmin
  • Issue/2432-publicationKpi
  • Issue/2492-respOrg
  • Issue/2518-docs
  • Issue/2568-betterLogging
  • Issue/2666-adminCronjobs
  • Issue/2666-adminCronjobs-theSequal
  • Issue/2847-reporting
  • Issue/2850-removeGrantId
  • Issue/2982-kpiDataPub
  • Issue/3005-kpiReportingBroken
  • Issue/3073-kpi
  • Issue/3142-kpiGenerator
  • dev
  • gitkeep
  • main
  • v0.1.0
  • v0.1.1
  • v0.1.10
  • v0.1.11
  • v0.1.12
  • v0.1.13
  • v0.1.14
  • v0.1.15
  • v0.1.16
  • v0.1.17
  • v0.1.18
  • v0.1.19
  • v0.1.2
  • v0.1.20
  • v0.1.21
  • v0.1.22
  • v0.1.23
  • v0.1.3
  • v0.1.4
  • v0.1.5
  • v0.1.6
  • v0.1.7
  • v0.1.8
  • v0.1.9
  • v1.0.1
  • v1.0.2
  • v1.0.3
  • v1.0.4
  • v1.0.5
  • v1.0.6
  • v1.0.7
  • v1.0.8
  • v1.0.9
  • v1.1.0
  • v1.1.1
  • v1.2.0
  • v1.2.1
  • v1.2.10
  • v1.2.2
  • v1.2.3
  • v1.2.4
  • v1.2.5
  • v1.2.6
  • v1.2.7
  • v1.2.8
  • v1.2.9
79 results
Show changes
Commits on Source (3)
...@@ -7,7 +7,7 @@ ...@@ -7,7 +7,7 @@
<AssemblyName>Coscine.KpiGenerator</AssemblyName> <AssemblyName>Coscine.KpiGenerator</AssemblyName>
<ImplicitUsings>enable</ImplicitUsings> <ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable> <Nullable>enable</Nullable>
<Version>0.1.3</Version> <Version>0.1.4</Version>
</PropertyGroup> </PropertyGroup>
<PropertyGroup> <PropertyGroup>
......
using Coscine.Configuration; using Coscine.Configuration;
using Coscine.Metadata; using Coscine.Metadata;
using GitLabApiClient; using GitLabApiClient;
using GitLabApiClient.Models.Branches.Requests;
using GitLabApiClient.Models.Commits.Requests.CreateCommitRequest; using GitLabApiClient.Models.Commits.Requests.CreateCommitRequest;
using KPIGenerator.Utils; using KPIGenerator.Utils;
using System.Text; using System.Text;
using System.Web;
using VDS.RDF.Query; using VDS.RDF.Query;
using static KPIGenerator.Utils.CommandLineOptions; using static KPIGenerator.Utils.CommandLineOptions;
...@@ -21,18 +23,18 @@ public abstract class Reporting<O> where O : class ...@@ -21,18 +23,18 @@ public abstract class Reporting<O> where O : class
private static string InstanceName { get; set; } = null!; private static string InstanceName { get; set; } = null!;
public virtual string ReportingFileName { get; init; } = null!; public virtual string ReportingFileName { get; init; } = null!;
private string Domain { get; init; }
private bool ReportingEnabled { get; init; }
private string ReportingDatabaseProjectId { get; init; }
private string ReportingBranch { get; init; }
public readonly Organization _otherOrganization = new() public readonly Organization _otherOrganization = new()
{ {
Name = "Other", Name = "Other",
RorUrl = "https://ror.org/_other", RorUrl = "https://ror.org/_other",
}; };
/// <summary>
/// Reporting Database GitLab Project URL
/// </summary>
/// <remarks>https://git.rwth-aachen.de/coscine/reporting/reporting-database</remarks>
private static readonly int ReportingDatabaseProjectId = 75304;
public Reporting(O options) public Reporting(O options)
{ {
InstanceName = this.GetType().Name; InstanceName = this.GetType().Name;
...@@ -42,6 +44,11 @@ public abstract class Reporting<O> where O : class ...@@ -42,6 +44,11 @@ public abstract class Reporting<O> where O : class
QueryEndpoint = new SparqlRemoteEndpoint(new Uri(Configuration.GetStringAndWait("coscine/local/virtuoso/additional/url"))); QueryEndpoint = new SparqlRemoteEndpoint(new Uri(Configuration.GetStringAndWait("coscine/local/virtuoso/additional/url")));
GitLabClient = new GitLabClient(HostUrl, Configuration.GetStringAndWait("coscine/global/gitlabtoken")); GitLabClient = new GitLabClient(HostUrl, Configuration.GetStringAndWait("coscine/global/gitlabtoken"));
Organizations = new List<Organization>() { _otherOrganization }; Organizations = new List<Organization>() { _otherOrganization };
ReportingEnabled = Configuration.GetStringAndWait("coscine/local/reporting/enabled", "false") == "true";
Domain = Configuration.GetStringAndWait("coscine/local/profilesync/domain");
ReportingDatabaseProjectId = Configuration.GetStringAndWait("coscine/local/reporting/gitlab_project_id");
ReportingBranch = Configuration.GetStringAndWait("coscine/local/reporting/branch");
} }
public abstract IEnumerable<ReportingFileObject> GenerateReporting(); public abstract IEnumerable<ReportingFileObject> GenerateReporting();
...@@ -49,13 +56,14 @@ public abstract class Reporting<O> where O : class ...@@ -49,13 +56,14 @@ public abstract class Reporting<O> where O : class
public bool Run() public bool Run()
{ {
// Console text output // Console text output
Console.Write($"{new string('=', 80)}\n{InstanceName}"); Console.Write($"{new string('=', 80)}\n {Domain} | {InstanceName}");
var baseOptions = Options as BaseOptions; var baseOptions = Options as BaseOptions;
if (baseOptions is not null && baseOptions.DummyMode) if (baseOptions is not null && baseOptions.DummyMode)
{ {
Console.Write(" : DUMMY MODE"); Console.Write(" : DUMMY MODE");
} }
Console.WriteLine($"\n{new string('-', 80)}"); Console.WriteLine($"\n{new string('-', 80)}");
EnsureGitLabInformationIsSetAndCorrect();
// Generate Reporting based on CLI input // Generate Reporting based on CLI input
var reportingFiles = GenerateReporting(); var reportingFiles = GenerateReporting();
Console.WriteLine($"\n{new string('=', 80)}"); Console.WriteLine($"\n{new string('=', 80)}");
...@@ -72,7 +80,7 @@ public abstract class Reporting<O> where O : class ...@@ -72,7 +80,7 @@ public abstract class Reporting<O> where O : class
{ {
// Retrieve Reporting Database project // Retrieve Reporting Database project
var reportingDatabaseProject = await GitLabClient.Projects.GetAsync(ReportingDatabaseProjectId); var reportingDatabaseProject = await GitLabClient.Projects.GetAsync(ReportingDatabaseProjectId);
var commitBranch = reportingDatabaseProject.DefaultBranch; var commitBranch = ReportingBranch;
var commitMessage = $"{InstanceName} Generated - {DateTime.Now:dd.MM.yyyy HH:mm}"; // CompleteReporting Generated - 31.08.2022 10:25 var commitMessage = $"{InstanceName} Generated - {DateTime.Now:dd.MM.yyyy HH:mm}"; // CompleteReporting Generated - 31.08.2022 10:25
Console.WriteLine($" - Commit: \"{commitMessage}\""); Console.WriteLine($" - Commit: \"{commitMessage}\"");
...@@ -172,6 +180,11 @@ public abstract class Reporting<O> where O : class ...@@ -172,6 +180,11 @@ public abstract class Reporting<O> where O : class
return result; return result;
} }
public static string SanitizeOrganizationRor(string organizationRor)
{
return HttpUtility.UrlEncode(organizationRor.Replace("https://ror.org/", "").ToLower());
}
public static string GetReportingPathGeneral(string fileName) public static string GetReportingPathGeneral(string fileName)
{ {
return string.Format("General/{0}", fileName); return string.Format("General/{0}", fileName);
...@@ -179,7 +192,7 @@ public abstract class Reporting<O> where O : class ...@@ -179,7 +192,7 @@ public abstract class Reporting<O> where O : class
public static string GetReportingPathOrganization(string organizationRor, string fileName) public static string GetReportingPathOrganization(string organizationRor, string fileName)
{ {
return string.Format("Organizations/{0}/{1}", organizationRor, fileName); return string.Format("Organizations/{0}/{1}", SanitizeOrganizationRor(organizationRor), fileName);
} }
public static Stream ConvertStringContentsToStream(string contents) public static Stream ConvertStringContentsToStream(string contents)
...@@ -187,4 +200,31 @@ public abstract class Reporting<O> where O : class ...@@ -187,4 +200,31 @@ public abstract class Reporting<O> where O : class
byte[] byteArray = Encoding.UTF8.GetBytes(contents); byte[] byteArray = Encoding.UTF8.GetBytes(contents);
return new MemoryStream(byteArray); return new MemoryStream(byteArray);
} }
public void EnsureGitLabInformationIsSetAndCorrect()
{
if (!ReportingEnabled)
{
throw new ApplicationException($"\nReporting is deactivated on this machine! \nTo enable it, set the Consul Key \"coscine/local/reporting/enabled\" to \"true\".");
}
if (string.IsNullOrWhiteSpace(ReportingDatabaseProjectId) || string.IsNullOrWhiteSpace(ReportingBranch))
{
throw new ArgumentNullException($"\nNo valid Reporting Project ID or Branch were provided!");
}
var project = GitLabClient.Projects.GetAsync(ReportingDatabaseProjectId).Result;
Console.WriteLine($" - Report Generation to be uploaded to GitLab Project \"{project.Name}\" on branch \"{ReportingBranch}\"");
var branch = GitLabClient.Branches.GetAsync(project.Id, o => o.Search = ReportingBranch).Result;
if (!branch.Any(b => b.Name.Equals(ReportingBranch)) && Domain.Equals("DEVLEF") && !project.DefaultBranch.Equals(ReportingBranch))
{
Console.WriteLine($" - Branch \"{ReportingBranch}\" does not exist. Working on Domain {Domain}. Creating branch...");
GitLabClient.Branches.CreateAsync(ReportingDatabaseProjectId, new CreateBranchRequest(ReportingBranch, project.DefaultBranch)).Wait();
Console.WriteLine($" - Branch \"{ReportingBranch}\" successfully created");
}
else if (!branch.Any(b => b.Name.Equals(ReportingBranch)))
{
throw new ArgumentNullException($"\nBranch \"{ReportingBranch}\" does not exist!");
}
Console.WriteLine();
}
} }
\ No newline at end of file
...@@ -87,10 +87,10 @@ public class ProjectReporting : Reporting<ProjectReportingOptions> ...@@ -87,10 +87,10 @@ public class ProjectReporting : Reporting<ProjectReportingOptions>
var returnObjectsForOrganization = returnObjects.Where(ro => ro.Organizations.Select(o => o.RorUrl).Any(e => e.Equals(entry.RorUrl))); var returnObjectsForOrganization = returnObjects.Where(ro => ro.Organizations.Select(o => o.RorUrl).Any(e => e.Equals(entry.RorUrl)));
reportingFilesPerOrganization.Add(new ReportingFileObject reportingFilesPerOrganization.Add(new ReportingFileObject
{ {
Path = GetReportingPathOrganization(organization.RorUrl.Replace("https://ror.org/", "").ToLower(), ReportingFileName), Path = GetReportingPathOrganization(organization.RorUrl, ReportingFileName),
Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjectsForOrganization, Formatting.Indented)) Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjectsForOrganization, Formatting.Indented))
}); });
Console.WriteLine($" - {GetType().Name}: \"{GetReportingPathOrganization(organization.RorUrl.Replace("https://ror.org/", "").ToLower(), ReportingFileName)}\" generated successfully"); Console.WriteLine($" - {GetType().Name}: \"{GetReportingPathOrganization(organization.RorUrl, ReportingFileName)}\" generated successfully");
} }
return reportingFilesPerOrganization; return reportingFilesPerOrganization;
......
...@@ -87,10 +87,10 @@ public class ResourceReporting : Reporting<ResourceReportingOptions> ...@@ -87,10 +87,10 @@ public class ResourceReporting : Reporting<ResourceReportingOptions>
reportingFilesPerOrganization.Add(new ReportingFileObject reportingFilesPerOrganization.Add(new ReportingFileObject
{ {
Path = GetReportingPathOrganization(organization.RorUrl.Replace("https://ror.org/", "").ToLower(), ReportingFileName), Path = GetReportingPathOrganization(organization.RorUrl, ReportingFileName),
Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjectsForOrganization, Formatting.Indented)) Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjectsForOrganization, Formatting.Indented))
}); });
Console.WriteLine($" - {GetType().Name}: \"{GetReportingPathOrganization(organization.RorUrl.Replace("https://ror.org/", "").ToLower(), ReportingFileName)}\" generated successfully"); Console.WriteLine($" - {GetType().Name}: \"{GetReportingPathOrganization(organization.RorUrl, ReportingFileName)}\" generated successfully");
} }
return reportingFilesPerOrganization; return reportingFilesPerOrganization;
} }
......
...@@ -89,10 +89,10 @@ public class UserReporting : Reporting<UserReportingOptions> ...@@ -89,10 +89,10 @@ public class UserReporting : Reporting<UserReportingOptions>
reportingFilesPerOrganization.Add(new ReportingFileObject reportingFilesPerOrganization.Add(new ReportingFileObject
{ {
Path = GetReportingPathOrganization(organization.RorUrl.Replace("https://ror.org/", "").ToLower(), ReportingFileName), Path = GetReportingPathOrganization(organization.RorUrl, ReportingFileName),
Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjectsForOrganization, Formatting.Indented)) Content = ConvertStringContentsToStream(JsonConvert.SerializeObject(returnObjectsForOrganization, Formatting.Indented))
}); });
Console.WriteLine($" - {GetType().Name}: \"{GetReportingPathOrganization(organization.RorUrl.Replace("https://ror.org/", "").ToLower(), ReportingFileName)}\" generated successfully"); Console.WriteLine($" - {GetType().Name}: \"{GetReportingPathOrganization(organization.RorUrl, ReportingFileName)}\" generated successfully");
} }
return reportingFilesPerOrganization; return reportingFilesPerOrganization;
} }
......