Skip to content
Snippets Groups Projects
Select Git revision
  • Product/704-basicReporting
  • master default protected
  • gitkeep
  • dev protected
  • Hotfix/2562-organizations
  • Issue/2518-docs
  • Sprint/2022-01
  • Sprint/2021-19
  • Issues/0028-maxQuotaFix
  • Sprint/2021-08
  • Product/1414-fhPrivileges
  • Topic/1425-fhPrivileges
  • Sprint/2020-20
  • Topic/1051-basicReporting protected
  • Hotfix/986-fixOrganizationParserPipeline
  • Hotfix/953-fileNameWithSpace
  • Sprint/2020-13
  • Product/754-automatedImport
  • Topic/888-automatedImport protected
  • v1.5.0
  • v1.4.0
  • v1.3.2
  • v1.3.1
  • v1.3.0
  • v1.2.0
  • v1.1.2
  • v1.1.1
  • v1.1.0
28 results

Readme.md

Blame
  • Code owners
    Assign users and groups as approvers for specific file changes. Learn more.
    Reporting.cs 12.42 KiB
    using Coscine.Configuration;
    using Coscine.Metadata;
    using GitLabApiClient;
    using GitLabApiClient.Models.Branches.Requests;
    using GitLabApiClient.Models.Commits.Requests.CreateCommitRequest;
    using KPIGenerator.Utils;
    using System.Text;
    using System.Web;
    using VDS.RDF.Query;
    using static KPIGenerator.Utils.CommandLineOptions;
    
    namespace KPIGenerator;
    
    public abstract class Reporting<O> where O : class
    {
        public O Options { get; init; }
        public GitLabClient GitLabClient { get; set; }
        public List<Organization> Organizations { get; set; }
        public RdfStoreConnector RdfStoreConnector { get; init; }
        public SparqlRemoteEndpoint QueryEndpoint { get; init; }
        public ConsulConfiguration Configuration { get; }
        private static string HostUrl { get; } = "https://git.rwth-aachen.de/";
        private static string InstanceName { get; set; } = null!;
        public virtual string ReportingFileName { get; init; } = null!;
    
        private string Domain { get; init; }
        private bool ReportingEnabled { get; init; }
        private string ReportingDatabaseProjectId { get; init; }
        private string ReportingBranch { get; init; }
        public string RwthRor { get; init; }
    
    
        public readonly Organization _otherOrganization = new()
        {
            Name = "Other",
            RorUrl = "https://ror.org/_other",
        };
    
        public Reporting(O options)
        {
            InstanceName = this.GetType().Name;
            Options = options;
            Configuration = new ConsulConfiguration();
            RdfStoreConnector = new RdfStoreConnector(Configuration.GetStringAndWait("coscine/local/virtuoso/additional/url"));
            QueryEndpoint = new SparqlRemoteEndpoint(new Uri(Configuration.GetStringAndWait("coscine/local/virtuoso/additional/url")));
            GitLabClient = new GitLabClient(HostUrl, Configuration.GetStringAndWait("coscine/global/gitlabtoken"));
            Organizations = new List<Organization>() { _otherOrganization };
    
            ReportingEnabled = Configuration.GetStringAndWait("coscine/local/reporting/enabled", "false") == "true";
            Domain = Configuration.GetStringAndWait("coscine/local/profilesync/domain");
            ReportingDatabaseProjectId = Configuration.GetStringAndWait("coscine/local/reporting/gitlab_project_id");
            ReportingBranch = Configuration.GetStringAndWait("coscine/local/reporting/branch");
    
            RwthRor = Configuration.GetStringAndWait("coscine/global/organizations/rwth/ror_url");
        }
    
        public abstract IEnumerable<ReportingFileObject> GenerateReporting();
    
        public bool Run()
        {
            // Console text output
            Console.Write($"{new string('=', 80)}\n {Domain} | {InstanceName}");
            var baseOptions = Options as BaseOptions;
            if (baseOptions is not null && baseOptions.DummyMode)
            {
                Console.Write(" : DUMMY MODE");
            }
            Console.WriteLine($"\n{new string('-', 80)}");
            EnsureGitLabInformationIsSetAndCorrect();
            // Generate Reporting based on CLI input
            var reportingFiles = GenerateReporting();
            Console.WriteLine($"\n{new string('=', 80)}");
            Console.WriteLine(" - Reporting generated successfully. Publishing...");
            // Publish Report
            var success = PublishAsync(reportingFiles).Result;
            Console.WriteLine(success ? " - Published successfully." : " - Publishing FAILED!");
            return success;
        }
    
        private async Task<bool> PublishAsync(IEnumerable<ReportingFileObject> files)
        {
            try
            {
                // Retrieve Reporting Database project
                var reportingDatabaseProject = await GitLabClient.Projects.GetAsync(ReportingDatabaseProjectId);
                var commitBranch = ReportingBranch;
                var commitMessage = $"{InstanceName} Generated - {DateTime.Now:dd.MM.yyyy HH:mm}"; // CompleteReporting Generated - 31.08.2022 10:25
    
                Console.WriteLine($" - Commit: \"{commitMessage}\"");
    
                var projectTree = await GitLabClient.Trees.GetAsync(reportingDatabaseProject, o =>
                {
                    o.Recursive = true;
                    o.Reference = commitBranch;
                });
    
                // Define commit actions
                var actions = new List<CreateCommitRequestAction>();
    
                // Delete files or organizations that are not valid anymore
                foreach (var fileInProject in projectTree.Where(file => file.Type.Equals("blob")))
                {
                    if (!files.Any(f => f.Path.Equals(fileInProject.Path)) && !fileInProject.Path.Equals("README.md"))
                    {
                        // Add Action
                        actions.Add(new CreateCommitRequestAction(CreateCommitRequestActionType.Delete, fileInProject.Path));
                    }
                }
    
                // Create a commit per file with its contents
                foreach (var file in files)
                {
                    // Write file contents to bytes
                    byte[] bytes;
                    using (var ms = new MemoryStream())
                    {
                        file.Content.CopyTo(ms);
                        bytes = ms.ToArray();
                    }
    
                    // Distinguish between Creating or Updating a file
                    var actionType = CreateCommitRequestActionType.Create;
                    if (projectTree.Any(f => f.Path.Equals(file.Path)))
                    {
                        actionType = CreateCommitRequestActionType.Update;
                    }
    
                    // Add Action
                    actions.Add(new CreateCommitRequestAction(actionType, file.Path)
                    {
                        Content = Convert.ToBase64String(bytes),
                        Encoding = CreateCommitRequestActionEncoding.Base64
                    });
                }
                // Push Commit
                var baseOptions = Options as BaseOptions;
                if (baseOptions is not null && !baseOptions.DummyMode)
                {
                    await GitLabClient.Commits.CreateAsync(reportingDatabaseProject, new CreateCommitRequest(commitBranch, commitMessage, actions));
                }
                return true;
            }
            catch (Exception e)
            {
                Console.WriteLine(e.Message);
                return false;
            }
        }
    
        public Organization FetchOrganizationByRor(string rorUrl)
        {
            var result = new Organization();
            var organizationFound = Organizations.Find(o => o.RorUrl.Equals(rorUrl));
            if (organizationFound is not null)
            {
                result = new Organization
                {
                    Name = organizationFound.Name,
                    RorUrl = organizationFound.RorUrl,
                };
            }
            else
            {
                var organizationTriples = RdfStoreConnector.GetLabelForSubject(new Uri(Uri.UnescapeDataString(rorUrl))).ToList();
                if (organizationTriples.Any())
                {
                    result = new Organization
                    {
                        // Only one entry possible per organization, take 0th element
                        Name = organizationTriples[0].Object.ToString(),
                        RorUrl = organizationTriples[0].Subject.ToString(),
                    };
                    Organizations.Add(result); // Cache the fetched organization
                }
                else
                {
                    result = _otherOrganization;
                    Console.ForegroundColor = ConsoleColor.Yellow;
                    Console.WriteLine($"   WARNING!: Organization with ROR \"{rorUrl}\" could not be correctly identified. Will use \"{result.RorUrl}\" and \"{result.Name}\".");
                    Console.ResetColor();
                }
            }
            return result;
        }
    
        public IEnumerable<Organization> GetTopLevelOrganizationsFromEntries(IEnumerable<Organization> organizations)
        {
            var result = new List<Organization>();
            foreach (var org in organizations)
            {
                var ror = org.RorUrl;
                if (ror.Contains("www.rwth-aachen.de"))
                {
                    ror = ConvertOldRwthOrganizationToRor(ror); // e.g. <https://www.rwth-aachen.de/22000> turns into <https://ror.org/04xfq0f34#ORG-42NHW>
                }
                result.Add(new Organization
                {
                    RorUrl = ror.Contains('#') ? ror[..ror.IndexOf('#')] : ror, // e.g. <https://ror.org/04xfq0f34#ORG-42NHW> turns into <https://ror.org/04xfq0f34>
                    Name = org.Name
                });
            }
            return result.DistinctBy(r => r.RorUrl);
        }
    
        public static string SanitizeOrganizationRor(string organizationRor)
        {
            return HttpUtility.UrlEncode(organizationRor.Replace("https://ror.org/", "").ToLower());
        }
    
        public static string GetReportingPathGeneral(string fileName)
        {
            return string.Format("General/{0}", fileName);
        }
    
        public static string GetReportingPathOrganization(string organizationRor, string fileName)
        {
            return string.Format("Organizations/{0}/{1}", SanitizeOrganizationRor(organizationRor), fileName);
        }
    
        public static Stream ConvertStringContentsToStream(string contents)
        {
            byte[] byteArray = Encoding.UTF8.GetBytes(contents);
            return new MemoryStream(byteArray);
        }
    
        public void EnsureGitLabInformationIsSetAndCorrect()
        {
            if (!ReportingEnabled)
            {
                throw new ApplicationException($"\nReporting is deactivated on this machine! \nTo enable it, set the Consul Key \"coscine/local/reporting/enabled\" to  \"true\".");
            }
            if (string.IsNullOrWhiteSpace(ReportingDatabaseProjectId) || string.IsNullOrWhiteSpace(ReportingBranch))
            {
                throw new ArgumentNullException($"\nNo valid Reporting Project ID or Branch were provided!");
            }
            var project = GitLabClient.Projects.GetAsync(ReportingDatabaseProjectId).Result;
            Console.WriteLine($" - Report Generation to be uploaded to GitLab Project \"{project.Name}\" on branch \"{ReportingBranch}\"");
            var branch = GitLabClient.Branches.GetAsync(project.Id, o => o.Search = ReportingBranch).Result;
    
            if (!branch.Any(b => b.Name.Equals(ReportingBranch)) && Domain.Equals("DEVLEF") && !project.DefaultBranch.Equals(ReportingBranch))
            {
                Console.WriteLine($" - Branch \"{ReportingBranch}\" does not exist. Working on Domain {Domain}. Creating branch...");
                GitLabClient.Branches.CreateAsync(ReportingDatabaseProjectId, new CreateBranchRequest(ReportingBranch, project.DefaultBranch)).Wait();
                Console.WriteLine($" - Branch \"{ReportingBranch}\" successfully created");
            }
            else if (!branch.Any(b => b.Name.Equals(ReportingBranch)))
            {
                throw new ArgumentNullException($"\nBranch \"{ReportingBranch}\" does not exist!");
            }
            Console.WriteLine();
        }
    
        public string ConvertOldRwthOrganizationToRor(string organization)
        {
            // Converts values like https://www.rwth-aachen.de/22000 to its corresponding RoR (here: https://ror.org/04xfq0f34#ORG-42NHW)
            var _queryString = new SparqlParameterizedString
            {
                CommandText = $@"
                    PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
                    PREFIX org: <http://www.w3.org/ns/org#>
    
                    SELECT DISTINCT ?source ?inst 
                    WHERE {{
    	                # Define ?source URI
    	                VALUES ?source
    	                {{
    		                <{organization}>
    	                }} .
    
    	                # Get Display Name of the institution
    	                ?source rdfs:label ?name .
    
    	                # Get Institute Identifier in ?ikzid
    	                ?source org:identifier ?ikzid .
    
    	                # IKZ ID is in the form 'ikz:<ID>' or 'ikz:0<ID>'
    	                BIND(concat('ikz:', ?ikzid) AS ?ikz) .
    	                BIND(concat('ikz:0', ?ikzid) AS ?ikz0) .
    
    	                # Fetch all institutes from RWTH
    	                <{RwthRor}> org:hasUnit ?inst .
    
    	                # OR statement to search by ikz variations and name
    	                {{ ?inst org:identifier ?ikz .}} UNION {{ ?inst org:identifier ?ikz0 .}} UNION {{ ?inst rdfs:label ?name .}}
                    }}
                    GROUP BY ?inst
                "
            };
            using var results = RdfStoreConnector.QueryEndpoint.QueryWithResultSet(_queryString.ToString());
            if (!results.IsEmpty)
            {
                var inst = results.Select(x => x.Value("inst").ToString()).ToList()[0];   // Get the value for ?inst
                if (!string.IsNullOrWhiteSpace(inst))
                {
                    Console.WriteLine($"   Organization {organization} found to match {inst}");
                    return inst;
                }
            }
            Console.WriteLine($"   Could not find match for {organization}");
            return organization;
        }
    }