Skip to content
Snippets Groups Projects
Select Git revision
  • d3ce8a0532bf3a9320e159134709a71d650d4a44
  • master default protected
  • gitkeep
  • dev protected
  • Issue/2518-docs
  • Issue/2324-workingUserImport
  • Sprint/2022-01
  • Sprint/2021-01
  • Product/1149-dfnaai
  • Topic/1278-dfnAAI
  • Sprint/2020-20
  • Product/1054-userImport
  • Topic/1086-userImport
  • v1.2.1
  • v1.2.0
  • v1.1.0
16 results

Program.cs

Blame
  • Benedikt Heinrichs's avatar
    Benedikt Heinrichs authored and Marcel Nellesen committed
    9a453d3c
    History
    Code owners
    Assign users and groups as approvers for specific file changes. Learn more.
    Program.cs 7.44 KiB
    using Coscine.Configuration;
    using Coscine.Database.Models;
    using Coscine.Metadata;
    using System;
    using System.Linq;
    using System.Collections.Generic;
    using VDS.RDF;
    using VDS.RDF.Query;
    using VDS.RDF.Query.Builder;
    using VDS.RDF.Storage;
    using VDS.RDF.Update;
    using Coscine.Database.DataModel;
    using Coscine.ActiveDirectory;
    using LinqToDB.Data;
    using Coscine.Database.Settings;
    
    namespace Coscine.UserImporter
    {
        public class Program
        {
    
            private static Dictionary<string, string> RORMap = new Dictionary<string, string>()
            {
                { "https://ror.org/04xfq0f34", "https://login.rz.rwth-aachen.de/shibboleth" },
            };
    
            public static void Main(string[] args)
            {
                var configuration = new ConsulConfiguration();
                DataConnection.DefaultSettings = new CoscineSettings(configuration);
    
                var virtuosoServer = configuration.GetString("coscine/local/virtuoso/additional/url");
                using (var rdfStoreConnector = new RdfStoreConnector(virtuosoServer))
                {
                    var externalIdModel = new ExternalIdModel();
                    var userModel = new UserModel();
    
                    var externalAuthenticatorModel = new ExternalAuthenticatorModel();
                    var shib = externalAuthenticatorModel.GetWhere((entry) => entry.DisplayName == "Shibboleth");
    
                    var updateEndpoint = new SparqlRemoteUpdateEndpoint(new Uri(string.Format(virtuosoServer)));
                    var queryEndpoint = new SparqlRemoteEndpoint(new Uri(string.Format(virtuosoServer)));
                    var readWriteSparqlConnector = new ReadWriteSparqlConnector(queryEndpoint, updateEndpoint);
    
                    var graphs = readWriteSparqlConnector.ListGraphs();
    
                    foreach (var graph in graphs)
                    {
                        var absoluteUri = graph.AbsoluteUri;
                        if (absoluteUri.StartsWith("https://ror.org/"))
                        {
                            SparqlResultSet pers = null;
                            var offset = 0;
                            var graphImpl = new Graph
                            {
                                BaseUri = graph
                            };
    
                            // Get all users from the current graph which are a person
                            // The graph is queried in a batch of 5000 entries since there seems to be a hard limit on 10000 entries per request
                            do
                            {
                                var query = QueryBuilder.Select(new SparqlVariable("s"), new SparqlVariable("p"), new SparqlVariable("o"))
                                    .Graph(graph, graphEntry => graphEntry
                                        .Where((where) => where.Subject("s").Predicate("p").Object("o"))
                                        .Where((where) => where.Subject("s")
                                            .PredicateUri(new Uri("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"))
                                            .Object(new Uri("http://xmlns.com/foaf/0.1/Person"))))
                                    .Limit(5000)
                                    .Offset(offset)
                                    .BuildQuery();
    
    
                                pers = (SparqlResultSet)readWriteSparqlConnector.Query(query.ToString());
                                foreach (var res in pers.Results)
                                {
                                    graphImpl.Assert(res.Value("s"), res.Value("p"), res.Value("o"));
                                }
    
                                offset += 5000;
                            } while (pers.Count > 0);
    
                            var rdfType = graphImpl.CreateUriNode(new Uri("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"));
                            var person = graphImpl.CreateUriNode(new Uri("http://xmlns.com/foaf/0.1/Person"));
                            var openId = graphImpl.CreateUriNode(new Uri("http://xmlns.com/foaf/0.1/openId"));
    
                            var createdUsers = 0;
    
                            foreach (var entry in graphImpl.GetTriplesWithPredicateObject(rdfType, person))
                            {
                                foreach (var triple in graphImpl.GetTriplesWithSubjectPredicate(entry.Subject, openId))
                                {
                                    if (externalIdModel.GetAllWhere((externalId) => 
                                            externalId.ExternalIdColumn == triple.Object.ToString()
                                            && externalId.ExternalAuthenticatorId == shib.Id).Count() == 0)
                                    {
                                        try
                                        {
                                            var user = CreateUserObjectFromGraph(graphImpl, entry.Subject);
                                            StoreUser(configuration, externalIdModel, userModel, shib, triple.Object.ToString(), user, absoluteUri);
                                            createdUsers++;
                                            if (createdUsers%100 == 0)
                                            {
                                                Console.WriteLine("Created " + createdUsers + " users.");
                                            }
                                        }
                                        catch (Exception)
                                        {
                                            Console.WriteLine("Error on " + entry.Subject.ToString());
                                        }
                                    }
                                }
                            }
    
                            Console.WriteLine($"Finish: Created {createdUsers} users for {graph}");
                        }
                    }
                }
            }
    
            private static void StoreUser(IConfiguration configuration, ExternalIdModel externalIdModel, UserModel userModel, ExternalAuthenticator shib, string externalId, User user, string absoluteUri)
            {
                userModel.Insert(user);
                externalIdModel.Insert(new ExternalId
                {
                    ExternalIdColumn = externalId,
                    ExternalAuthenticatorId = shib.Id,
                    Organization = RORMap.ContainsKey(absoluteUri) ? RORMap[absoluteUri] : "",
                    UserId = user.Id
                });
                ADHandler.AddUser(user, configuration);
            }
    
            // If additional attributes like e.g. E-Mail need to be mapped, this Dictionary can be extended
            private static readonly Dictionary<Uri, Action<User, string>> _mapping = new Dictionary<Uri, Action<User, string>>()
            {
                { new Uri("http://xmlns.com/foaf/0.1/familyName"), (user, surName) => user.Surname = surName },
                { new Uri("http://xmlns.com/foaf/0.1/givenName"), (user, givenName) => user.Givenname = givenName },
                { new Uri("http://xmlns.com/foaf/0.1/name"), (user, name) => user.DisplayName = name },
            };
    
            private static User CreateUserObjectFromGraph(Graph graphImpl, INode subject)
            {
                var user = new User();
                foreach (var kv in _mapping)
                {
                    var predicate = graphImpl.CreateUriNode(kv.Key);
                    var triples = graphImpl.GetTriplesWithSubjectPredicate(subject, predicate);
                    if (triples.Count() > 0)
                    {
                        kv.Value(user, triples.First().Object.ToString());
                    }
                    else
                    {
                        Console.WriteLine($"Error on {subject} with {predicate}");
                    }
                }
                return user;
            }
        }
    }