Select Git revision

Benedikt Heinrichs authored and
Marcel Nellesen
committed
Code owners
Assign users and groups as approvers for specific file changes. Learn more.
Program.cs 7.44 KiB
using Coscine.Configuration;
using Coscine.Database.Models;
using Coscine.Metadata;
using System;
using System.Linq;
using System.Collections.Generic;
using VDS.RDF;
using VDS.RDF.Query;
using VDS.RDF.Query.Builder;
using VDS.RDF.Storage;
using VDS.RDF.Update;
using Coscine.Database.DataModel;
using Coscine.ActiveDirectory;
using LinqToDB.Data;
using Coscine.Database.Settings;
namespace Coscine.UserImporter
{
public class Program
{
private static Dictionary<string, string> RORMap = new Dictionary<string, string>()
{
{ "https://ror.org/04xfq0f34", "https://login.rz.rwth-aachen.de/shibboleth" },
};
public static void Main(string[] args)
{
var configuration = new ConsulConfiguration();
DataConnection.DefaultSettings = new CoscineSettings(configuration);
var virtuosoServer = configuration.GetString("coscine/local/virtuoso/additional/url");
using (var rdfStoreConnector = new RdfStoreConnector(virtuosoServer))
{
var externalIdModel = new ExternalIdModel();
var userModel = new UserModel();
var externalAuthenticatorModel = new ExternalAuthenticatorModel();
var shib = externalAuthenticatorModel.GetWhere((entry) => entry.DisplayName == "Shibboleth");
var updateEndpoint = new SparqlRemoteUpdateEndpoint(new Uri(string.Format(virtuosoServer)));
var queryEndpoint = new SparqlRemoteEndpoint(new Uri(string.Format(virtuosoServer)));
var readWriteSparqlConnector = new ReadWriteSparqlConnector(queryEndpoint, updateEndpoint);
var graphs = readWriteSparqlConnector.ListGraphs();
foreach (var graph in graphs)
{
var absoluteUri = graph.AbsoluteUri;
if (absoluteUri.StartsWith("https://ror.org/"))
{
SparqlResultSet pers = null;
var offset = 0;
var graphImpl = new Graph
{
BaseUri = graph
};
// Get all users from the current graph which are a person
// The graph is queried in a batch of 5000 entries since there seems to be a hard limit on 10000 entries per request
do
{
var query = QueryBuilder.Select(new SparqlVariable("s"), new SparqlVariable("p"), new SparqlVariable("o"))
.Graph(graph, graphEntry => graphEntry
.Where((where) => where.Subject("s").Predicate("p").Object("o"))
.Where((where) => where.Subject("s")
.PredicateUri(new Uri("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"))
.Object(new Uri("http://xmlns.com/foaf/0.1/Person"))))
.Limit(5000)
.Offset(offset)
.BuildQuery();
pers = (SparqlResultSet)readWriteSparqlConnector.Query(query.ToString());
foreach (var res in pers.Results)
{
graphImpl.Assert(res.Value("s"), res.Value("p"), res.Value("o"));
}
offset += 5000;
} while (pers.Count > 0);
var rdfType = graphImpl.CreateUriNode(new Uri("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"));
var person = graphImpl.CreateUriNode(new Uri("http://xmlns.com/foaf/0.1/Person"));
var openId = graphImpl.CreateUriNode(new Uri("http://xmlns.com/foaf/0.1/openId"));
var createdUsers = 0;
foreach (var entry in graphImpl.GetTriplesWithPredicateObject(rdfType, person))
{
foreach (var triple in graphImpl.GetTriplesWithSubjectPredicate(entry.Subject, openId))
{
if (externalIdModel.GetAllWhere((externalId) =>
externalId.ExternalIdColumn == triple.Object.ToString()
&& externalId.ExternalAuthenticatorId == shib.Id).Count() == 0)
{
try
{
var user = CreateUserObjectFromGraph(graphImpl, entry.Subject);
StoreUser(configuration, externalIdModel, userModel, shib, triple.Object.ToString(), user, absoluteUri);
createdUsers++;
if (createdUsers%100 == 0)
{
Console.WriteLine("Created " + createdUsers + " users.");
}
}
catch (Exception)
{
Console.WriteLine("Error on " + entry.Subject.ToString());
}
}
}
}
Console.WriteLine($"Finish: Created {createdUsers} users for {graph}");
}
}
}
}
private static void StoreUser(IConfiguration configuration, ExternalIdModel externalIdModel, UserModel userModel, ExternalAuthenticator shib, string externalId, User user, string absoluteUri)
{
userModel.Insert(user);
externalIdModel.Insert(new ExternalId
{
ExternalIdColumn = externalId,
ExternalAuthenticatorId = shib.Id,
Organization = RORMap.ContainsKey(absoluteUri) ? RORMap[absoluteUri] : "",
UserId = user.Id
});
ADHandler.AddUser(user, configuration);
}
// If additional attributes like e.g. E-Mail need to be mapped, this Dictionary can be extended
private static readonly Dictionary<Uri, Action<User, string>> _mapping = new Dictionary<Uri, Action<User, string>>()
{
{ new Uri("http://xmlns.com/foaf/0.1/familyName"), (user, surName) => user.Surname = surName },
{ new Uri("http://xmlns.com/foaf/0.1/givenName"), (user, givenName) => user.Givenname = givenName },
{ new Uri("http://xmlns.com/foaf/0.1/name"), (user, name) => user.DisplayName = name },
};
private static User CreateUserObjectFromGraph(Graph graphImpl, INode subject)
{
var user = new User();
foreach (var kv in _mapping)
{
var predicate = graphImpl.CreateUriNode(kv.Key);
var triples = graphImpl.GetTriplesWithSubjectPredicate(subject, predicate);
if (triples.Count() > 0)
{
kv.Value(user, triples.First().Object.ToString());
}
else
{
Console.WriteLine($"Error on {subject} with {predicate}");
}
}
return user;
}
}
}