Skip to content
Snippets Groups Projects
Commit 30f3afc7 authored by Petar Hristov's avatar Petar Hristov :speech_balloon:
Browse files

Merge branch 'dev' into 'main'

Dev

See merge request !28
parents 93076696 745af463
Branches
Tags
1 merge request!28Dev
Pipeline #1216678 passed
Showing
with 997 additions and 622 deletions
using Coscine.Database.DataModel; using Coscine.ApiClient;
using Coscine.Database.Models; using Coscine.ApiClient.Core.Model;
using Coscine.Database.Util; using SQL2Linked.Utils;
using VDS.RDF; using VDS.RDF;
namespace SQL2Linked.Implementations namespace SQL2Linked.Implementations;
{
public class ProjectStructuralData : StructuralData<Project, ProjectModel> /// <summary>
{ /// Class responsible for converting project data into linked data graphs.
public readonly Uri org = new("http://www.w3.org/ns/org#"); /// It retrieves project information from the API and then transforms this data into a series of RDF graphs,
public readonly Uri dcat = new("http://www.w3.org/ns/dcat#"); /// making use of predefined URIs and RDF constructs.
public readonly Uri dcterms = new("http://purl.org/dc/terms/"); /// </summary>
public readonly Uri foaf = new("http://xmlns.com/foaf/0.1/"); public class ProjectStructuralData : StructuralData<ProjectAdminDto>
public readonly Uri vcard = new("http://www.w3.org/2006/vcard/ns#"); {
public readonly Uri rdf = new("http://www.w3.org/1999/02/22-rdf-syntax-ns#"); /// <summary>
public readonly Uri cosc = new("https://purl.org/coscine/"); /// Asynchronously retrieves all project data, including deleted projects.
/// </summary>
private readonly VisibilityModel VisibilityModel = new(); /// <returns>A <see cref="Task"/> that represents the asynchronous operation and returns a collection of <see cref="ProjectAdminDto"/>.</returns>
private readonly ProjectRoleModel ProjectRoleModel = new(); /// <remarks>This override allows for the inclusion of deleted projects.</remarks>
private readonly ProjectResourceModel ProjectResourceModel = new(); public override async Task<IEnumerable<ProjectAdminDto>> GetAll()
private readonly ProjectInstituteModel ProjectInstituteModel = new(); {
return await RequestUtil.WrapPagedRequest<ProjectAdminDtoPagedResponse, ProjectAdminDto>(
// Override to also receive deleted projects (currentPage) => _adminApi.GetAllProjectsAsync(includeDeleted: true, pageNumber: currentPage, pageSize: 250)
public override IEnumerable<Project> GetAll() );
{
return DatabaseConnection.ConnectToDatabase((db) =>
{
return
(from tableEntry in Model.GetITableFromDatabase(db)
select tableEntry).ToList();
});
} }
public override IEnumerable<IGraph> ConvertToLinkedData(IEnumerable<Project> entries) /// <summary>
/// Converts a collection of project data entries into a set of RDF graphs.
/// Each project is transformed into a graph, with RDF triples representing various properties of the project.
/// </summary>
/// <param name="entries">A collection of <see cref="ProjectAdminDto"/> instances representing project data.</param>
/// <returns>A collection of <see cref="IGraph"/> instances, each representing an RDF graph of a project.</returns>
public override async Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<ProjectAdminDto> entries)
{ {
IEnumerable<Visibility> visibilities = VisibilityModel.GetAll();
IEnumerable<ProjectRole> projectRoles = ProjectRoleModel.GetAll();
IEnumerable<ProjectResource> projectResources = ProjectResourceModel.GetAll();
IEnumerable<ProjectInstitute> projectInstitutes = ProjectInstituteModel.GetAll();
var graphs = new List<IGraph>(); var graphs = new List<IGraph>();
var projectUrlHandlePrefix = "https://hdl.handle.net/" + Prefix; var coscineHandlePrefix = UriHelper.TryCombineUri(RdfUris.HandlePrefix, _pidConfiguration.Prefix)
var projectUrlPrefix = "https://purl.org/coscine/projects"; ?? throw new Exception("Could not combine handle prefix with PID prefix");
var resourceUrlPrefix = "https://purl.org/coscine/resources";
var coscineGraph = new Graph var coscineGraph = new Graph
{ {
BaseUri = cosc BaseUri = RdfUris.CoscinePrefix
}; };
AssertToGraphUriNode(coscineGraph, cosc.AbsoluteUri, dcat + "catalog", projectUrlPrefix + "/"); AssertToGraphUriNode(coscineGraph, RdfUris.CoscinePrefix, RdfUris.DcatCatalog, RdfUris.CoscineProjects);
AssertToGraphUriNode(coscineGraph, cosc.AbsoluteUri, dcat + "catalog", resourceUrlPrefix + "/"); AssertToGraphUriNode(coscineGraph, RdfUris.CoscinePrefix, RdfUris.DcatCatalog, RdfUris.CoscineResources);
graphs.Add(coscineGraph); graphs.Add(coscineGraph);
foreach (var entry in entries) foreach (var entry in entries)
{ {
var projectGraphName = $"{projectUrlPrefix}/{entry.Id}"; var projectGraphName = UriHelper.TryCombineUri(RdfUris.CoscineProjects, entry.Id)
var projectHandleName = $"{projectUrlHandlePrefix}/{entry.Id}"; ?? throw new Exception("Could not combine projects prefix with project ID");
var projectHandleName = UriHelper.TryCombineUri(coscineHandlePrefix, entry.Id);
var graph = new Graph var graph = new Graph
{ {
BaseUri = new Uri(projectGraphName) BaseUri = projectGraphName
}; };
AssertToGraphUriNode(graph, projectGraphName, rdf + "type", dcat + "Catalog"); AssertToGraphUriNode(graph, projectGraphName, RdfUris.A, RdfUris.DcatCatalogClass);
Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {rdf}type {dcat}Catalog'. "); Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {RdfUris.A} {RdfUris.DcatCatalogClass}'. ");
AssertToGraphUriNode(graph, projectGraphName, rdf + "type", org + "OrganizationalCollaboration"); AssertToGraphUriNode(graph, projectGraphName, RdfUris.A, RdfUris.OrgOrganizationalCollaborationClass);
Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {rdf}type {org}OrganizationalCollaboration'. "); Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {RdfUris.A} {RdfUris.OrgOrganizationalCollaborationClass}'. ");
AssertToGraphUriNode(graph, projectGraphName, rdf + "type", vcard + "Group"); AssertToGraphUriNode(graph, projectGraphName, RdfUris.A, RdfUris.VcardGroupClass);
Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {rdf}type {vcard}Group'. "); Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {RdfUris.A} {RdfUris.VcardGroupClass}'. ");
AssertToGraphLiteralNode(graph, projectGraphName, dcterms + "title", entry.ProjectName); AssertToGraphLiteralNode(graph, projectGraphName, RdfUris.DcTermsTitle, entry.Name);
Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {dcterms}title {entry.ProjectName}'. "); Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {RdfUris.DcTermsTitle} {entry.Name}'. ");
AssertToGraphLiteralNode(graph, projectGraphName, dcterms + "description", entry.Description); AssertToGraphLiteralNode(graph, projectGraphName, RdfUris.DcTermsDescription, entry.Description);
Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {dcterms}description {entry.Description}'. "); Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {RdfUris.DcTermsDescription} {entry.Description}'. ");
AssertToGraphLiteralNode(graph, projectGraphName, dcat + "startDate", entry.StartDate.ToString(), new Uri("http://www.w3.org/2001/XMLSchema#dateTime")); AssertToGraphLiteralNode(graph, projectGraphName, RdfUris.DcTermsStartDate, entry.StartDate.ToString(), new Uri("http://www.w3.org/2001/XMLSchema#dateTime"));
Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {dcat}startDate {entry.StartDate}'. "); Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {RdfUris.DcTermsStartDate} {entry.StartDate}'. ");
AssertToGraphLiteralNode(graph, projectGraphName, dcat + "endDate", entry.EndDate.ToString(), new Uri("http://www.w3.org/2001/XMLSchema#dateTime")); AssertToGraphLiteralNode(graph, projectGraphName, RdfUris.DcTermsEndDate, entry.EndDate.ToString(), new Uri("http://www.w3.org/2001/XMLSchema#dateTime"));
Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {dcat}endDate {entry.EndDate}'. "); Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {RdfUris.DcTermsEndDate} {entry.EndDate}'. ");
if (!string.IsNullOrWhiteSpace(entry.Keywords)) if (entry.Keywords.Count > 0)
{ {
var listKeywords = entry.Keywords.Split(';').ToList(); foreach (var keyword in entry.Keywords)
foreach (var keyword in listKeywords)
{ {
AssertToGraphLiteralNode(graph, projectGraphName, dcterms + "subject", keyword); AssertToGraphLiteralNode(graph, projectGraphName, RdfUris.DcTermsSubject, keyword);
Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {dcterms}subject {keyword}'. "); Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {RdfUris.DcTermsSubject} {keyword}'. ");
} }
} }
AssertToGraphLiteralNode(graph, projectGraphName, dcterms + "alternative", entry.DisplayName); AssertToGraphLiteralNode(graph, projectGraphName, RdfUris.DcTermsAlternative, entry.DisplayName);
Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {dcterms}alternative {entry.DisplayName}'. "); Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {RdfUris.DcTermsAlternative} {entry.DisplayName}'. ");
AssertToGraphLiteralNode(graph, projectGraphName, dcterms + "rightsHolder", entry.PrincipleInvestigators); AssertToGraphLiteralNode(graph, projectGraphName, RdfUris.DcTermsRightsHolder, entry.PrincipleInvestigators);
Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {dcterms}rightsHolder {entry.PrincipleInvestigators}'. "); Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {RdfUris.DcTermsRightsHolder} {entry.PrincipleInvestigators}'. ");
AssertToGraphLiteralNode(graph, projectGraphName, "https://schema.org/funding", entry.GrantId); AssertToGraphLiteralNode(graph, projectGraphName, RdfUris.SchemaFunding, entry.GrantId);
Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} https://schema.org/funding {entry.GrantId}'. "); Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {RdfUris.SchemaFunding} {entry.GrantId}'. ");
foreach (var visibility in visibilities) if (entry.Visibility.DisplayName.Contains("Public"))
{
if (entry.VisibilityId == visibility.Id && visibility.DisplayName.Contains("Public"))
{ {
AssertToGraphUriNode(graph, projectGraphName, cosc + "terms/project#visibility", cosc + $"terms/visibility#public"); AssertToGraphUriNode(graph, projectGraphName, RdfUris.CoscineTermsProjectVisibility, RdfUris.CoscineTermsVisibilityPublic);
Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {cosc}terms/project#visibility {cosc}terms/visibility#public'. "); Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {RdfUris.CoscineTermsProjectVisibility} {RdfUris.CoscineTermsVisibilityPublic}'. ");
break;
} }
else if (entry.VisibilityId == visibility.Id && visibility.DisplayName.Contains("Project Members")) else if (entry.Visibility.DisplayName.Contains("Project Members"))
{ {
AssertToGraphUriNode(graph, projectGraphName, cosc + "terms/project#visibility", cosc + $"terms/visibility#projectMember"); AssertToGraphUriNode(graph, projectGraphName, RdfUris.CoscineTermsProjectVisibility, RdfUris.CoscineTermsVisibilityProjectMember);
Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {cosc}terms/project#visibility {cosc}terms/visibility#projectMember'. "); Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {RdfUris.CoscineTermsProjectVisibility} {RdfUris.CoscineTermsVisibilityProjectMember}'. ");
break;
}
} }
AssertToGraphLiteralNode(graph, projectGraphName, cosc + "terms/project#deleted", entry.Deleted.ToString().ToLower(), new Uri("http://www.w3.org/2001/XMLSchema#boolean")); AssertToGraphLiteralNode(graph, projectGraphName, RdfUris.CoscineTermsProjectDeleted, entry.Deleted.ToString().ToLower(), RdfUris.XsdBoolean);
Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {cosc}terms/project#deleted {entry.Deleted}'. "); Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {RdfUris.CoscineTermsProjectDeleted} {entry.Deleted}'. ");
AssertToGraphLiteralNode(graph, projectGraphName, cosc + "terms/project#slug", entry.Slug); AssertToGraphLiteralNode(graph, projectGraphName, RdfUris.CoscineTermsProjectSlug, entry.Slug);
Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {cosc}terms/project#slug {entry.Slug}'. "); Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {RdfUris.CoscineTermsProjectSlug} {entry.Slug}'. ");
AssertToGraphUriNode(graph, projectGraphName, foaf + "homepage", projectHandleName); AssertToGraphUriNode(graph, projectGraphName, RdfUris.FoafHomepage, projectHandleName);
Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {foaf}homepage {projectHandleName}'. "); Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {RdfUris.FoafHomepage} {projectHandleName}'. ");
foreach (var projectRole in projectRoles.Where(p => p.ProjectId.Equals(entry.Id))) foreach (var projectRole in entry.ProjectRoles)
{ {
AssertToGraphUriNode(graph, projectGraphName, vcard + "hasMember", cosc + "users/" + projectRole.UserId); AssertToGraphUriNode(graph, projectGraphName, RdfUris.VcardHasMember, UriHelper.TryCombineUri(RdfUris.CoscineUsers, projectRole.UserId));
Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {vcard}hasMember {cosc}users/{projectRole.UserId}'. "); Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {RdfUris.VcardHasMember} {UriHelper.TryCombineUri(RdfUris.CoscineUsers, projectRole.UserId)}'. ");
var blankNode = graph.CreateBlankNode(); var blankNode = graph.CreateBlankNode();
AssertToGraphBlankAndUriNode(graph, blankNode, rdf + "type", org + "Membership"); AssertToGraphBlankAndUriNode(graph, blankNode, RdfUris.A, RdfUris.OrgMembershipClass);
Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{blankNode} {rdf}type {org}Membership'. "); Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{blankNode} {RdfUris.A} {RdfUris.OrgMembershipClass}'. ");
AssertToGraphBlankAndUriNode(graph, blankNode, org + "organization", projectGraphName); AssertToGraphBlankAndUriNode(graph, blankNode, RdfUris.OrgOrganization, projectGraphName);
Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{blankNode} {org}organization {projectGraphName}'. "); Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{blankNode} {RdfUris.OrgOrganization} {projectGraphName}'. ");
AssertToGraphBlankAndUriNode(graph, blankNode, org + "role", cosc + "roles/" + projectRole.RoleId); AssertToGraphBlankAndUriNode(graph, blankNode, RdfUris.OrgRole, UriHelper.TryCombineUri(RdfUris.CoscineRoles, projectRole.RoleId));
Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{blankNode} {org}role {cosc}roles/{projectRole.RoleId}'. "); Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{blankNode} {RdfUris.OrgRole} {UriHelper.TryCombineUri(RdfUris.CoscineRoles, projectRole.RoleId)}'. ");
AssertToGraphBlankAndUriNode(graph, blankNode, org + "member", cosc + "users/" + projectRole.UserId); AssertToGraphBlankAndUriNode(graph, blankNode, RdfUris.OrgMember, UriHelper.TryCombineUri(RdfUris.CoscineUsers, projectRole.UserId));
Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{blankNode} {org}member {cosc}users/{projectRole.UserId}'. "); Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{blankNode} {RdfUris.OrgMember} {UriHelper.TryCombineUri(RdfUris.CoscineUsers, projectRole.UserId)}'. ");
} }
foreach (var projectResource in projectResources.Where(p => p.ProjectId.Equals(entry.Id))) foreach (var projectResource in entry.ProjectResources)
{ {
AssertToGraphUriNode(graph, projectGraphName, dcat + "catalog", $"{resourceUrlPrefix}/{projectResource.ResourceId}"); AssertToGraphUriNode(graph, projectGraphName, RdfUris.DcatCatalog, UriHelper.TryCombineUri(RdfUris.CoscineResources, projectResource.ResourceId));
Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {dcat}catalog {resourceUrlPrefix}/{projectResource.ResourceId}'. "); Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {RdfUris.DcatCatalog} {UriHelper.TryCombineUri(RdfUris.CoscineResources, projectResource.ResourceId)}'. ");
} }
foreach (var projectInstitute in projectInstitutes.Where(p => p.ProjectId.Equals(entry.Id))) foreach (var projectInstitute in entry.Organizations)
{ {
AssertToGraphUriNode(graph, projectGraphName, org + "organization", projectInstitute.OrganizationUrl); AssertToGraphUriNode(graph, projectGraphName, RdfUris.OrgOrganization, new Uri(projectInstitute.Uri));
Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {org}organization {projectInstitute.OrganizationUrl}'. "); Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {RdfUris.OrgOrganization} {projectInstitute.Uri}'. ");
} }
if (entry.Creator is not null) if (entry.Creator is not null)
{ {
AssertToGraphUriNode(graph, projectGraphName, dcterms + "creator", cosc + $"users/{entry.Creator}"); AssertToGraphUriNode(graph, projectGraphName, RdfUris.DcTermsCreator, UriHelper.TryCombineUri(RdfUris.CoscineUsers, entry.Creator.Id));
Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {dcterms}creator {cosc}users/{entry.Creator}'. "); Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {RdfUris.DcTermsCreator} {UriHelper.TryCombineUri(RdfUris.CoscineUsers, entry.Creator.Id)}'. ");
} }
if (entry.DateCreated is not null && entry.DateCreated.HasValue) if (entry.CreationDate is not null && entry.CreationDate.HasValue)
{ {
AssertToGraphLiteralNode(graph, projectGraphName, dcterms + "created", entry.DateCreated.Value.ToString(), new Uri("http://www.w3.org/2001/XMLSchema#dateTime")); AssertToGraphLiteralNode(graph, projectGraphName, RdfUris.DcTermsCreated, entry.CreationDate.Value.ToString(), RdfUris.XsdDateTime);
Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {dcterms}created {entry.DateCreated}'. "); Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {RdfUris.DcTermsCreated} {entry.CreationDate}'. ");
} }
graphs.Add(graph); graphs.Add(graph);
} }
return graphs; return await Task.FromResult(graphs);
}
} }
} }
\ No newline at end of file
This diff is collapsed.
using Coscine.Database.DataModel; using Coscine.ApiClient.Core.Api;
using Coscine.Database.Models; using Coscine.ApiClient.Core.Model;
using SQL2Linked.Utils;
using VDS.RDF; using VDS.RDF;
using VDS.RDF.Parsing;
namespace SQL2Linked.Implementations namespace SQL2Linked.Implementations;
{
public class ResourceTypeStructuralData : StructuralData<ResourceType, ResourceTypeModel> /// <summary>
/// Class responsible for converting resource type data into linked data graphs.
/// It retrieves resource type information from the API and then transforms this data into a series of RDF graphs,
/// making use of predefined URIs and RDF constructs.
/// </summary>
public class ResourceTypeStructuralData : StructuralData<ResourceTypeInformationDto>
{ {
public readonly string ResourceTypeUrlPrefix = "https://purl.org/coscine/resourcetypes"; /// <summary>
public readonly Uri rdf = new("http://www.w3.org/1999/02/22-rdf-syntax-ns#"); /// Asynchronously retrieves all resource type data.
public readonly Uri dcat = new("http://www.w3.org/ns/dcat#"); /// </summary>
public readonly Uri dcterms = new("http://purl.org/dc/terms/"); /// <returns>A <see cref="Task"/> that represents the asynchronous operation and returns a collection of <see cref="ResourceTypeInformationDto"/>.</returns>
public override IEnumerable<IGraph> ConvertToLinkedData(IEnumerable<ResourceType> entries) public override async Task<IEnumerable<ResourceTypeInformationDto>> GetAll()
{ {
var resourceTypeApi = new ResourceTypeApi(_apiConfiguration);
var resourceTypeInformationsResponse = await resourceTypeApi.GetAllResourceTypesInformationAsync();
return resourceTypeInformationsResponse.Data;
}
/// <summary>
/// Converts a collection of resource type data entries into a set of RDF graphs.
/// Each resource type is transformed into a graph, with RDF triples representing various properties of the resource type.
/// </summary>
/// <param name="entries">A collection of <see cref="ResourceTypeInformationDto"/> instances representing resource type data.</param>
/// <returns>A collection of <see cref="IGraph"/> instances, each representing an RDF graph of a resource type.</returns>
public override async Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<ResourceTypeInformationDto> entries)
{
var graphs = new List<IGraph>(); var graphs = new List<IGraph>();
foreach (var entry in entries) foreach (var entry in entries)
{ {
var resourceTypeGraphName = $"{ResourceTypeUrlPrefix}/{entry.Id}"; var resourceTypeGraphName = UriHelper.TryCombineUri(RdfUris.CoscineResourceTypes, entry.Id)
var graph = RdfStoreConnector.GetGraph(resourceTypeGraphName); ?? throw new Exception("Could not combine resource types prefix with resource type ID");
var response = _adminApi.GetMetadataGraph(resourceTypeGraphName.AbsoluteUri, RdfFormat.TextTurtle);
var graph = new Graph()
{
BaseUri = resourceTypeGraphName
};
graph.LoadFromString(response.Data.Content, new TurtleParser());
// check if a triple with a dcat:DataService already exists in the resourcetype graph // check if a triple with a dcat:DataService already exists in the resourcetype graph
var getTriplesDcatDataService = graph.GetTriplesWithObject(new Uri(dcat + "DataService")); var getTriplesDcatDataService = graph.GetTriplesWithObject(RdfUris.DcatDataServiceClass);
if (!getTriplesDcatDataService.Any()) if (!getTriplesDcatDataService.Any())
{ {
AssertToGraphUriNode(graph, resourceTypeGraphName, rdf + "type", dcat + "DataService"); AssertToGraphUriNode(graph, resourceTypeGraphName, RdfUris.A, RdfUris.DcatDataServiceClass);
Console.WriteLine($"For resource type '{entry.DisplayName}' will migrate triple '{graph.BaseUri} {rdf}type {dcat}DataService'. "); Console.WriteLine($"For resource type '{entry.SpecificType}' will migrate triple '{graph.BaseUri} {RdfUris.A} {RdfUris.DcatDataServiceClass}'. ");
} }
else else
{ {
Console.WriteLine($"For resource type '{entry.DisplayName}' will NOT migrate triple '{graph.BaseUri} {rdf}type {dcat}DataService'. "); Console.WriteLine($"For resource type '{entry.SpecificType}' will NOT migrate triple '{graph.BaseUri} {RdfUris.A} {RdfUris.DcatDataServiceClass}'. ");
} }
// check if a triple with dcterms:title '{entry.DisplayName}' already exists in the role graph // check if a triple with dcterms:title '{entry.DisplayName}' already exists in the role graph
var getTriplesDctermsTitle = graph.GetTriplesWithPredicate(new Uri(dcterms + "title")); var getTriplesDctermsTitle = graph.GetTriplesWithPredicate(RdfUris.DcTermsTitle);
if (!getTriplesDctermsTitle.Any()) if (!getTriplesDctermsTitle.Any())
{ {
AssertToGraphLiteralNode(graph, resourceTypeGraphName, dcterms + "title", entry.DisplayName); AssertToGraphLiteralNode(graph, resourceTypeGraphName, RdfUris.DcTermsTitle, entry.SpecificType);
Console.WriteLine($"For resource type '{entry.DisplayName}' will migrate triple '{graph.BaseUri} {dcterms}title {entry.DisplayName}'. "); Console.WriteLine($"For resource type '{entry.SpecificType}' will migrate triple '{graph.BaseUri} {RdfUris.DcTermsTitle} {entry.SpecificType}'. ");
} }
else else
{ {
Console.WriteLine($"For resource type '{entry.DisplayName}' will NOT migrate triple '{graph.BaseUri} {dcterms}title {entry.DisplayName}'. "); Console.WriteLine($"For resource type '{entry.SpecificType}' will NOT migrate triple '{graph.BaseUri} {RdfUris.DcTermsTitle} {entry.SpecificType}'. ");
} }
// check if a triple with dcterms:title '{entry.DisplayName}' already exists in the role graph
var getTriplesDctermsAlternative = graph.GetTriplesWithPredicate(RdfUris.DcTermsAlternative);
if (!getTriplesDctermsAlternative.Any())
{
AssertToGraphLiteralNode(graph, resourceTypeGraphName, RdfUris.DcTermsAlternative, entry.GeneralType);
Console.WriteLine($"For resource type '{entry.SpecificType}' will migrate triple '{graph.BaseUri} {RdfUris.DcTermsAlternative} {entry.GeneralType}'. ");
}
else
{
Console.WriteLine($"For resource type '{entry.SpecificType}' will NOT migrate triple '{graph.BaseUri} {RdfUris.DcTermsAlternative} {entry.GeneralType}'. ");
}
if (!getTriplesDcatDataService.Any() || !getTriplesDctermsTitle.Any()) if (!getTriplesDcatDataService.Any() || !getTriplesDctermsTitle.Any())
{ {
graphs.Add(graph); graphs.Add(graph);
} }
} }
return graphs; return await Task.FromResult(graphs);
} }
} }
\ No newline at end of file
}
\ No newline at end of file
using Coscine.Database.DataModel; using Coscine.ApiClient;
using Coscine.Database.Models; using Coscine.ApiClient.Core.Api;
using Coscine.ApiClient.Core.Model;
using SQL2Linked.Utils;
using VDS.RDF; using VDS.RDF;
using VDS.RDF.Parsing;
namespace SQL2Linked.Implementations namespace SQL2Linked.Implementations;
/// <summary>
/// Class responsible for converting role data into linked data graphs.
/// It retrieves role information from the API and then transforms this data into a series of RDF graphs,
/// making use of predefined URIs and RDF constructs.
/// </summary>
public class RoleStructuralData : StructuralData<RoleDto>
{ {
public class RoleStructuralData : StructuralData<Role, RoleModel> /// <summary>
/// Asynchronously retrieves all role data.
/// </summary>
/// <returns>A <see cref="Task"/> that represents the asynchronous operation and returns a collection of <see cref="RoleDto"/>.</returns>
public override async Task<IEnumerable<RoleDto>> GetAll()
{ {
public readonly string RoleUrlPrefix = "https://purl.org/coscine/roles"; var roleApi = new RoleApi(_apiConfiguration);
public readonly Uri cosc = new("https://purl.org/coscine/"); return await RequestUtil.WrapPagedRequest<RoleDtoPagedResponse, RoleDto>(
public readonly Uri rdf = new("http://www.w3.org/1999/02/22-rdf-syntax-ns#"); (currentPage) => roleApi.GetRolesAsync(pageNumber: currentPage)
public readonly Uri org = new("http://www.w3.org/ns/org#"); );
public readonly Uri dcterms = new("http://purl.org/dc/terms/"); }
public override IEnumerable<IGraph> ConvertToLinkedData(IEnumerable<Role> entries) /// <summary>
/// Converts a collection of role data entries into a set of RDF graphs.
/// Each role is transformed into a graph, with RDF triples representing various properties of the role.
/// </summary>
/// <param name="entries">A collection of <see cref="RoleDto"/> instances representing role data.</param>
/// <returns>A collection of <see cref="IGraph"/> instances, each representing an RDF graph of a role.</returns>
public override async Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<RoleDto> entries)
{ {
var graphs = new List<IGraph>(); var graphs = new List<IGraph>();
foreach (var entry in entries) foreach (var entry in entries)
{ {
var roleGraphName = $"{RoleUrlPrefix}/{entry.Id}"; var roleGraphName = UriHelper.TryCombineUri(RdfUris.CoscineRoles, entry.Id)
var graph = RdfStoreConnector.GetGraph(roleGraphName); ?? throw new Exception("Could not combine role prefix with role ID");
var response = await _adminApi.GetMetadataGraphAsync(roleGraphName.AbsoluteUri, RdfFormat.TextTurtle);
var graph = new Graph()
{
BaseUri = roleGraphName
};
graph.LoadFromString(response.Data.Content, new TurtleParser());
// check if a triple with a org:role already exists in the role graph // check if a triple with a org:role already exists in the role graph
var getTriplesOrgRole = graph.GetTriplesWithObject(new Uri(org + "Role")); var getTriplesOrgRole = graph.GetTriplesWithObject(RdfUris.OrgRoleClass);
if (!getTriplesOrgRole.Any()) if (!getTriplesOrgRole.Any())
{ {
AssertToGraphUriNode(graph, roleGraphName, rdf + "type", org + "Role"); AssertToGraphUriNode(graph, roleGraphName, RdfUris.A, RdfUris.OrgRoleClass);
Console.WriteLine($"For role '{entry.DisplayName}' will migrate triple '{graph.BaseUri} {rdf}type {org}Role'. "); Console.WriteLine($"For role '{entry.DisplayName}' will migrate triple '{graph.BaseUri} {RdfUris.A} {RdfUris.OrgRoleClass}'. ");
} }
else else
{ {
Console.WriteLine($"For role '{entry.DisplayName}' will NOT migrate triple '{graph.BaseUri} {rdf}type {org}Role'. "); Console.WriteLine($"For role '{entry.DisplayName}' will NOT migrate triple '{graph.BaseUri} {RdfUris.A} {RdfUris.OrgRoleClass}'. ");
} }
// check if a triple with dcterms:title '{entry.DisplayName}' already exists in the role graph // check if a triple with dcterms:title '{entry.DisplayName}' already exists in the role graph
var getTriplesDctermsTitle = graph.GetTriplesWithPredicate(new Uri(dcterms + "title")); var getTriplesDctermsTitle = graph.GetTriplesWithPredicate(RdfUris.DcTermsTitle);
if (!getTriplesDctermsTitle.Any()) if (!getTriplesDctermsTitle.Any())
{ {
AssertToGraphLiteralNode(graph, roleGraphName, dcterms + "title", entry.DisplayName); AssertToGraphLiteralNode(graph, roleGraphName, RdfUris.DcTermsTitle, entry.DisplayName);
Console.WriteLine($"For role '{entry.DisplayName}' will migrate triple '{graph.BaseUri} {dcterms}title {entry.DisplayName}'. "); Console.WriteLine($"For role '{entry.DisplayName}' will migrate triple '{graph.BaseUri} {RdfUris.DcTermsTitle} {entry.DisplayName}'. ");
} }
else else
{ {
Console.WriteLine($"For role '{entry.DisplayName}' will NOT migrate triple '{graph.BaseUri} {dcterms}title {entry.DisplayName}'. "); Console.WriteLine($"For role '{entry.DisplayName}' will NOT migrate triple '{graph.BaseUri} {RdfUris.DcTermsTitle} {entry.DisplayName}'. ");
} }
if (!getTriplesOrgRole.Any() || !getTriplesDctermsTitle.Any()) if (!getTriplesOrgRole.Any() || !getTriplesDctermsTitle.Any())
{ {
...@@ -54,4 +82,3 @@ namespace SQL2Linked.Implementations ...@@ -54,4 +82,3 @@ namespace SQL2Linked.Implementations
return graphs; return graphs;
} }
} }
\ No newline at end of file
}
\ No newline at end of file
using Coscine.Database.DataModel; using Coscine.ApiClient;
using Coscine.Database.Models; using Coscine.ApiClient.Core.Model;
using Coscine.Metadata; using SQL2Linked.Utils;
using VDS.RDF; using VDS.RDF;
using VDS.RDF.Parsing;
namespace SQL2Linked.Implementations namespace SQL2Linked.Implementations;
{
public class UserStructuralData : StructuralData<User, UserModel>
{
public readonly string UserUrlPrefix = "https://purl.org/coscine/users";
public readonly Uri rdf = new("http://www.w3.org/1999/02/22-rdf-syntax-ns#");
public readonly Uri foaf = new("http://xmlns.com/foaf/0.1/");
// Override to only transform users which have accepted the TOS /// <summary>
public override IEnumerable<User> GetAll() /// Class responsible for converting user data into linked data graphs.
/// It retrieves user information from the API and then transforms this data into a series of RDF graphs,
/// making use of predefined URIs and RDF constructs.
/// </summary>
public class UserStructuralData : StructuralData<UserDto>
{ {
return Model.GetAllWhere((user) => user.Tosaccepteds.Any()); /// <summary>
/// Asynchronously retrieves all user data only for users who have accepted the ToS.
/// </summary>
/// <returns>A <see cref="Task"/> that represents the asynchronous operation and returns a collection of <see cref="UserDto"/>.</returns>
/// <remarks>This override allows for the retrieval of users who have accepted the ToS.</remarks>
public override async Task<IEnumerable<UserDto>> GetAll()
{
return await RequestUtil.WrapPagedRequest<UserDtoPagedResponse, UserDto>(
(currentPage) => _adminApi.GetAllUsersAsync(tosAccepted: true, pageNumber: currentPage, pageSize: 250)
);
} }
public override IEnumerable<IGraph> ConvertToLinkedData(IEnumerable<User> entries) /// <summary>
/// Converts a collection of user data entries into a set of RDF graphs.
/// Each user is transformed into a graph, with RDF triples representing various properties of the user.
/// </summary>
/// <param name="entries">A collection of <see cref="UserDto"/> instances representing user data.</param>
/// <returns>A collection of <see cref="IGraph"/> instances, each representing an RDF graph of a user.</returns>
public override async Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<UserDto> entries)
{ {
var graphs = new List<IGraph>(); var graphs = new List<IGraph>();
foreach (var entry in entries) foreach (var entry in entries)
{ {
var userGraphName = $"{UserUrlPrefix}/{entry.Id}"; var userGraphName = UriHelper.TryCombineUri(RdfUris.CoscineUsers, entry.Id)
var graph = RdfStoreConnector.GetGraph(userGraphName); ?? throw new Exception("Could not combine users prefix with user ID");
var response = _adminApi.GetMetadataGraph(userGraphName.AbsoluteUri, RdfFormat.TextTurtle);
var graph = new Graph()
{
BaseUri = userGraphName
};
graph.LoadFromString(response.Data.Content, new TurtleParser());
// check if a triple with a foaf:Person already exists in the user graph // check if a triple with a foaf:Person already exists in the user graph
var getTriples = graph.GetTriplesWithObject(new Uri(foaf + "Person")); var getTriples = graph.GetTriplesWithObject(RdfUris.FoafPersonClass);
// check if the current display name is already applied in the user graph // check if the current display name is already applied in the user graph
var getTriplesName = graph.GetTriplesWithPredicate(new Uri(foaf + "name")); var getTriplesName = graph.GetTriplesWithPredicate(RdfUris.FoafName);
if (!getTriples.Any() || !getTriplesName.Any((triple) => triple.Object.ToString() == entry.DisplayName)) if (!getTriples.Any() || !getTriplesName.Any((triple) => (triple.Object as ILiteralNode)?.Value == entry.DisplayName))
{ {
AssertToGraphUriNode(graph, userGraphName, rdf + "type", foaf + "Person"); AssertToGraphUriNode(graph, userGraphName, RdfUris.A, RdfUris.FoafPersonClass);
Console.WriteLine($"For user '{entry.DisplayName}' will migrate triple '{userGraphName} {RdfUris.A} {RdfUris.FoafPersonClass}'. ");
graph.Retract(getTriplesName); graph.Retract(getTriplesName);
AssertToGraphLiteralNode(graph, userGraphName, foaf + "name", entry.DisplayName); AssertToGraphLiteralNode(graph, userGraphName, RdfUris.FoafName, entry.DisplayName);
graphs.Add(graph); graphs.Add(graph);
...@@ -47,7 +70,6 @@ namespace SQL2Linked.Implementations ...@@ -47,7 +70,6 @@ namespace SQL2Linked.Implementations
Console.WriteLine($"Will NOT migrate user '{entry.DisplayName}' with id '{entry.Id}'."); Console.WriteLine($"Will NOT migrate user '{entry.DisplayName}' with id '{entry.Id}'.");
} }
} }
return graphs; return await Task.FromResult(graphs);
}
} }
} }
namespace SQL2Linked.Models.ConfigurationModels;
/// <summary>
/// Represents the configuration settings for the PID ePIC API used in the application.
/// </summary>
public class PidConfiguration
{
/// <summary>
/// The section name in the configuration file.
/// </summary>
public static readonly string Section = "PidConfiguration";
/// <summary>
/// Prefix for the PID ePIC API.
/// </summary>
public string Prefix { get; init; } = null!;
}
namespace SQL2Linked.Models.ConfigurationModels;
/// <summary>
/// Represents the configuration settings used in the application.
/// </summary>
public class SQL2LinkedConfiguration
{
/// <summary>
/// The section name in the configuration file.
/// </summary>
public static readonly string Section = "SQL2LinkedConfiguration";
/// <summary>
/// Value indicating whether SQL2Linked is enabled.
/// </summary>
public bool IsEnabled { get; init; }
}
\ No newline at end of file
using SQL2Linked.Implementations; using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using SQL2Linked.Implementations;
using SQL2Linked.Models.ConfigurationModels;
using Winton.Extensions.Configuration.Consul;
namespace SQL2Linked;
public class Program
{
private static IServiceProvider _serviceProvider = null!;
private static async Task Main(string[] args)
{
InitializeServices();
var sql2LinkedConfiguration = _serviceProvider.GetRequiredService<IOptionsMonitor<SQL2LinkedConfiguration>>().CurrentValue;
if (!sql2LinkedConfiguration.IsEnabled)
{
Console.WriteLine("SQL 2 Linked Data migration is disabled. To enable it, set the \"IsEnabled\" property to \"true\" in the configuration.");
return;
}
var dummyMode = !(args.Length > 0 && args[0] == "--noDryRun"); var dummyMode = !(args.Length > 0 && args[0] == "--noDryRun");
if (dummyMode) if (dummyMode)
...@@ -9,19 +31,68 @@ if (dummyMode) ...@@ -9,19 +31,68 @@ if (dummyMode)
Console.WriteLine("\nBegin SQL 2 Linked Data migration"); Console.WriteLine("\nBegin SQL 2 Linked Data migration");
var roleStructuralData = new RoleStructuralData(); var roleStructuralData = _serviceProvider.GetRequiredService<RoleStructuralData>();
roleStructuralData.Migrate(dummyMode); await roleStructuralData.Migrate(dummyMode);
var userStructuralData = new UserStructuralData(); var userStructuralData = _serviceProvider.GetRequiredService<UserStructuralData>();
userStructuralData.Migrate(dummyMode); await userStructuralData.Migrate(dummyMode);
var resourceTypeStructuralData = new ResourceTypeStructuralData(); var resourceTypeStructuralData = _serviceProvider.GetRequiredService<ResourceTypeStructuralData>();
resourceTypeStructuralData.Migrate(dummyMode); await resourceTypeStructuralData.Migrate(dummyMode);
var projectStructuralData = new ProjectStructuralData(); var projectStructuralData = _serviceProvider.GetRequiredService<ProjectStructuralData>();
projectStructuralData.Migrate(dummyMode); await projectStructuralData.Migrate(dummyMode);
var resourceStructuralData = new ResourceStructuralData(); var resourceStructuralData = _serviceProvider.GetRequiredService<ResourceStructuralData>();
resourceStructuralData.Migrate(dummyMode); await resourceStructuralData.Migrate(dummyMode);
Console.WriteLine("\n Finished."); Console.WriteLine("\n Finished.");
}
private static void InitializeServices()
{
// Create a new instance of ConfigurationBuilder
var configBuilder = new ConfigurationBuilder();
// Define the Consul URL
var consulUrl = Environment.GetEnvironmentVariable("CONSUL_URL") ?? "http://localhost:8500";
// Remove the default sources
configBuilder.Sources.Clear();
// Add Consul as a configuration source
var configuration = configBuilder
.AddConsul(
"coscine/Coscine.Infrastructure/SQL2Linked/appsettings",
options =>
{
options.ConsulConfigurationOptions =
cco => cco.Address = new Uri(consulUrl);
options.Optional = true;
options.ReloadOnChange = true;
options.PollWaitTime = TimeSpan.FromSeconds(5);
options.OnLoadException = exceptionContext => exceptionContext.Ignore = true;
}
)
.AddEnvironmentVariables()
.Build();
var services = new ServiceCollection()
.AddSingleton<IConfiguration>(configuration);
// Add the configuration to the service collection
services.Configure<SQL2LinkedConfiguration>(settings =>
{
configuration.GetSection(SQL2LinkedConfiguration.Section).Bind(settings);
});
// Add the services to the service collection
services.AddTransient<ProjectStructuralData>();
services.AddTransient<ResourceStructuralData>();
services.AddTransient<ResourceTypeStructuralData>();
services.AddTransient<RoleStructuralData>();
services.AddTransient<UserStructuralData>();
_serviceProvider = services.BuildServiceProvider();
}
}
\ No newline at end of file
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup> <PropertyGroup>
<OutputType>Exe</OutputType> <OutputType>Exe</OutputType>
<TargetFramework>net6.0</TargetFramework> <TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings> <ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable> <Nullable>enable</Nullable>
<Version>0.1.19</Version></PropertyGroup> <Version>0.1.19</Version>
</PropertyGroup>
<PropertyGroup>
<Authors>RWTH Aachen University</Authors>
<Company>IT Center, RWTH Aachen University</Company>
<Copyright>©2024 IT Center, RWTH Aachen University</Copyright>
<Description>SQL2Linked is a part of the Coscine group.</Description>
</PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="Coscine.Database" Version="2.*-*" /> <PackageReference Include="Coscine.ApiClient" Version="1.3.0" />
<PackageReference Include="Coscine.Metadata" Version="2.*-*" /> <PackageReference Include="dotNetRdf" Version="3.1.1" />
<PackageReference Include="Polly" Version="7.2.3" /> <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Options" Version="8.0.1" />
</ItemGroup> </ItemGroup>
</Project> </Project>
using Coscine.Configuration; using Coscine.ApiClient;
using Coscine.Database.Models; using Coscine.ApiClient.Core.Api;
using Coscine.Metadata; using Coscine.ApiClient.Core.Client;
using Polly; using Coscine.ApiClient.Core.Model;
using Microsoft.Extensions.Configuration;
using SQL2Linked.Models.ConfigurationModels;
using VDS.RDF; using VDS.RDF;
namespace SQL2Linked namespace SQL2Linked;
{
public abstract class StructuralData<S, T> where S : class where T : DatabaseModel<S>, new()
{
public T Model { get; init; }
public ConsulConfiguration Configuration { get; init; }
public RdfStoreConnector RdfStoreConnector { get; init; }
public static string Prefix { get; set; }
/// <summary>
/// Provides an abstract base class for handling structural data transformations and migrations.
/// </summary>
/// <typeparam name="S">The type of data to be transformed or migrated.</typeparam>
/// <remarks>
/// This class is partially shared between this script and the Trellis Migrator script.
/// </remarks>
public abstract class StructuralData<S>
{
private readonly string _adminToken;
protected readonly Configuration _apiConfiguration;
protected readonly AdminApi _adminApi;
protected readonly PidConfiguration _pidConfiguration; // Comes from the API Client, not from the application's own configuration
/// <summary>
/// Initializes a new instance of the <see cref="StructuralData{S}"/> class.
/// </summary>
public StructuralData() public StructuralData()
{ {
Configuration = new ConsulConfiguration(); // Retrieve the configuration settings for the PID ePIC API from the API Client
RdfStoreConnector = new RdfStoreConnector(Configuration.GetStringAndWait("coscine/local/virtuoso/additional/url")); var apiConfiguration = ApiConfigurationUtil.RetrieveApiConfiguration();
Model = new T(); _pidConfiguration = apiConfiguration.GetSection(PidConfiguration.Section).Get<PidConfiguration>() ?? new();
Prefix = Configuration.GetStringAndWait("coscine/global/epic/prefix");
// 100 second timeout
var timeout = 100000;
RdfStoreConnector.QueryEndpoint.Timeout = timeout;
RdfStoreConnector.UpdateEndpoint.Timeout = timeout;
RdfStoreConnector.ReadWriteSparqlConnector.Timeout = timeout;
}
public abstract IEnumerable<IGraph> ConvertToLinkedData(IEnumerable<S> entries); // Ensiure that the prefix is not null or empty.
ArgumentException.ThrowIfNullOrWhiteSpace(_pidConfiguration.Prefix, nameof(_pidConfiguration.Prefix));
public virtual IEnumerable<S> GetAll() // Generate an admin token for the API Client
var jwtConfiguration = ApiConfigurationUtil.RetrieveJwtConfiguration();
_adminToken = ApiConfigurationUtil.GenerateAdminToken(jwtConfiguration);
_apiConfiguration = new Configuration()
{ {
return Model.GetAll(); BasePath = "http://localhost:7206/coscine",
ApiKeyPrefix = { { "Authorization", "Bearer" } },
ApiKey = { { "Authorization", _adminToken } },
};
_adminApi = new AdminApi(_apiConfiguration);
} }
public void Migrate(bool dummyMode) /// <summary>
/// Converts the given entries to linked data graphs.
/// </summary>
/// <param name="entries">The entries to convert.</param>
/// <returns>An enumerable collection of graphs representing the linked data.</returns>
public abstract Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<S> entries);
/// <summary>
/// Retrieves all entries of type <typeparamref name="S"/>.
/// </summary>
/// <returns>A task that represents the asynchronous operation. The task result contains an enumerable of all entries.</returns>
public abstract Task<IEnumerable<S>> GetAll();
/// <summary>
/// Migrates the data, optionally in a dummy mode where changes are not persisted.
/// </summary>
/// <param name="dummyMode">If set to <c>true</c>, the migration is simulated but not executed.</param>
/// <returns>A task representing the asynchronous operation.</returns>
public async Task Migrate(bool dummyMode)
{ {
var spacer = new string('-', 35); var spacer = new string('-', 35);
Console.WriteLine($"\n{spacer}\n{typeof(T).Name}\n{spacer}"); Console.WriteLine($"\n{spacer}\n{typeof(S).Name}\n{spacer}");
var graphs = ConvertToLinkedData(GetAll()); var graphs = await ConvertToLinkedDataAsync(await GetAll());
if (!dummyMode) if (!dummyMode)
{ {
StoreGraphs(graphs); await StoreGraphs(graphs);
} }
} }
public void StoreGraphs(IEnumerable<IGraph> graphs) /// <summary>
/// Stores the given graphs in the underlying data store.
/// </summary>
/// <param name="graphs">The graphs to be stored.</param>
/// <returns>A task representing the asynchronous operation.</returns>
public async Task StoreGraphs(IEnumerable<IGraph> graphs)
{ {
var formatEnum = RdfFormat.TextTurtle;
var format = "text/turtle";
foreach (var graph in graphs) foreach (var graph in graphs)
{
try
{ {
Console.WriteLine($" ({graph.BaseUri})"); Console.WriteLine($" ({graph.BaseUri})");
if (graph is WrapperGraph) try
{
var wrapperGraph = (WrapperGraph)graph;
// Chunking since the size otherwise can be too large
foreach (var triples in wrapperGraph.AssertList.Chunk(100))
{
WrapRequest(() => RdfStoreConnector.ReadWriteSparqlConnector.UpdateGraph(graph.BaseUri, triples, new List<Triple>()));
}
// Chunking since the size otherwise can be too large
foreach (var triples in wrapperGraph.RetractList.Chunk(100))
{
WrapRequest(() => RdfStoreConnector.ReadWriteSparqlConnector.UpdateGraph(graph.BaseUri, new List<Triple>(), triples));
}
}
else
{
var exists = WrapRequest(() => RdfStoreConnector.HasGraph(graph.BaseUri));
if (exists)
{ {
Console.WriteLine($" - Graph {graph.BaseUri} exists"); var rdfWriter = MimeTypesHelper.GetWriter(format);
var content = VDS.RDF.Writing.StringWriter.Write(graph, rdfWriter);
// Clear the existing graph from the store await _adminApi.UpdateMetadataGraphAsync(
WrapRequest(() => RdfStoreConnector.ClearGraph(graph.BaseUri)); graph.BaseUri.AbsoluteUri,
Console.WriteLine($" - Cleared Graph {graph.BaseUri}"); new MetadataUpdateAdminParameters(new RdfDefinitionForManipulationDto(content, formatEnum))
} );
// Chunking since the size otherwise can be too large
// Don't change to only addition of triples, otherwise this could break things
foreach (var triples in graph.Triples.Chunk(100))
{
WrapRequest(() => RdfStoreConnector.ReadWriteSparqlConnector.UpdateGraph(graph.BaseUri, triples, new List<Triple>()));
}
}
Console.WriteLine($" - Graph {graph.BaseUri} added successfully"); Console.WriteLine($" - Graph {graph.BaseUri} added successfully");
Console.WriteLine(); Console.WriteLine();
...@@ -99,74 +112,67 @@ namespace SQL2Linked ...@@ -99,74 +112,67 @@ namespace SQL2Linked
} }
} }
public void AssertToGraphUriNode(IGraph graph, string graphSubject, string graphPredicate, string? graphObject) /// <summary>
/// Asserts a triple to the graph with a URI node object.
/// </summary>
/// <param name="graph">The graph to assert to.</param>
/// <param name="graphSubject">The subject URI of the triple.</param>
/// <param name="graphPredicate">The predicate URI of the triple.</param>
/// <param name="graphObject">The object URI of the triple.</param>
public void AssertToGraphUriNode(IGraph graph, Uri graphSubject, Uri graphPredicate, Uri? graphObject)
{ {
if (graphObject != null) if (graphObject != null)
{ {
graph.Assert( graph.Assert(
new Triple( new Triple(
graph.CreateUriNode(new Uri(graphSubject)), graph.CreateUriNode(graphSubject),
graph.CreateUriNode(new Uri(graphPredicate)), graph.CreateUriNode(graphPredicate),
graph.CreateUriNode(new Uri(graphObject)) graph.CreateUriNode(graphObject)
) )
); );
} }
} }
public void AssertToGraphLiteralNode(IGraph graph, string graphSubject, string graphPredicate, string? graphObject, Uri? objectType = null) /// <summary>
/// Asserts a triple to the graph with a literal node object.
/// </summary>
/// <param name="graph">The graph to assert to.</param>
/// <param name="graphSubject">The subject URI of the triple.</param>
/// <param name="graphPredicate">The predicate URI of the triple.</param>
/// <param name="graphObject">The literal object of the triple.</param>
/// <param name="objectType">The data type URI of the literal object, if any.</param>
public void AssertToGraphLiteralNode(IGraph graph, Uri graphSubject, Uri graphPredicate, string? graphObject, Uri? objectType = null)
{ {
if (graphObject != null) if (graphObject != null)
{ {
graph.Assert( graph.Assert(
new Triple( new Triple(
graph.CreateUriNode(new Uri(graphSubject)), graph.CreateUriNode(graphSubject),
graph.CreateUriNode(new Uri(graphPredicate)), graph.CreateUriNode(graphPredicate),
graph.CreateLiteralNode(graphObject, objectType) objectType is not null ? graph.CreateLiteralNode(graphObject, objectType) : graph.CreateLiteralNode(graphObject)
) )
); );
} }
} }
public void AssertToGraphBlankAndUriNode(IGraph graph, IBlankNode graphSubject, string graphPredicate, string? graphObject) /// <summary>
/// Asserts a triple to the graph with a blank node subject and a URI node object.
/// </summary>
/// <param name="graph">The graph to assert to.</param>
/// <param name="graphSubject">The blank node subject of the triple.</param>
/// <param name="graphPredicate">The predicate URI of the triple.</param>
/// <param name="graphObject">The object URI of the triple.</param>
public void AssertToGraphBlankAndUriNode(IGraph graph, IBlankNode graphSubject, Uri graphPredicate, Uri? graphObject)
{ {
if (graphObject != null) if (graphObject != null)
{ {
graph.Assert( graph.Assert(
new Triple( new Triple(
graphSubject, graphSubject,
graph.CreateUriNode(new Uri(graphPredicate)), graph.CreateUriNode(graphPredicate),
graph.CreateUriNode(new Uri(graphObject)) graph.CreateUriNode(graphObject)
) )
); );
} }
} }
/// <summary>
/// Retry Virtuoso Requests since they sometimes just fail
/// </summary>
/// <typeparam name="W"></typeparam>
/// <param name="function"></param>
/// <returns></returns>
public void WrapRequest(Action action)
{
Policy
.Handle<Exception>()
.WaitAndRetry(5, retryNumber => TimeSpan.FromMilliseconds(200))
.Execute(() => action.Invoke());
}
/// <summary>
/// Retry Virtuoso Requests since they sometimes just fail
/// </summary>
/// <typeparam name="W"></typeparam>
/// <param name="function"></param>
/// <returns></returns>
public W WrapRequest<W>(Func<W> function)
{
return Policy
.Handle<Exception>()
.WaitAndRetry(5, retryNumber => TimeSpan.FromMilliseconds(200))
.ExecuteAndCapture(() => function.Invoke()).Result;
}
}
} }
// Ignore Spelling: Foaf Dcat Dcterms Ebocore Fdp Ldp Xsd
/// ------------------
/// TAKEN FROM THE API
/// ------------------
namespace SQL2Linked.Utils;
/// <summary>
/// Contains URIs relevant to RDF (Resource Description Framework) and related schemas/ontologies.
/// </summary>
public static class RdfUris
{
// ACL
public static readonly Uri AclPrefix = new("http://www.w3.org/ns/auth/acl#");
public static readonly Uri AclAccessTo = new("http://www.w3.org/ns/auth/acl#accessTo");
public static readonly Uri AclAgentGroup = new("http://www.w3.org/ns/auth/acl#agentGroup");
public static readonly Uri AclDefault = new("http://www.w3.org/ns/auth/acl#default");
public static readonly Uri AclMode = new("http://www.w3.org/ns/auth/acl#mode");
public static readonly Uri AclAuthorizationClass = new("http://www.w3.org/ns/auth/acl#Authorization");
public static readonly Uri AclReadClass = new("http://www.w3.org/ns/auth/acl#Read");
public static readonly Uri AclWriteClass = new("http://www.w3.org/ns/auth/acl#Write");
// Coscine
public static readonly Uri CoscinePrefix = new("https://purl.org/coscine/");
public static readonly Uri CoscineApplicationProfile = new("https://purl.org/coscine/ap/");
public static readonly Uri CoscineFixedValue = new("https://purl.org/coscine/fixedValue");
public static readonly Uri CoscineMetadataExtractionVersion = new("https://purl.org/coscine/terms/metatadataextraction#version");
public static readonly Uri CoscineProjects = new("https://purl.org/coscine/projects/");
public static readonly Uri CoscineResourceTypes = new("https://purl.org/coscine/resourcetypes/");
public static readonly Uri CoscineResources = new("https://purl.org/coscine/resources/");
public static readonly Uri CoscineRoles = new("https://purl.org/coscine/roles/");
public static readonly Uri CoscineUsers = new("https://purl.org/coscine/users/");
[Obsolete($"Beware the side-effects before changing! Use {nameof(CoscineUsers)} instead.")]
public static readonly Uri CoscineUserLegacy = new("https://coscine.rwth-aachen.de/u/");
// Coscine Terms
public static readonly Uri CoscineTermsPrefix = new("https://purl.org/coscine/terms/");
public static readonly Uri CoscineTermsLinkedBody = new("https://purl.org/coscine/terms/linked#body");
public static readonly Uri CoscineTermsMetadataTrackerAgent = new("https://purl.org/coscine/terms/metadatatracker#Agent");
public static readonly Uri CoscineTermsProject = new("https://purl.org/coscine/terms/project#");
public static readonly Uri CoscineTermsProjectDeleted = new("https://purl.org/coscine/terms/project#deleted");
public static readonly Uri CoscineTermsProjectSlug = new("https://purl.org/coscine/terms/project#slug");
public static readonly Uri CoscineTermsProjectVisibility = new("https://purl.org/coscine/terms/project#visibility");
public static readonly Uri CoscineTermsResource = new("https://purl.org/coscine/terms/resource#");
public static readonly Uri CoscineTermsResourceArchived = new("https://purl.org/coscine/terms/resource#archived");
public static readonly Uri CoscineTermsResourceDeleted = new("https://purl.org/coscine/terms/resource#deleted");
public static readonly Uri CoscineTermsResourceFixedValues = new("https://purl.org/coscine/terms/resource#fixedValues");
public static readonly Uri CoscineTermsResourceVisibility = new("https://purl.org/coscine/terms/resource#visibility");
public static readonly Uri CoscineTermsTypes = new("https://purl.org/coscine/terms/types#");
public static readonly Uri CoscineTermsUserAgent = new("https://purl.org/coscine/terms/user#Agent");
public static readonly Uri CoscineTermsVisibility = new("https://purl.org/coscine/terms/visibility#");
public static readonly Uri CoscineTermsVisibilityPublic = new("https://purl.org/coscine/terms/visibility#public");
public static readonly Uri CoscineTermsVisibilityProjectMember = new("https://purl.org/coscine/terms/visibility#projectMember");
public static readonly Uri CoscineTermsRole = new("https://purl.org/coscine/terms/role#");
// CSMD
public static readonly Uri CsmdPrefix = new("http://www.purl.org/net/CSMD/4.0#");
// DCAT
public static readonly Uri DcatPrefix = new("http://www.w3.org/ns/dcat#");
public static readonly Uri DcatCatalog = new("http://www.w3.org/ns/dcat#catalog");
public static readonly Uri DcatDataset = new("http://www.w3.org/ns/dcat#dataset");
public static readonly Uri DcatDistribution = new("http://www.w3.org/ns/dcat#distribution");
public static readonly Uri DcatService = new("http://www.w3.org/ns/dcat#service");
public static readonly Uri DcatCatalogClass = new("http://www.w3.org/ns/dcat#Catalog");
public static readonly Uri DcatDataServiceClass = new("http://www.w3.org/ns/dcat#DataService");
public static readonly Uri DcatDatasetClass = new("http://www.w3.org/ns/dcat#Dataset");
// DCMI Type
public static readonly Uri DcmiTypePrefix = new("http://purl.org/dc/dcmitype/");
// DC
public static readonly Uri DcPrefix = new("http://purl.org/dc/elements/1.1/");
// DC Terms
public static readonly Uri DcTermsPrefix = new("http://purl.org/dc/terms/");
public static readonly Uri DcTermsAlternative = new("http://purl.org/dc/terms/alternative");
public static readonly Uri DcTermsConformsTo = new("http://purl.org/dc/terms/conformsTo");
public static readonly Uri DcTermsCreator = new("http://purl.org/dc/terms/creator");
public static readonly Uri DcTermsCreated = new("http://purl.org/dc/terms/created");
public static readonly Uri DcTermsDescription = new("http://purl.org/dc/terms/description");
public static readonly Uri DcTermsEndDate = new("http://purl.org/dc/terms/endDate");
public static readonly Uri DcTermsIdentifier = new("http://purl.org/dc/terms/identifier");
public static readonly Uri DcTermsIsPartOf = new("http://purl.org/dc/terms/isPartOf");
public static readonly Uri DcTermsLicense = new("http://purl.org/dc/terms/license");
public static readonly Uri DcTermsModified = new("http://purl.org/dc/terms/modified");
public static readonly Uri DcTermsRights = new("http://purl.org/dc/terms/rights");
public static readonly Uri DcTermsRightsHolder = new("http://purl.org/dc/terms/rightsHolder");
public static readonly Uri DcTermsStartDate = new("http://purl.org/dc/terms/startDate");
public static readonly Uri DcTermsSubject = new("http://purl.org/dc/terms/subject");
public static readonly Uri DcTermsTitle = new("http://purl.org/dc/terms/title");
// EBUCORE
public static readonly Uri EbocorePrefix = new("http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#");
public static readonly Uri EbocoreHashFunction = new("http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#hashFunction");
public static readonly Uri EbocoreHashType = new("http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#hashType");
public static readonly Uri EbocoreHashValue = new("http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#hashValue");
// FDP
public static readonly Uri FdpPrefix = new("http://purl.org/fdp/fdp-o#");
public static readonly Uri FdpHasMetadata = new("http://purl.org/fdp/fdp-o#hasMetadata");
public static readonly Uri FdpIsMetadataOf = new("http://purl.org/fdp/fdp-o#isMetadataOf");
public static readonly Uri FdpMetadataServiceClass = new("http://purl.org/fdp/fdp-o#MetadataService");
// FOAF
public static readonly Uri FoafPrefix = new("http://xmlns.com/foaf/0.1/");
public static readonly Uri FoafHomepage = new("http://xmlns.com/foaf/0.1/homepage");
public static readonly Uri FoafName = new("http://xmlns.com/foaf/0.1/name");
public static readonly Uri FoafPersonClass = new("http://xmlns.com/foaf/0.1/Person");
// Handle
public static readonly Uri HandlePrefix = new("https://hdl.handle.net/");
// LDP
public static readonly Uri LdpPrefix = new("http://www.w3.org/ns/ldp#");
public static readonly Uri LdpDescribedBy = new("http://www.w3.org/ns/ldp#describedBy");
public static readonly Uri LdpBasicContainerClass = new("http://www.w3.org/ns/ldp#BasicContainer");
public static readonly Uri LdpNonRDFSourceClass = new("http://www.w3.org/ns/ldp#NonRDFSource");
public static readonly Uri LdpRDFSourceClass = new("http://www.w3.org/ns/ldp#RDFSource");
// ORG
public static readonly Uri OrgPrefix = new("http://www.w3.org/ns/org#");
public static readonly Uri OrgMember = new("http://www.w3.org/ns/org#member");
public static readonly Uri OrgOrganization = new("http://www.w3.org/ns/org#organization");
public static readonly Uri OrgRole = new("http://www.w3.org/ns/org#role");
public static readonly Uri OrgMembershipClass = new("http://www.w3.org/ns/org#Membership");
public static readonly Uri OrgOrganizationalCollaborationClass = new("http://www.w3.org/ns/org#OrganizationalCollaboration");
public static readonly Uri OrgRoleClass = new("http://www.w3.org/ns/org#Role");
// OWL
public static readonly Uri OwlPrefix = new("http://www.w3.org/2002/07/owl#");
public static readonly Uri OwlImports = new("http://www.w3.org/2002/07/owl#imports");
// PIM
public static readonly Uri PimPrefix = new("http://www.w3.org/ns/pim/space#");
public static readonly Uri PimStorageClass = new("http://www.w3.org/ns/pim/space#Storage");
// PROV
public static readonly Uri ProvPrefix = new("http://www.w3.org/ns/prov#");
public static readonly Uri ProvEntityClass = new("http://www.w3.org/ns/prov#Entity");
public static readonly Uri ProvGeneratedAtTime = new("http://www.w3.org/ns/prov#generatedAtTime");
public static readonly Uri ProvWasInvalidatedBy = new("http://www.w3.org/ns/prov#wasInvalidatedBy");
public static readonly Uri ProvWasRevisionOf = new("http://www.w3.org/ns/prov#wasRevisionOf");
// RDF
public static readonly Uri RdfUrlPrefix = new("http://www.w3.org/1999/02/22-rdf-syntax-ns#");
public static readonly Uri A = new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type");
// RDFS
public static readonly Uri RdfsUrlPrefix = new("http://www.w3.org/2000/01/rdf-schema#");
public static readonly Uri SubClassOf = new("http://www.w3.org/2000/01/rdf-schema#subClassOf");
// ROR IDs
/// <summary>
/// Represents the RWTH Aachen's ROR (Research Organization Registry) ID.
/// </summary>
public static readonly Uri RwthRorId = new("https://ror.org/04xfq0f34");
// SCHEMA
public static readonly Uri SchemaPrefix = new("http://schema.org/");
public static readonly Uri SchemaFunding = new("http://schema.org/funding");
// SHACL
public static readonly Uri ShaclPrefix = new("http://www.w3.org/ns/shacl#");
public static readonly Uri ShaclDatatype = new("http://www.w3.org/ns/shacl#datatype");
public static readonly Uri ShaclPath = new("http://www.w3.org/ns/shacl#path");
public static readonly Uri ShaclProperty = new("http://www.w3.org/ns/shacl#property");
public static readonly Uri ShaclTargetClass = new("http://www.w3.org/ns/shacl#targetClass");
public static readonly Uri Class = new("http://www.w3.org/ns/shacl#class");
// Trellis
public static readonly Uri TrellisPrefix = new("http://www.trellisldp.org/ns/trellis#");
public static readonly Uri TrellisGraph = new("http://www.trellisldp.org/ns/trellis#PreferServerManaged");
// Vcard
public static readonly Uri VcardPrefix = new("http://www.w3.org/2006/vcard/ns#");
public static readonly Uri VcardHasMember = new("http://www.w3.org/2006/vcard/ns#hasMember");
public static readonly Uri VcardGroupClass = new("http://www.w3.org/2006/vcard/ns#Group");
// XSD
public static readonly Uri XsdPrefix = new("http://www.w3.org/2001/XMLSchema#");
public static readonly Uri XsdDateTime = new("http://www.w3.org/2001/XMLSchema#dateTime");
public static readonly Uri XsdBoolean = new("http://www.w3.org/2001/XMLSchema#boolean");
public static readonly Uri XsdHexBinary = new("http://www.w3.org/2001/XMLSchema#hexBinary");
}
\ No newline at end of file
namespace SQL2Linked.Utils;
/// <summary>
///
/// </summary>
public static class UriHelper
{
/// <summary>
///
/// </summary>
/// <param name="baseUri"></param>
/// <param name="relativePath"></param>
/// <returns></returns>
public static Uri? TryCombineUri(Uri baseUri, string relativePath)
{
Uri.TryCreate(baseUri, relativePath, out Uri? result);
return result;
}
/// <summary>
///
/// </summary>
/// <param name="baseUri"></param>
/// <param name="relativePath"></param>
/// <returns></returns>
public static Uri? TryCombineUri(Uri baseUri, Guid relativePath)
{
Uri.TryCreate(baseUri, relativePath.ToString(), out Uri? result);
return result;
}
}
using VDS.RDF;
namespace SQL2Linked
{
public class WrapperGraph : Graph
{
public List<Triple> AssertList { get; set; }
public List<Triple> RetractList { get; set; }
public WrapperGraph() : base()
{
AssertList = new List<Triple>();
RetractList = new List<Triple>();
}
public override bool Assert(Triple t)
{
AssertList.Add(t);
return base.Assert(t);
}
public override bool Assert(IEnumerable<Triple> triples)
{
AssertList.AddRange(triples);
return base.Assert(triples);
}
public override bool Retract(Triple t)
{
RetractList.Add(t);
return base.Retract(t);
}
public override bool Retract(IEnumerable<Triple> triples)
{
RetractList.AddRange(triples);
return base.Retract(triples);
}
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment