Skip to content
Snippets Groups Projects
Commit c10492a1 authored by Sirieam Marie Hunke's avatar Sirieam Marie Hunke
Browse files

Cleaning

parent 883ab7b6
Branches
No related tags found
1 merge request!26New: MigrateSQL2Linked to GD
using System.Globalization; using Coscine.ApiClient;
using Coscine.ApiClient;
using Coscine.ApiClient.Core.Api; using Coscine.ApiClient.Core.Api;
using Coscine.ApiClient.Core.Model; using Coscine.ApiClient.Core.Model;
using Coscine.GraphDeployer.Models.ConfigurationModels; using Coscine.GraphDeployer.Models.ConfigurationModels;
...@@ -9,7 +8,6 @@ using Microsoft.Extensions.Logging; ...@@ -9,7 +8,6 @@ using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using Polly; using Polly;
using VDS.RDF; using VDS.RDF;
using VDS.RDF.Parsing;
using static Coscine.GraphDeployer.Utils.CommandLineOptions; using static Coscine.GraphDeployer.Utils.CommandLineOptions;
using Configuration = Coscine.ApiClient.Core.Client.Configuration; using Configuration = Coscine.ApiClient.Core.Client.Configuration;
...@@ -22,8 +20,11 @@ public class Deployer ...@@ -22,8 +20,11 @@ public class Deployer
private readonly AdminApi _adminApi; private readonly AdminApi _adminApi;
private readonly ResourceTypeApi _resourceTypeApi; private readonly ResourceTypeApi _resourceTypeApi;
private readonly RoleApi _roleApi; private readonly RoleApi _roleApi;
private readonly RoleStructuralData _roleStructuralData;
public Deployer(ILogger<Deployer> logger, IOptionsMonitor<GraphDeployerConfiguration> graphDeployerConfiguration) private readonly ResourceTypeStructuralData _resourceTypeStructuralData;
public Deployer(ILogger<Deployer> logger, IOptionsMonitor<GraphDeployerConfiguration> graphDeployerConfiguration, RoleStructuralData roleStructuralData, ResourceTypeStructuralData resourceTypeStructuralData)
{ {
_logger = logger; _logger = logger;
_graphDeployerConfiguration = graphDeployerConfiguration.CurrentValue; _graphDeployerConfiguration = graphDeployerConfiguration.CurrentValue;
...@@ -46,6 +47,8 @@ public class Deployer ...@@ -46,6 +47,8 @@ public class Deployer
_adminApi = new(apiClientConfig); _adminApi = new(apiClientConfig);
_resourceTypeApi = new ResourceTypeApi(apiClientConfig); _resourceTypeApi = new ResourceTypeApi(apiClientConfig);
_roleApi = new RoleApi(apiClientConfig); _roleApi = new RoleApi(apiClientConfig);
_roleStructuralData = roleStructuralData;
_resourceTypeStructuralData = resourceTypeStructuralData;
} }
public static string WorkingFolder { get; set; } = "./output/"; public static string WorkingFolder { get; set; } = "./output/";
...@@ -93,11 +96,7 @@ public class Deployer ...@@ -93,11 +96,7 @@ public class Deployer
(currentPage) => _roleApi.GetRolesAsync(pageNumber: currentPage)); (currentPage) => _roleApi.GetRolesAsync(pageNumber: currentPage));
var resourceTypeInformationsResponse = await _resourceTypeApi.GetAllResourceTypesInformationAsync(); var resourceTypeInformationsResponse = await _resourceTypeApi.GetAllResourceTypesInformationAsync();
var resourceTypeList = new List<ResourceTypeInformationDto>(); var resourceTypeInformations = resourceTypeInformationsResponse.Data.ToAsyncEnumerable();
foreach (var item in resourceTypeInformationsResponse.Data)
{
resourceTypeList.Add(item);
}
// Iterate over the repositories and deploy the graphs // Iterate over the repositories and deploy the graphs
foreach (var graphRepo in _graphDeployerConfiguration.GitLab.Repositories) foreach (var graphRepo in _graphDeployerConfiguration.GitLab.Repositories)
...@@ -194,8 +193,15 @@ public class Deployer ...@@ -194,8 +193,15 @@ public class Deployer
// Clean up the working folder // Clean up the working folder
EmptyWorkingFolder(); EmptyWorkingFolder();
} }
_logger.LogInformation("\nBegin SQL 2 Linked Data migration");
// Convert to linked data // Convert to linked data
// Store the graph var roleStructuralGraphs = _roleStructuralData.ConvertToLinkedDataAsync(roleResponse);
var resourceTypeStructuralGraphs = _resourceTypeStructuralData.ConvertToLinkedDataAsync(resourceTypeInformations);
// Add all graphs
var structuralGraphs = roleStructuralGraphs.Concat(resourceTypeStructuralGraphs);
// Store the graphs
await StoreGraphs(structuralGraphs);
Console.WriteLine("\n Finished.");
return true; return true;
} }
...@@ -341,81 +347,4 @@ public class Deployer ...@@ -341,81 +347,4 @@ public class Deployer
} }
} }
} }
// TODO: Move to own class
public void AddModifiedDate(IGraph graph, Uri rootUri)
{
AssertToGraphLiteralNode(
graph,
rootUri,
RdfUris.DcTermsModified,
DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture),
new Uri(XmlSpecsHelper.XmlSchemaDataTypeDateTime)
);
}
/// <summary>
/// Asserts a triple to the graph with a URI node object.
/// </summary>
/// <param name="graph">The graph to assert to.</param>
/// <param name="graphSubject">The subject URI of the triple.</param>
/// <param name="graphPredicate">The predicate URI of the triple.</param>
/// <param name="graphObject">The object URI of the triple.</param>
public void AssertToGraphUriNode(IGraph graph, Uri graphSubject, Uri graphPredicate, Uri? graphObject)
{
if (graphObject != null)
{
graph.Assert(
new Triple(
graph.CreateUriNode(graphSubject),
graph.CreateUriNode(graphPredicate),
graph.CreateUriNode(graphObject)
)
);
}
}
/// <summary>
/// Asserts a triple to the graph with a literal node object.
/// </summary>
/// <param name="graph">The graph to assert to.</param>
/// <param name="graphSubject">The subject URI of the triple.</param>
/// <param name="graphPredicate">The predicate URI of the triple.</param>
/// <param name="graphObject">The literal object of the triple.</param>
/// <param name="objectType">The data type URI of the literal object, if any.</param>
public void AssertToGraphLiteralNode(IGraph graph, Uri graphSubject, Uri graphPredicate, string? graphObject, Uri? objectType = null)
{
if (graphObject != null)
{
graph.Assert(
new Triple(
graph.CreateUriNode(graphSubject),
graph.CreateUriNode(graphPredicate),
objectType is not null ? graph.CreateLiteralNode(graphObject, objectType) : graph.CreateLiteralNode(graphObject)
)
);
}
}
/// <summary>
/// Asserts a triple to the graph with a blank node subject and a URI node object.
/// </summary>
/// <param name="graph">The graph to assert to.</param>
/// <param name="graphSubject">The blank node subject of the triple.</param>
/// <param name="graphPredicate">The predicate URI of the triple.</param>
/// <param name="graphObject">The object URI of the triple.</param>
public void AssertToGraphBlankAndUriNode(IGraph graph, IBlankNode graphSubject, Uri graphPredicate, Uri? graphObject)
{
if (graphObject != null)
{
graph.Assert(
new Triple(
graphSubject,
graph.CreateUriNode(graphPredicate),
graph.CreateUriNode(graphObject)
)
);
}
}
} }
\ No newline at end of file
...@@ -11,13 +11,10 @@ namespace Coscine.GraphDeployer; ...@@ -11,13 +11,10 @@ namespace Coscine.GraphDeployer;
/// </summary> /// </summary>
public class ResourceTypeStructuralData public class ResourceTypeStructuralData
{ {
private readonly Deployer _deployer;
private readonly AdminApi _adminApi; private readonly AdminApi _adminApi;
public ResourceTypeStructuralData(AdminApi adminApi, Deployer deployer) public ResourceTypeStructuralData(AdminApi adminApi)
{ {
_adminApi = adminApi; _adminApi = adminApi;
_deployer = deployer;
} }
public async IAsyncEnumerable<IGraph> ConvertToLinkedDataAsync(IAsyncEnumerable<ResourceTypeInformationDto> entries) public async IAsyncEnumerable<IGraph> ConvertToLinkedDataAsync(IAsyncEnumerable<ResourceTypeInformationDto> entries)
{ {
...@@ -39,7 +36,7 @@ public class ResourceTypeStructuralData ...@@ -39,7 +36,7 @@ public class ResourceTypeStructuralData
if (!getTriplesDcatDataService.Any()) if (!getTriplesDcatDataService.Any())
{ {
_deployer.AssertToGraphUriNode(graph, resourceTypeGraphName, RdfUris.A, RdfUris.DcatDataServiceClass); GraphExtensions.AssertToGraph(graph, resourceTypeGraphName, RdfUris.A, RdfUris.DcatDataServiceClass);
Console.WriteLine($"For resource type '{entry.SpecificType}' will migrate triple '{graph.BaseUri} {RdfUris.A} {RdfUris.DcatDataServiceClass}'. "); Console.WriteLine($"For resource type '{entry.SpecificType}' will migrate triple '{graph.BaseUri} {RdfUris.A} {RdfUris.DcatDataServiceClass}'. ");
} }
else else
...@@ -52,7 +49,7 @@ public class ResourceTypeStructuralData ...@@ -52,7 +49,7 @@ public class ResourceTypeStructuralData
if (!getTriplesDctermsTitle.Any()) if (!getTriplesDctermsTitle.Any())
{ {
_deployer.AssertToGraphLiteralNode(graph, resourceTypeGraphName, RdfUris.DcTermsTitle, entry.SpecificType); GraphExtensions.AssertToGraph(graph, resourceTypeGraphName, RdfUris.DcTermsTitle, entry.SpecificType);
Console.WriteLine($"For resource type '{entry.SpecificType}' will migrate triple '{graph.BaseUri} {RdfUris.DcTermsTitle} {entry.SpecificType}'. "); Console.WriteLine($"For resource type '{entry.SpecificType}' will migrate triple '{graph.BaseUri} {RdfUris.DcTermsTitle} {entry.SpecificType}'. ");
} }
else else
...@@ -66,7 +63,7 @@ public class ResourceTypeStructuralData ...@@ -66,7 +63,7 @@ public class ResourceTypeStructuralData
if (!getTriplesDctermsAlternative.Any()) if (!getTriplesDctermsAlternative.Any())
{ {
_deployer.AssertToGraphLiteralNode(graph, resourceTypeGraphName, RdfUris.DcTermsAlternative, entry.GeneralType); GraphExtensions.AssertToGraph(graph, resourceTypeGraphName, RdfUris.DcTermsAlternative, entry.GeneralType);
Console.WriteLine($"For resource type '{entry.SpecificType}' will migrate triple '{graph.BaseUri} {RdfUris.DcTermsAlternative} {entry.GeneralType}'. "); Console.WriteLine($"For resource type '{entry.SpecificType}' will migrate triple '{graph.BaseUri} {RdfUris.DcTermsAlternative} {entry.GeneralType}'. ");
} }
else else
......
using Coscine.ApiClient;
using Coscine.ApiClient.Core.Api; using Coscine.ApiClient.Core.Api;
using Coscine.ApiClient.Core.Model; using Coscine.ApiClient.Core.Model;
using VDS.RDF; using VDS.RDF;
...@@ -14,11 +13,9 @@ namespace Coscine.GraphDeployer; ...@@ -14,11 +13,9 @@ namespace Coscine.GraphDeployer;
public class RoleStructuralData public class RoleStructuralData
{ {
private readonly AdminApi _adminApi; private readonly AdminApi _adminApi;
private readonly Deployer _deployer; public RoleStructuralData(AdminApi adminApi)
public RoleStructuralData(AdminApi adminApi, Deployer deployer)
{ {
_adminApi = adminApi; _adminApi = adminApi;
_deployer = deployer;
} }
public async IAsyncEnumerable<IGraph> ConvertToLinkedDataAsync(IAsyncEnumerable<RoleDto> entries) public async IAsyncEnumerable<IGraph> ConvertToLinkedDataAsync(IAsyncEnumerable<RoleDto> entries)
...@@ -41,7 +38,7 @@ public class RoleStructuralData ...@@ -41,7 +38,7 @@ public class RoleStructuralData
if (!getTriplesOrgRole.Any()) if (!getTriplesOrgRole.Any())
{ {
_deployer.AssertToGraphUriNode(graph, roleGraphName, RdfUris.A, RdfUris.OrgRoleClass); GraphExtensions.AssertToGraph(graph, roleGraphName, RdfUris.A, RdfUris.OrgRoleClass);
Console.WriteLine($"For role '{entry.DisplayName}' will migrate triple '{graph.BaseUri} {RdfUris.A} {RdfUris.OrgRoleClass}'. "); Console.WriteLine($"For role '{entry.DisplayName}' will migrate triple '{graph.BaseUri} {RdfUris.A} {RdfUris.OrgRoleClass}'. ");
} }
else else
...@@ -54,7 +51,7 @@ public class RoleStructuralData ...@@ -54,7 +51,7 @@ public class RoleStructuralData
if (!getTriplesDctermsTitle.Any()) if (!getTriplesDctermsTitle.Any())
{ {
_deployer.AssertToGraphLiteralNode(graph, roleGraphName, RdfUris.DcTermsTitle, entry.DisplayName); GraphExtensions.AssertToGraph(graph, roleGraphName, RdfUris.DcTermsTitle, entry.DisplayName);
Console.WriteLine($"For role '{entry.DisplayName}' will migrate triple '{graph.BaseUri} {RdfUris.DcTermsTitle} {entry.DisplayName}'. "); Console.WriteLine($"For role '{entry.DisplayName}' will migrate triple '{graph.BaseUri} {RdfUris.DcTermsTitle} {entry.DisplayName}'. ");
} }
else else
...@@ -67,5 +64,4 @@ public class RoleStructuralData ...@@ -67,5 +64,4 @@ public class RoleStructuralData
} }
} }
} }
} }
using System.Globalization;
using VDS.RDF;
using VDS.RDF.Parsing;
namespace Coscine.GraphDeployer;
/// <summary>
/// Utility functions to simplify interactions with graphs.
/// </summary>
public static class GraphExtensions
{
/// <summary>
/// Asserts the given triple using <see cref="string"/> parameters to the graph.
/// </summary>
/// <param name="graph">The graph to which the triple will be asserted.</param>
/// <param name="subject">The subject of the triple as a string URI.</param>
/// <param name="predicate">The predicate of the triple as a string URI.</param>
/// <param name="uriObject">The object of the triple as a URI.</param>
public static void AssertToGraph(this IGraph graph, string subject, string predicate, Uri uriObject)
{
AssertToGraph(graph, new Uri(subject), new Uri(predicate), uriObject);
}
/// <summary>
/// Asserts the given triple using <see cref="Uri"/> parameters to the graph.
/// </summary>
/// <param name="graph">The graph to which the triple will be asserted.</param>
/// <param name="subject">The subject of the triple as a URI.</param>
/// <param name="predicate">The predicate of the triple as a URI.</param>
/// <param name="uriObject">The object of the triple as a URI.</param>
public static void AssertToGraph(this IGraph graph, Uri subject, Uri predicate, Uri uriObject)
{
graph.Assert(
new Triple(
graph.CreateUriNode(subject),
graph.CreateUriNode(predicate),
graph.CreateUriNode(uriObject)
)
);
}
/// <summary>
/// Asserts the given triple using <see cref="Uri"/> parameters to the graph.
/// </summary>
/// <param name="graph">The graph to which the triple will be asserted.</param>
/// <param name="subject">The subject of the triple as a URI.</param>
/// <param name="predicate">The predicate of the triple as a URI.</param>
/// <param name="uriObject">The object of the triple as a INode.</param>
public static void AssertToGraph(this IGraph graph, Uri subject, Uri predicate, INode uriObject)
{
graph.Assert(
new Triple(
graph.CreateUriNode(subject),
graph.CreateUriNode(predicate),
uriObject
)
);
}
/// <summary>
/// Asserts the given triple using <see cref="Uri"/> parameters to the graph.
/// </summary>
/// <param name="graph">The graph to which the triple will be asserted.</param>
/// <param name="subject">The subject of the triple as a INode.</param>
/// <param name="predicate">The predicate of the triple as a URI.</param>
/// <param name="uriObject">The object of the triple as a Uri.</param>
public static void AssertToGraph(this IGraph graph, INode subject, Uri predicate, Uri uriObject)
{
graph.Assert(
new Triple(
subject,
graph.CreateUriNode(predicate),
graph.CreateUriNode(uriObject)
)
);
}
/// <summary>
/// Asserts the given triple using <see cref="Uri"/> parameters to the graph.
/// </summary>
/// <param name="graph">The graph to which the triple will be asserted.</param>
/// <param name="subject">The subject of the triple as a INode.</param>
/// <param name="predicate">The predicate of the triple as a URI.</param>
/// <param name="uriObject">The object of the triple as a INode.</param>
public static void AssertToGraph(this IGraph graph, INode subject, Uri predicate, INode uriObject)
{
graph.Assert(
new Triple(
subject,
graph.CreateUriNode(predicate),
uriObject
)
);
}
/// <summary>
/// Asserts the given triple using <see cref="string"/> parameters and a <i>literal object</i> to the graph.
/// </summary>
/// <param name="graph">The graph to which the triple will be asserted.</param>
/// <param name="subject">The subject of the triple as a string URI.</param>
/// <param name="predicate">The predicate of the triple as a string URI.</param>
/// <param name="literalObject">The object of the triple as a literal string.</param>
/// <param name="objectType">Optional URI that represents the datatype of the literal object.</param>
public static void AssertToGraph(this IGraph graph, string subject, string predicate, string literalObject, Uri? objectType = null)
{
AssertToGraph(graph, new Uri(subject), new Uri(predicate), literalObject, objectType);
}
/// <summary>
/// Asserts the given triple using <see cref="Uri"/> parameters and a <i>literal object</i> to the graph.
/// </summary>
/// <param name="graph">The graph to which the triple will be asserted.</param>
/// <param name="subject">The subject of the triple as a URI.</param>
/// <param name="predicate">The predicate of the triple as a URI.</param>
/// <param name="literalObject">The object of the triple as a literal string.</param>
/// <param name="objectType">Optional URI that represents the datatype of the literal object.</param>
public static void AssertToGraph(this IGraph graph, Uri subject, Uri predicate, string literalObject, Uri? objectType = null)
{
graph.Assert(
new Triple(
graph.CreateUriNode(subject),
graph.CreateUriNode(predicate),
objectType != null ? graph.CreateLiteralNode(literalObject, objectType) : graph.CreateLiteralNode(literalObject)
)
);
}
/// <summary>
/// Asserts the given triple using <see cref="Uri"/> parameters and a <i>literal object</i> to the graph.
/// </summary>
/// <param name="graph">The graph to which the triple will be asserted.</param>
/// <param name="subject">The subject of the triple as a INode.</param>
/// <param name="predicate">The predicate of the triple as a URI.</param>
/// <param name="literalObject">The object of the triple as a literal string.</param>
/// <param name="objectType">Optional URI that represents the datatype of the literal object.</param>
public static void AssertToGraph(this IGraph graph, INode subject, Uri predicate, string literalObject, Uri? objectType = null)
{
graph.Assert(
new Triple(
subject,
graph.CreateUriNode(predicate),
objectType != null ? graph.CreateLiteralNode(literalObject, objectType) : graph.CreateLiteralNode(literalObject)
)
);
}
/// <summary>
/// Asserts the modified date.
/// </summary>
/// <param name="graph">The graph to which the triple will be asserted.</param>
/// <param name="rootUri">The root URI.</param>
public static void AddModifiedDate(IGraph graph, Uri rootUri)
{
AssertToGraph(
graph,
rootUri,
RdfUris.DcTermsModified,
DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture),
new Uri(XmlSpecsHelper.XmlSchemaDataTypeDateTime)
);
}
}
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment