Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found
Select Git revision
Loading items

Target

Select target project
  • coscine/backend/scripts/sql2linked
1 result
Select Git revision
Loading items
Show changes
Commits on Source (3)
......@@ -12,27 +12,12 @@ namespace SQL2Linked.Implementations;
/// </summary>
public class ProjectStructuralData : StructuralData<ProjectAdminDto>
{
/// <summary>
/// Asynchronously retrieves all project data, including deleted projects.
/// </summary>
/// <returns>A <see cref="Task"/> that represents the asynchronous operation and returns a collection of <see cref="ProjectAdminDto"/>.</returns>
/// <remarks>This override allows for the inclusion of deleted projects.</remarks>
public override async Task<IEnumerable<ProjectAdminDto>> GetAll()
{
return await RequestUtil.WrapPagedRequest<ProjectAdminDtoPagedResponse, ProjectAdminDto>(
(currentPage) => _adminApi.GetAllProjectsAsync(includeDeleted: true, pageNumber: currentPage, pageSize: 250)
);
}
public override IAsyncEnumerable<ProjectAdminDto> GetAll() =>
PaginationHelper.GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>(
(currentPage) => _adminApi.GetAllProjectsAsync(includeDeleted: true, pageNumber: currentPage, pageSize: 50));
/// <summary>
/// Converts a collection of project data entries into a set of RDF graphs.
/// Each project is transformed into a graph, with RDF triples representing various properties of the project.
/// </summary>
/// <param name="entries">A collection of <see cref="ProjectAdminDto"/> instances representing project data.</param>
/// <returns>A collection of <see cref="IGraph"/> instances, each representing an RDF graph of a project.</returns>
public override async Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<ProjectAdminDto> entries)
public override async IAsyncEnumerable<IGraph> ConvertToLinkedDataAsync(IAsyncEnumerable<ProjectAdminDto> entries)
{
var graphs = new List<IGraph>();
var coscineHandlePrefix = UriHelper.TryCombinePath(RdfUris.HandlePrefix, _pidConfiguration.Prefix)
?? throw new Exception("Could not combine handle prefix with PID prefix");
......@@ -42,9 +27,10 @@ public class ProjectStructuralData : StructuralData<ProjectAdminDto>
};
AssertToGraphUriNode(coscineGraph, RdfUris.CoscinePrefix, RdfUris.DcatCatalog, RdfUris.CoscineProjects);
AssertToGraphUriNode(coscineGraph, RdfUris.CoscinePrefix, RdfUris.DcatCatalog, RdfUris.CoscineResources);
graphs.Add(coscineGraph);
foreach (var entry in entries)
yield return coscineGraph; // yeld coscineGraph first
await foreach (var entry in entries)
{
var projectGraphName = UriHelper.TryCombineUri(RdfUris.CoscineProjects, entry.Id)
?? throw new Exception("Could not combine projects prefix with project ID");
......@@ -159,8 +145,7 @@ public class ProjectStructuralData : StructuralData<ProjectAdminDto>
Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {RdfUris.DcTermsCreated} {entry.CreationDate}'. ");
}
graphs.Add(graph);
yield return graph;
}
return await Task.FromResult(graphs);
}
}
\ No newline at end of file
......@@ -14,31 +14,16 @@ namespace SQL2Linked.Implementations;
/// </summary>
public class ResourceStructuralData : StructuralData<ResourceAdminDto>
{
/// <summary>
/// Asynchronously retrieves all resource data, including deleted resources.
/// </summary>
/// <returns>A <see cref="Task"/> that represents the asynchronous operation and returns a collection of <see cref="ResourceAdminDto"/>.</returns>
/// <remarks>This override allows for the inclusion of deleted resources.</remarks>
public override async Task<IEnumerable<ResourceAdminDto>> GetAll()
{
return await RequestUtil.WrapPagedRequest<ResourceAdminDtoPagedResponse, ResourceAdminDto>(
(currentPage) => _adminApi.GetAllResourcesAsync(includeDeleted: true, pageNumber: currentPage, pageSize: 250)
);
}
public override IAsyncEnumerable<ResourceAdminDto> GetAll() =>
PaginationHelper.GetAllAsync<ResourceAdminDtoPagedResponse, ResourceAdminDto>(
(currentPage) => _adminApi.GetAllResourcesAsync(includeDeleted: true, pageNumber: currentPage, pageSize: 50));
/// <summary>
/// Converts a collection of resource data entries into a set of RDF graphs.
/// Each resource is transformed into a graph, with RDF triples representing various properties of the resource.
/// </summary>
/// <param name="entries">A collection of <see cref="ResourceAdminDto"/> instances representing resource data.</param>
/// <returns>A collection of <see cref="IGraph"/> instances, each representing an RDF graph of a resource.</returns>
public override async Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<ResourceAdminDto> entries)
public override async IAsyncEnumerable<IGraph> ConvertToLinkedDataAsync(IAsyncEnumerable<ResourceAdminDto> entries)
{
var graphs = new List<IGraph>();
var coscineHandlePrefix = UriHelper.TryCombinePath(RdfUris.HandlePrefix, _pidConfiguration.Prefix)
?? throw new Exception("Could not combine handle prefix with PID prefix");
foreach (var entry in entries)
await foreach (var entry in entries)
{
var resourceGraphName = UriHelper.TryCombineUri(RdfUris.CoscineResources, entry.Id)
?? throw new Exception("Could not combine resources prefix with resource ID");
......@@ -180,9 +165,7 @@ public class ResourceStructuralData : StructuralData<ResourceAdminDto>
Console.WriteLine($"For resource '{entry.DisplayName}' will migrate triple '{resourceGraphName} {RdfUris.DcTermsCreated} {entry.DateCreated}'. ");
}
graphs.Add(graph);
yield return graph;
}
return await Task.FromResult(graphs);
}
}
......@@ -13,28 +13,19 @@ namespace SQL2Linked.Implementations;
/// </summary>
public class ResourceTypeStructuralData : StructuralData<ResourceTypeInformationDto>
{
/// <summary>
/// Asynchronously retrieves all resource type data.
/// </summary>
/// <returns>A <see cref="Task"/> that represents the asynchronous operation and returns a collection of <see cref="ResourceTypeInformationDto"/>.</returns>
public override async Task<IEnumerable<ResourceTypeInformationDto>> GetAll()
public override async IAsyncEnumerable<ResourceTypeInformationDto> GetAll()
{
var resourceTypeApi = new ResourceTypeApi(_apiConfiguration);
var resourceTypeInformationsResponse = await resourceTypeApi.GetAllResourceTypesInformationAsync();
return resourceTypeInformationsResponse.Data;
foreach (var item in resourceTypeInformationsResponse.Data)
{
yield return item; // Yield each item in the list to keep the IAsyncEnumerable logic
}
}
/// <summary>
/// Converts a collection of resource type data entries into a set of RDF graphs.
/// Each resource type is transformed into a graph, with RDF triples representing various properties of the resource type.
/// </summary>
/// <param name="entries">A collection of <see cref="ResourceTypeInformationDto"/> instances representing resource type data.</param>
/// <returns>A collection of <see cref="IGraph"/> instances, each representing an RDF graph of a resource type.</returns>
public override async Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<ResourceTypeInformationDto> entries)
public override async IAsyncEnumerable<IGraph> ConvertToLinkedDataAsync(IAsyncEnumerable<ResourceTypeInformationDto> entries)
{
var graphs = new List<IGraph>();
foreach (var entry in entries)
await foreach (var entry in entries)
{
var resourceTypeGraphName = UriHelper.TryCombineUri(RdfUris.CoscineResourceTypes, entry.Id)
?? throw new Exception("Could not combine resource types prefix with resource type ID");
......@@ -89,9 +80,8 @@ public class ResourceTypeStructuralData : StructuralData<ResourceTypeInformation
if (!getTriplesDcatDataService.Any() || !getTriplesDctermsTitle.Any())
{
graphs.Add(graph);
yield return graph;
}
}
return await Task.FromResult(graphs);
}
}
\ No newline at end of file
......@@ -14,29 +14,16 @@ namespace SQL2Linked.Implementations;
/// </summary>
public class RoleStructuralData : StructuralData<RoleDto>
{
/// <summary>
/// Asynchronously retrieves all role data.
/// </summary>
/// <returns>A <see cref="Task"/> that represents the asynchronous operation and returns a collection of <see cref="RoleDto"/>.</returns>
public override async Task<IEnumerable<RoleDto>> GetAll()
public override IAsyncEnumerable<RoleDto> GetAll()
{
var roleApi = new RoleApi(_apiConfiguration);
return await RequestUtil.WrapPagedRequest<RoleDtoPagedResponse, RoleDto>(
(currentPage) => roleApi.GetRolesAsync(pageNumber: currentPage)
);
return PaginationHelper.GetAllAsync<RoleDtoPagedResponse, RoleDto>(
(currentPage) => roleApi.GetRolesAsync(pageNumber: currentPage));
}
/// <summary>
/// Converts a collection of role data entries into a set of RDF graphs.
/// Each role is transformed into a graph, with RDF triples representing various properties of the role.
/// </summary>
/// <param name="entries">A collection of <see cref="RoleDto"/> instances representing role data.</param>
/// <returns>A collection of <see cref="IGraph"/> instances, each representing an RDF graph of a role.</returns>
public override async Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<RoleDto> entries)
public override async IAsyncEnumerable<IGraph> ConvertToLinkedDataAsync(IAsyncEnumerable<RoleDto> entries)
{
var graphs = new List<IGraph>();
foreach (var entry in entries)
await foreach (var entry in entries)
{
var roleGraphName = UriHelper.TryCombineUri(RdfUris.CoscineRoles, entry.Id)
?? throw new Exception("Could not combine role prefix with role ID");
......@@ -76,9 +63,8 @@ public class RoleStructuralData : StructuralData<RoleDto>
}
if (!getTriplesOrgRole.Any() || !getTriplesDctermsTitle.Any())
{
graphs.Add(graph);
yield return graph;
}
}
return graphs;
}
}
\ No newline at end of file
......@@ -13,29 +13,15 @@ namespace SQL2Linked.Implementations;
/// </summary>
public class UserStructuralData : StructuralData<UserDto>
{
/// <summary>
/// Asynchronously retrieves all user data only for users who have accepted the ToS.
/// </summary>
/// <returns>A <see cref="Task"/> that represents the asynchronous operation and returns a collection of <see cref="UserDto"/>.</returns>
/// <remarks>This override allows for the retrieval of users who have accepted the ToS.</remarks>
public override async Task<IEnumerable<UserDto>> GetAll()
public override IAsyncEnumerable<UserDto> GetAll()
{
return await RequestUtil.WrapPagedRequest<UserDtoPagedResponse, UserDto>(
(currentPage) => _adminApi.GetAllUsersAsync(tosAccepted: true, pageNumber: currentPage, pageSize: 250)
);
return PaginationHelper.GetAllAsync<UserDtoPagedResponse, UserDto>(
(currentPage) => _adminApi.GetAllUsersAsync(tosAccepted: true, pageNumber: currentPage, pageSize: 50));
}
/// <summary>
/// Converts a collection of user data entries into a set of RDF graphs.
/// Each user is transformed into a graph, with RDF triples representing various properties of the user.
/// </summary>
/// <param name="entries">A collection of <see cref="UserDto"/> instances representing user data.</param>
/// <returns>A collection of <see cref="IGraph"/> instances, each representing an RDF graph of a user.</returns>
public override async Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<UserDto> entries)
public override async IAsyncEnumerable<IGraph> ConvertToLinkedDataAsync(IAsyncEnumerable<UserDto> entries)
{
var graphs = new List<IGraph>();
foreach (var entry in entries)
await foreach (var entry in entries)
{
var userGraphName = UriHelper.TryCombineUri(RdfUris.CoscineUsers, entry.Id)
?? throw new Exception("Could not combine users prefix with user ID");
......@@ -61,15 +47,14 @@ public class UserStructuralData : StructuralData<UserDto>
graph.Retract(getTriplesName);
AssertToGraphLiteralNode(graph, userGraphName, RdfUris.FoafName, entry.DisplayName);
graphs.Add(graph);
Console.WriteLine($"Will migrate user '{entry.DisplayName}' with id '{entry.Id}'.");
yield return graph;
}
else
{
Console.WriteLine($"Will NOT migrate user '{entry.DisplayName}' with id '{entry.Id}'.");
}
}
return await Task.FromResult(graphs);
}
}
......@@ -5,7 +5,7 @@
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<Version>0.1.21</Version>
<Version>0.1.22</Version>
</PropertyGroup>
<PropertyGroup>
......@@ -16,10 +16,11 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Coscine.ApiClient" Version="1.3.1" />
<PackageReference Include="Coscine.ApiClient" Version="1.6.0-issue-2847-repor0007" />
<PackageReference Include="dotNetRdf" Version="3.1.1" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Options" Version="8.0.1" />
<PackageReference Include="Microsoft.Extensions.Options" Version="8.0.2" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
</ItemGroup>
</Project>
......@@ -42,33 +42,36 @@ public abstract class StructuralData<S>
BasePath = "http://localhost:7206/coscine",
ApiKeyPrefix = { { "Authorization", "Bearer" } },
ApiKey = { { "Authorization", _adminToken } },
Timeout = 300000, // 5 minutes
};
_adminApi = new AdminApi(_apiConfiguration);
}
/// <summary>
/// Converts the given entries to linked data graphs.
/// Asynchronously converts a collection of entries into linked data graphs.
/// </summary>
/// <param name="entries">The entries to convert.</param>
/// <returns>An enumerable collection of graphs representing the linked data.</returns>
public abstract Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<S> entries);
/// <param name="entries">A collection of entries of type <typeparamref name="S"/> to convert.</param>
/// <typeparam name="S">The type of the entries to be converted into linked data graphs.</typeparam>
/// <returns>An asynchronous enumerable (<see cref="IAsyncEnumerable{IGraph}"/>) of graphs, where each graph represents the linked data derived from an entry.</returns>
public abstract IAsyncEnumerable<IGraph> ConvertToLinkedDataAsync(IAsyncEnumerable<S> entries);
/// <summary>
/// Retrieves all entries of type <typeparamref name="S"/>.
/// Asynchronously retrieves all entries of type <typeparamref name="S"/>.
/// </summary>
/// <returns>A task that represents the asynchronous operation. The task result contains an enumerable of all entries.</returns>
public abstract Task<IEnumerable<S>> GetAll();
/// <typeparam name="S">The type of the entries to retrieve.</typeparam>
/// <returns>An <see cref="IAsyncEnumerable{S}"/> representing an asynchronous stream of entries.</returns>
public abstract IAsyncEnumerable<S> GetAll();
/// <summary>
/// Migrates the data, optionally in a dummy mode where changes are not persisted.
/// Asynchronously migrates data, with an option to simulate the migration process without making any changes (dummy mode).
/// </summary>
/// <param name="dummyMode">If set to <c>true</c>, the migration is simulated but not executed.</param>
/// <returns>A task representing the asynchronous operation.</returns>
/// <param name="dummyMode">If set to <c>true</c>, the method simulates the migration process without executing any changes to the data store.</param>
/// <returns>A task representing the asynchronous migration operation.</returns>
public async Task Migrate(bool dummyMode)
{
var spacer = new string('-', 35);
Console.WriteLine($"\n{spacer}\n{typeof(S).Name}\n{spacer}");
var graphs = await ConvertToLinkedDataAsync(await GetAll());
var graphs = ConvertToLinkedDataAsync(GetAll());
if (!dummyMode)
{
await StoreGraphs(graphs);
......@@ -76,16 +79,16 @@ public abstract class StructuralData<S>
}
/// <summary>
/// Stores the given graphs in the underlying data store.
/// Asynchronously stores a collection of RDF graphs in the underlying data store.
/// </summary>
/// <param name="graphs">The graphs to be stored.</param>
/// <returns>A task representing the asynchronous operation.</returns>
public async Task StoreGraphs(IEnumerable<IGraph> graphs)
/// <param name="graphs">An asynchronous enumerable of RDF graphs (<see cref="IAsyncEnumerable{IGraph}"/>) to be stored.</param>
/// <returns>A task representing the asynchronous operation of storing the graphs.</returns>
public async Task StoreGraphs(IAsyncEnumerable<IGraph> graphs)
{
var formatEnum = RdfFormat.TextTurtle;
var format = "text/turtle";
foreach (var graph in graphs)
await foreach (var graph in graphs)
{
Console.WriteLine($" ({graph.BaseUri})");
......