Skip to content
Snippets Groups Projects
Commit 90cf2319 authored by L. Ellenbeck's avatar L. Ellenbeck
Browse files

Merge branch 'Issue/2847-reporting' into 'main'

Update: Now using IAsyncEnumerable to yield results

See merge request !30
parents 097c9baa 9c18d0af
Branches
Tags
1 merge request!30Update: Now using IAsyncEnumerable to yield results
Pipeline #1301835 passed
......@@ -12,27 +12,12 @@ namespace SQL2Linked.Implementations;
/// </summary>
public class ProjectStructuralData : StructuralData<ProjectAdminDto>
{
/// <summary>
/// Asynchronously retrieves all project data, including deleted projects.
/// </summary>
/// <returns>A <see cref="Task"/> that represents the asynchronous operation and returns a collection of <see cref="ProjectAdminDto"/>.</returns>
/// <remarks>This override allows for the inclusion of deleted projects.</remarks>
public override async Task<IEnumerable<ProjectAdminDto>> GetAll()
{
return await RequestUtil.WrapPagedRequest<ProjectAdminDtoPagedResponse, ProjectAdminDto>(
(currentPage) => _adminApi.GetAllProjectsAsync(includeDeleted: true, pageNumber: currentPage, pageSize: 250)
);
}
public override IAsyncEnumerable<ProjectAdminDto> GetAll() =>
PaginationHelper.GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>(
(currentPage) => _adminApi.GetAllProjectsAsync(includeDeleted: true, pageNumber: currentPage, pageSize: 50));
/// <summary>
/// Converts a collection of project data entries into a set of RDF graphs.
/// Each project is transformed into a graph, with RDF triples representing various properties of the project.
/// </summary>
/// <param name="entries">A collection of <see cref="ProjectAdminDto"/> instances representing project data.</param>
/// <returns>A collection of <see cref="IGraph"/> instances, each representing an RDF graph of a project.</returns>
public override async Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<ProjectAdminDto> entries)
public override async IAsyncEnumerable<IGraph> ConvertToLinkedDataAsync(IAsyncEnumerable<ProjectAdminDto> entries)
{
var graphs = new List<IGraph>();
var coscineHandlePrefix = UriHelper.TryCombinePath(RdfUris.HandlePrefix, _pidConfiguration.Prefix)
?? throw new Exception("Could not combine handle prefix with PID prefix");
......@@ -42,9 +27,10 @@ public class ProjectStructuralData : StructuralData<ProjectAdminDto>
};
AssertToGraphUriNode(coscineGraph, RdfUris.CoscinePrefix, RdfUris.DcatCatalog, RdfUris.CoscineProjects);
AssertToGraphUriNode(coscineGraph, RdfUris.CoscinePrefix, RdfUris.DcatCatalog, RdfUris.CoscineResources);
graphs.Add(coscineGraph);
foreach (var entry in entries)
yield return coscineGraph; // yeld coscineGraph first
await foreach (var entry in entries)
{
var projectGraphName = UriHelper.TryCombineUri(RdfUris.CoscineProjects, entry.Id)
?? throw new Exception("Could not combine projects prefix with project ID");
......@@ -159,8 +145,7 @@ public class ProjectStructuralData : StructuralData<ProjectAdminDto>
Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {RdfUris.DcTermsCreated} {entry.CreationDate}'. ");
}
graphs.Add(graph);
yield return graph;
}
return await Task.FromResult(graphs);
}
}
\ No newline at end of file
......@@ -14,31 +14,16 @@ namespace SQL2Linked.Implementations;
/// </summary>
public class ResourceStructuralData : StructuralData<ResourceAdminDto>
{
/// <summary>
/// Asynchronously retrieves all resource data, including deleted resources.
/// </summary>
/// <returns>A <see cref="Task"/> that represents the asynchronous operation and returns a collection of <see cref="ResourceAdminDto"/>.</returns>
/// <remarks>This override allows for the inclusion of deleted resources.</remarks>
public override async Task<IEnumerable<ResourceAdminDto>> GetAll()
{
return await RequestUtil.WrapPagedRequest<ResourceAdminDtoPagedResponse, ResourceAdminDto>(
(currentPage) => _adminApi.GetAllResourcesAsync(includeDeleted: true, pageNumber: currentPage, pageSize: 250)
);
}
public override IAsyncEnumerable<ResourceAdminDto> GetAll() =>
PaginationHelper.GetAllAsync<ResourceAdminDtoPagedResponse, ResourceAdminDto>(
(currentPage) => _adminApi.GetAllResourcesAsync(includeDeleted: true, pageNumber: currentPage, pageSize: 50));
/// <summary>
/// Converts a collection of resource data entries into a set of RDF graphs.
/// Each resource is transformed into a graph, with RDF triples representing various properties of the resource.
/// </summary>
/// <param name="entries">A collection of <see cref="ResourceAdminDto"/> instances representing resource data.</param>
/// <returns>A collection of <see cref="IGraph"/> instances, each representing an RDF graph of a resource.</returns>
public override async Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<ResourceAdminDto> entries)
public override async IAsyncEnumerable<IGraph> ConvertToLinkedDataAsync(IAsyncEnumerable<ResourceAdminDto> entries)
{
var graphs = new List<IGraph>();
var coscineHandlePrefix = UriHelper.TryCombinePath(RdfUris.HandlePrefix, _pidConfiguration.Prefix)
?? throw new Exception("Could not combine handle prefix with PID prefix");
foreach (var entry in entries)
await foreach (var entry in entries)
{
var resourceGraphName = UriHelper.TryCombineUri(RdfUris.CoscineResources, entry.Id)
?? throw new Exception("Could not combine resources prefix with resource ID");
......@@ -180,9 +165,7 @@ public class ResourceStructuralData : StructuralData<ResourceAdminDto>
Console.WriteLine($"For resource '{entry.DisplayName}' will migrate triple '{resourceGraphName} {RdfUris.DcTermsCreated} {entry.DateCreated}'. ");
}
graphs.Add(graph);
yield return graph;
}
return await Task.FromResult(graphs);
}
}
......@@ -13,28 +13,19 @@ namespace SQL2Linked.Implementations;
/// </summary>
public class ResourceTypeStructuralData : StructuralData<ResourceTypeInformationDto>
{
/// <summary>
/// Asynchronously retrieves all resource type data.
/// </summary>
/// <returns>A <see cref="Task"/> that represents the asynchronous operation and returns a collection of <see cref="ResourceTypeInformationDto"/>.</returns>
public override async Task<IEnumerable<ResourceTypeInformationDto>> GetAll()
public override async IAsyncEnumerable<ResourceTypeInformationDto> GetAll()
{
var resourceTypeApi = new ResourceTypeApi(_apiConfiguration);
var resourceTypeInformationsResponse = await resourceTypeApi.GetAllResourceTypesInformationAsync();
return resourceTypeInformationsResponse.Data;
foreach (var item in resourceTypeInformationsResponse.Data)
{
yield return item; // Yield each item in the list to keep the IAsyncEnumerable logic
}
}
/// <summary>
/// Converts a collection of resource type data entries into a set of RDF graphs.
/// Each resource type is transformed into a graph, with RDF triples representing various properties of the resource type.
/// </summary>
/// <param name="entries">A collection of <see cref="ResourceTypeInformationDto"/> instances representing resource type data.</param>
/// <returns>A collection of <see cref="IGraph"/> instances, each representing an RDF graph of a resource type.</returns>
public override async Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<ResourceTypeInformationDto> entries)
public override async IAsyncEnumerable<IGraph> ConvertToLinkedDataAsync(IAsyncEnumerable<ResourceTypeInformationDto> entries)
{
var graphs = new List<IGraph>();
foreach (var entry in entries)
await foreach (var entry in entries)
{
var resourceTypeGraphName = UriHelper.TryCombineUri(RdfUris.CoscineResourceTypes, entry.Id)
?? throw new Exception("Could not combine resource types prefix with resource type ID");
......@@ -89,9 +80,8 @@ public class ResourceTypeStructuralData : StructuralData<ResourceTypeInformation
if (!getTriplesDcatDataService.Any() || !getTriplesDctermsTitle.Any())
{
graphs.Add(graph);
yield return graph;
}
}
return await Task.FromResult(graphs);
}
}
\ No newline at end of file
......@@ -14,29 +14,16 @@ namespace SQL2Linked.Implementations;
/// </summary>
public class RoleStructuralData : StructuralData<RoleDto>
{
/// <summary>
/// Asynchronously retrieves all role data.
/// </summary>
/// <returns>A <see cref="Task"/> that represents the asynchronous operation and returns a collection of <see cref="RoleDto"/>.</returns>
public override async Task<IEnumerable<RoleDto>> GetAll()
public override IAsyncEnumerable<RoleDto> GetAll()
{
var roleApi = new RoleApi(_apiConfiguration);
return await RequestUtil.WrapPagedRequest<RoleDtoPagedResponse, RoleDto>(
(currentPage) => roleApi.GetRolesAsync(pageNumber: currentPage)
);
return PaginationHelper.GetAllAsync<RoleDtoPagedResponse, RoleDto>(
(currentPage) => roleApi.GetRolesAsync(pageNumber: currentPage));
}
/// <summary>
/// Converts a collection of role data entries into a set of RDF graphs.
/// Each role is transformed into a graph, with RDF triples representing various properties of the role.
/// </summary>
/// <param name="entries">A collection of <see cref="RoleDto"/> instances representing role data.</param>
/// <returns>A collection of <see cref="IGraph"/> instances, each representing an RDF graph of a role.</returns>
public override async Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<RoleDto> entries)
public override async IAsyncEnumerable<IGraph> ConvertToLinkedDataAsync(IAsyncEnumerable<RoleDto> entries)
{
var graphs = new List<IGraph>();
foreach (var entry in entries)
await foreach (var entry in entries)
{
var roleGraphName = UriHelper.TryCombineUri(RdfUris.CoscineRoles, entry.Id)
?? throw new Exception("Could not combine role prefix with role ID");
......@@ -76,9 +63,8 @@ public class RoleStructuralData : StructuralData<RoleDto>
}
if (!getTriplesOrgRole.Any() || !getTriplesDctermsTitle.Any())
{
graphs.Add(graph);
yield return graph;
}
}
return graphs;
}
}
\ No newline at end of file
......@@ -13,29 +13,15 @@ namespace SQL2Linked.Implementations;
/// </summary>
public class UserStructuralData : StructuralData<UserDto>
{
/// <summary>
/// Asynchronously retrieves all user data only for users who have accepted the ToS.
/// </summary>
/// <returns>A <see cref="Task"/> that represents the asynchronous operation and returns a collection of <see cref="UserDto"/>.</returns>
/// <remarks>This override allows for the retrieval of users who have accepted the ToS.</remarks>
public override async Task<IEnumerable<UserDto>> GetAll()
public override IAsyncEnumerable<UserDto> GetAll()
{
return await RequestUtil.WrapPagedRequest<UserDtoPagedResponse, UserDto>(
(currentPage) => _adminApi.GetAllUsersAsync(tosAccepted: true, pageNumber: currentPage, pageSize: 250)
);
return PaginationHelper.GetAllAsync<UserDtoPagedResponse, UserDto>(
(currentPage) => _adminApi.GetAllUsersAsync(tosAccepted: true, pageNumber: currentPage, pageSize: 50));
}
/// <summary>
/// Converts a collection of user data entries into a set of RDF graphs.
/// Each user is transformed into a graph, with RDF triples representing various properties of the user.
/// </summary>
/// <param name="entries">A collection of <see cref="UserDto"/> instances representing user data.</param>
/// <returns>A collection of <see cref="IGraph"/> instances, each representing an RDF graph of a user.</returns>
public override async Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<UserDto> entries)
public override async IAsyncEnumerable<IGraph> ConvertToLinkedDataAsync(IAsyncEnumerable<UserDto> entries)
{
var graphs = new List<IGraph>();
foreach (var entry in entries)
await foreach (var entry in entries)
{
var userGraphName = UriHelper.TryCombineUri(RdfUris.CoscineUsers, entry.Id)
?? throw new Exception("Could not combine users prefix with user ID");
......@@ -61,15 +47,14 @@ public class UserStructuralData : StructuralData<UserDto>
graph.Retract(getTriplesName);
AssertToGraphLiteralNode(graph, userGraphName, RdfUris.FoafName, entry.DisplayName);
graphs.Add(graph);
Console.WriteLine($"Will migrate user '{entry.DisplayName}' with id '{entry.Id}'.");
yield return graph;
}
else
{
Console.WriteLine($"Will NOT migrate user '{entry.DisplayName}' with id '{entry.Id}'.");
}
}
return await Task.FromResult(graphs);
}
}
......@@ -16,10 +16,11 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Coscine.ApiClient" Version="1.3.1" />
<PackageReference Include="Coscine.ApiClient" Version="1.6.0-issue-2847-repor0007" />
<PackageReference Include="dotNetRdf" Version="3.1.1" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Options" Version="8.0.1" />
<PackageReference Include="Microsoft.Extensions.Options" Version="8.0.2" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
</ItemGroup>
</Project>
......@@ -42,33 +42,36 @@ public abstract class StructuralData<S>
BasePath = "http://localhost:7206/coscine",
ApiKeyPrefix = { { "Authorization", "Bearer" } },
ApiKey = { { "Authorization", _adminToken } },
Timeout = 300000, // 5 minutes
};
_adminApi = new AdminApi(_apiConfiguration);
}
/// <summary>
/// Converts the given entries to linked data graphs.
/// Asynchronously converts a collection of entries into linked data graphs.
/// </summary>
/// <param name="entries">The entries to convert.</param>
/// <returns>An enumerable collection of graphs representing the linked data.</returns>
public abstract Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<S> entries);
/// <param name="entries">A collection of entries of type <typeparamref name="S"/> to convert.</param>
/// <typeparam name="S">The type of the entries to be converted into linked data graphs.</typeparam>
/// <returns>An asynchronous enumerable (<see cref="IAsyncEnumerable{IGraph}"/>) of graphs, where each graph represents the linked data derived from an entry.</returns>
public abstract IAsyncEnumerable<IGraph> ConvertToLinkedDataAsync(IAsyncEnumerable<S> entries);
/// <summary>
/// Retrieves all entries of type <typeparamref name="S"/>.
/// Asynchronously retrieves all entries of type <typeparamref name="S"/>.
/// </summary>
/// <returns>A task that represents the asynchronous operation. The task result contains an enumerable of all entries.</returns>
public abstract Task<IEnumerable<S>> GetAll();
/// <typeparam name="S">The type of the entries to retrieve.</typeparam>
/// <returns>An <see cref="IAsyncEnumerable{S}"/> representing an asynchronous stream of entries.</returns>
public abstract IAsyncEnumerable<S> GetAll();
/// <summary>
/// Migrates the data, optionally in a dummy mode where changes are not persisted.
/// Asynchronously migrates data, with an option to simulate the migration process without making any changes (dummy mode).
/// </summary>
/// <param name="dummyMode">If set to <c>true</c>, the migration is simulated but not executed.</param>
/// <returns>A task representing the asynchronous operation.</returns>
/// <param name="dummyMode">If set to <c>true</c>, the method simulates the migration process without executing any changes to the data store.</param>
/// <returns>A task representing the asynchronous migration operation.</returns>
public async Task Migrate(bool dummyMode)
{
var spacer = new string('-', 35);
Console.WriteLine($"\n{spacer}\n{typeof(S).Name}\n{spacer}");
var graphs = await ConvertToLinkedDataAsync(await GetAll());
var graphs = ConvertToLinkedDataAsync(GetAll());
if (!dummyMode)
{
await StoreGraphs(graphs);
......@@ -76,16 +79,16 @@ public abstract class StructuralData<S>
}
/// <summary>
/// Stores the given graphs in the underlying data store.
/// Asynchronously stores a collection of RDF graphs in the underlying data store.
/// </summary>
/// <param name="graphs">The graphs to be stored.</param>
/// <returns>A task representing the asynchronous operation.</returns>
public async Task StoreGraphs(IEnumerable<IGraph> graphs)
/// <param name="graphs">An asynchronous enumerable of RDF graphs (<see cref="IAsyncEnumerable{IGraph}"/>) to be stored.</param>
/// <returns>A task representing the asynchronous operation of storing the graphs.</returns>
public async Task StoreGraphs(IAsyncEnumerable<IGraph> graphs)
{
var formatEnum = RdfFormat.TextTurtle;
var format = "text/turtle";
foreach (var graph in graphs)
await foreach (var graph in graphs)
{
Console.WriteLine($" ({graph.BaseUri})");
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment