Skip to content
Snippets Groups Projects
Commit 9c18d0af authored by Petar Hristov's avatar Petar Hristov :speech_balloon:
Browse files

Update: Now using IAsyncEnumerable to yield results

parent 097c9baa
No related branches found
No related tags found
1 merge request!30Update: Now using IAsyncEnumerable to yield results
Pipeline #1298787 passed
...@@ -12,27 +12,12 @@ namespace SQL2Linked.Implementations; ...@@ -12,27 +12,12 @@ namespace SQL2Linked.Implementations;
/// </summary> /// </summary>
public class ProjectStructuralData : StructuralData<ProjectAdminDto> public class ProjectStructuralData : StructuralData<ProjectAdminDto>
{ {
/// <summary> public override IAsyncEnumerable<ProjectAdminDto> GetAll() =>
/// Asynchronously retrieves all project data, including deleted projects. PaginationHelper.GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>(
/// </summary> (currentPage) => _adminApi.GetAllProjectsAsync(includeDeleted: true, pageNumber: currentPage, pageSize: 50));
/// <returns>A <see cref="Task"/> that represents the asynchronous operation and returns a collection of <see cref="ProjectAdminDto"/>.</returns>
/// <remarks>This override allows for the inclusion of deleted projects.</remarks>
public override async Task<IEnumerable<ProjectAdminDto>> GetAll()
{
return await RequestUtil.WrapPagedRequest<ProjectAdminDtoPagedResponse, ProjectAdminDto>(
(currentPage) => _adminApi.GetAllProjectsAsync(includeDeleted: true, pageNumber: currentPage, pageSize: 250)
);
}
/// <summary> public override async IAsyncEnumerable<IGraph> ConvertToLinkedDataAsync(IAsyncEnumerable<ProjectAdminDto> entries)
/// Converts a collection of project data entries into a set of RDF graphs.
/// Each project is transformed into a graph, with RDF triples representing various properties of the project.
/// </summary>
/// <param name="entries">A collection of <see cref="ProjectAdminDto"/> instances representing project data.</param>
/// <returns>A collection of <see cref="IGraph"/> instances, each representing an RDF graph of a project.</returns>
public override async Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<ProjectAdminDto> entries)
{ {
var graphs = new List<IGraph>();
var coscineHandlePrefix = UriHelper.TryCombinePath(RdfUris.HandlePrefix, _pidConfiguration.Prefix) var coscineHandlePrefix = UriHelper.TryCombinePath(RdfUris.HandlePrefix, _pidConfiguration.Prefix)
?? throw new Exception("Could not combine handle prefix with PID prefix"); ?? throw new Exception("Could not combine handle prefix with PID prefix");
...@@ -42,9 +27,10 @@ public class ProjectStructuralData : StructuralData<ProjectAdminDto> ...@@ -42,9 +27,10 @@ public class ProjectStructuralData : StructuralData<ProjectAdminDto>
}; };
AssertToGraphUriNode(coscineGraph, RdfUris.CoscinePrefix, RdfUris.DcatCatalog, RdfUris.CoscineProjects); AssertToGraphUriNode(coscineGraph, RdfUris.CoscinePrefix, RdfUris.DcatCatalog, RdfUris.CoscineProjects);
AssertToGraphUriNode(coscineGraph, RdfUris.CoscinePrefix, RdfUris.DcatCatalog, RdfUris.CoscineResources); AssertToGraphUriNode(coscineGraph, RdfUris.CoscinePrefix, RdfUris.DcatCatalog, RdfUris.CoscineResources);
graphs.Add(coscineGraph);
foreach (var entry in entries) yield return coscineGraph; // yeld coscineGraph first
await foreach (var entry in entries)
{ {
var projectGraphName = UriHelper.TryCombineUri(RdfUris.CoscineProjects, entry.Id) var projectGraphName = UriHelper.TryCombineUri(RdfUris.CoscineProjects, entry.Id)
?? throw new Exception("Could not combine projects prefix with project ID"); ?? throw new Exception("Could not combine projects prefix with project ID");
...@@ -159,8 +145,7 @@ public class ProjectStructuralData : StructuralData<ProjectAdminDto> ...@@ -159,8 +145,7 @@ public class ProjectStructuralData : StructuralData<ProjectAdminDto>
Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {RdfUris.DcTermsCreated} {entry.CreationDate}'. "); Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {RdfUris.DcTermsCreated} {entry.CreationDate}'. ");
} }
graphs.Add(graph); yield return graph;
} }
return await Task.FromResult(graphs);
} }
} }
\ No newline at end of file
...@@ -14,31 +14,16 @@ namespace SQL2Linked.Implementations; ...@@ -14,31 +14,16 @@ namespace SQL2Linked.Implementations;
/// </summary> /// </summary>
public class ResourceStructuralData : StructuralData<ResourceAdminDto> public class ResourceStructuralData : StructuralData<ResourceAdminDto>
{ {
/// <summary> public override IAsyncEnumerable<ResourceAdminDto> GetAll() =>
/// Asynchronously retrieves all resource data, including deleted resources. PaginationHelper.GetAllAsync<ResourceAdminDtoPagedResponse, ResourceAdminDto>(
/// </summary> (currentPage) => _adminApi.GetAllResourcesAsync(includeDeleted: true, pageNumber: currentPage, pageSize: 50));
/// <returns>A <see cref="Task"/> that represents the asynchronous operation and returns a collection of <see cref="ResourceAdminDto"/>.</returns>
/// <remarks>This override allows for the inclusion of deleted resources.</remarks>
public override async Task<IEnumerable<ResourceAdminDto>> GetAll()
{
return await RequestUtil.WrapPagedRequest<ResourceAdminDtoPagedResponse, ResourceAdminDto>(
(currentPage) => _adminApi.GetAllResourcesAsync(includeDeleted: true, pageNumber: currentPage, pageSize: 250)
);
}
/// <summary> public override async IAsyncEnumerable<IGraph> ConvertToLinkedDataAsync(IAsyncEnumerable<ResourceAdminDto> entries)
/// Converts a collection of resource data entries into a set of RDF graphs.
/// Each resource is transformed into a graph, with RDF triples representing various properties of the resource.
/// </summary>
/// <param name="entries">A collection of <see cref="ResourceAdminDto"/> instances representing resource data.</param>
/// <returns>A collection of <see cref="IGraph"/> instances, each representing an RDF graph of a resource.</returns>
public override async Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<ResourceAdminDto> entries)
{ {
var graphs = new List<IGraph>();
var coscineHandlePrefix = UriHelper.TryCombinePath(RdfUris.HandlePrefix, _pidConfiguration.Prefix) var coscineHandlePrefix = UriHelper.TryCombinePath(RdfUris.HandlePrefix, _pidConfiguration.Prefix)
?? throw new Exception("Could not combine handle prefix with PID prefix"); ?? throw new Exception("Could not combine handle prefix with PID prefix");
foreach (var entry in entries) await foreach (var entry in entries)
{ {
var resourceGraphName = UriHelper.TryCombineUri(RdfUris.CoscineResources, entry.Id) var resourceGraphName = UriHelper.TryCombineUri(RdfUris.CoscineResources, entry.Id)
?? throw new Exception("Could not combine resources prefix with resource ID"); ?? throw new Exception("Could not combine resources prefix with resource ID");
...@@ -180,9 +165,7 @@ public class ResourceStructuralData : StructuralData<ResourceAdminDto> ...@@ -180,9 +165,7 @@ public class ResourceStructuralData : StructuralData<ResourceAdminDto>
Console.WriteLine($"For resource '{entry.DisplayName}' will migrate triple '{resourceGraphName} {RdfUris.DcTermsCreated} {entry.DateCreated}'. "); Console.WriteLine($"For resource '{entry.DisplayName}' will migrate triple '{resourceGraphName} {RdfUris.DcTermsCreated} {entry.DateCreated}'. ");
} }
graphs.Add(graph); yield return graph;
} }
return await Task.FromResult(graphs);
} }
} }
...@@ -13,28 +13,19 @@ namespace SQL2Linked.Implementations; ...@@ -13,28 +13,19 @@ namespace SQL2Linked.Implementations;
/// </summary> /// </summary>
public class ResourceTypeStructuralData : StructuralData<ResourceTypeInformationDto> public class ResourceTypeStructuralData : StructuralData<ResourceTypeInformationDto>
{ {
/// <summary> public override async IAsyncEnumerable<ResourceTypeInformationDto> GetAll()
/// Asynchronously retrieves all resource type data.
/// </summary>
/// <returns>A <see cref="Task"/> that represents the asynchronous operation and returns a collection of <see cref="ResourceTypeInformationDto"/>.</returns>
public override async Task<IEnumerable<ResourceTypeInformationDto>> GetAll()
{ {
var resourceTypeApi = new ResourceTypeApi(_apiConfiguration); var resourceTypeApi = new ResourceTypeApi(_apiConfiguration);
var resourceTypeInformationsResponse = await resourceTypeApi.GetAllResourceTypesInformationAsync(); var resourceTypeInformationsResponse = await resourceTypeApi.GetAllResourceTypesInformationAsync();
return resourceTypeInformationsResponse.Data; foreach (var item in resourceTypeInformationsResponse.Data)
{
yield return item; // Yield each item in the list to keep the IAsyncEnumerable logic
}
} }
/// <summary> public override async IAsyncEnumerable<IGraph> ConvertToLinkedDataAsync(IAsyncEnumerable<ResourceTypeInformationDto> entries)
/// Converts a collection of resource type data entries into a set of RDF graphs.
/// Each resource type is transformed into a graph, with RDF triples representing various properties of the resource type.
/// </summary>
/// <param name="entries">A collection of <see cref="ResourceTypeInformationDto"/> instances representing resource type data.</param>
/// <returns>A collection of <see cref="IGraph"/> instances, each representing an RDF graph of a resource type.</returns>
public override async Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<ResourceTypeInformationDto> entries)
{ {
var graphs = new List<IGraph>(); await foreach (var entry in entries)
foreach (var entry in entries)
{ {
var resourceTypeGraphName = UriHelper.TryCombineUri(RdfUris.CoscineResourceTypes, entry.Id) var resourceTypeGraphName = UriHelper.TryCombineUri(RdfUris.CoscineResourceTypes, entry.Id)
?? throw new Exception("Could not combine resource types prefix with resource type ID"); ?? throw new Exception("Could not combine resource types prefix with resource type ID");
...@@ -89,9 +80,8 @@ public class ResourceTypeStructuralData : StructuralData<ResourceTypeInformation ...@@ -89,9 +80,8 @@ public class ResourceTypeStructuralData : StructuralData<ResourceTypeInformation
if (!getTriplesDcatDataService.Any() || !getTriplesDctermsTitle.Any()) if (!getTriplesDcatDataService.Any() || !getTriplesDctermsTitle.Any())
{ {
graphs.Add(graph); yield return graph;
} }
} }
return await Task.FromResult(graphs);
} }
} }
\ No newline at end of file
...@@ -14,29 +14,16 @@ namespace SQL2Linked.Implementations; ...@@ -14,29 +14,16 @@ namespace SQL2Linked.Implementations;
/// </summary> /// </summary>
public class RoleStructuralData : StructuralData<RoleDto> public class RoleStructuralData : StructuralData<RoleDto>
{ {
/// <summary> public override IAsyncEnumerable<RoleDto> GetAll()
/// Asynchronously retrieves all role data.
/// </summary>
/// <returns>A <see cref="Task"/> that represents the asynchronous operation and returns a collection of <see cref="RoleDto"/>.</returns>
public override async Task<IEnumerable<RoleDto>> GetAll()
{ {
var roleApi = new RoleApi(_apiConfiguration); var roleApi = new RoleApi(_apiConfiguration);
return await RequestUtil.WrapPagedRequest<RoleDtoPagedResponse, RoleDto>( return PaginationHelper.GetAllAsync<RoleDtoPagedResponse, RoleDto>(
(currentPage) => roleApi.GetRolesAsync(pageNumber: currentPage) (currentPage) => roleApi.GetRolesAsync(pageNumber: currentPage));
);
} }
/// <summary> public override async IAsyncEnumerable<IGraph> ConvertToLinkedDataAsync(IAsyncEnumerable<RoleDto> entries)
/// Converts a collection of role data entries into a set of RDF graphs.
/// Each role is transformed into a graph, with RDF triples representing various properties of the role.
/// </summary>
/// <param name="entries">A collection of <see cref="RoleDto"/> instances representing role data.</param>
/// <returns>A collection of <see cref="IGraph"/> instances, each representing an RDF graph of a role.</returns>
public override async Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<RoleDto> entries)
{ {
var graphs = new List<IGraph>(); await foreach (var entry in entries)
foreach (var entry in entries)
{ {
var roleGraphName = UriHelper.TryCombineUri(RdfUris.CoscineRoles, entry.Id) var roleGraphName = UriHelper.TryCombineUri(RdfUris.CoscineRoles, entry.Id)
?? throw new Exception("Could not combine role prefix with role ID"); ?? throw new Exception("Could not combine role prefix with role ID");
...@@ -76,9 +63,8 @@ public class RoleStructuralData : StructuralData<RoleDto> ...@@ -76,9 +63,8 @@ public class RoleStructuralData : StructuralData<RoleDto>
} }
if (!getTriplesOrgRole.Any() || !getTriplesDctermsTitle.Any()) if (!getTriplesOrgRole.Any() || !getTriplesDctermsTitle.Any())
{ {
graphs.Add(graph); yield return graph;
} }
} }
return graphs;
} }
} }
\ No newline at end of file
...@@ -13,29 +13,15 @@ namespace SQL2Linked.Implementations; ...@@ -13,29 +13,15 @@ namespace SQL2Linked.Implementations;
/// </summary> /// </summary>
public class UserStructuralData : StructuralData<UserDto> public class UserStructuralData : StructuralData<UserDto>
{ {
/// <summary> public override IAsyncEnumerable<UserDto> GetAll()
/// Asynchronously retrieves all user data only for users who have accepted the ToS.
/// </summary>
/// <returns>A <see cref="Task"/> that represents the asynchronous operation and returns a collection of <see cref="UserDto"/>.</returns>
/// <remarks>This override allows for the retrieval of users who have accepted the ToS.</remarks>
public override async Task<IEnumerable<UserDto>> GetAll()
{ {
return await RequestUtil.WrapPagedRequest<UserDtoPagedResponse, UserDto>( return PaginationHelper.GetAllAsync<UserDtoPagedResponse, UserDto>(
(currentPage) => _adminApi.GetAllUsersAsync(tosAccepted: true, pageNumber: currentPage, pageSize: 250) (currentPage) => _adminApi.GetAllUsersAsync(tosAccepted: true, pageNumber: currentPage, pageSize: 50));
);
} }
/// <summary> public override async IAsyncEnumerable<IGraph> ConvertToLinkedDataAsync(IAsyncEnumerable<UserDto> entries)
/// Converts a collection of user data entries into a set of RDF graphs.
/// Each user is transformed into a graph, with RDF triples representing various properties of the user.
/// </summary>
/// <param name="entries">A collection of <see cref="UserDto"/> instances representing user data.</param>
/// <returns>A collection of <see cref="IGraph"/> instances, each representing an RDF graph of a user.</returns>
public override async Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<UserDto> entries)
{ {
var graphs = new List<IGraph>(); await foreach (var entry in entries)
foreach (var entry in entries)
{ {
var userGraphName = UriHelper.TryCombineUri(RdfUris.CoscineUsers, entry.Id) var userGraphName = UriHelper.TryCombineUri(RdfUris.CoscineUsers, entry.Id)
?? throw new Exception("Could not combine users prefix with user ID"); ?? throw new Exception("Could not combine users prefix with user ID");
...@@ -61,15 +47,14 @@ public class UserStructuralData : StructuralData<UserDto> ...@@ -61,15 +47,14 @@ public class UserStructuralData : StructuralData<UserDto>
graph.Retract(getTriplesName); graph.Retract(getTriplesName);
AssertToGraphLiteralNode(graph, userGraphName, RdfUris.FoafName, entry.DisplayName); AssertToGraphLiteralNode(graph, userGraphName, RdfUris.FoafName, entry.DisplayName);
graphs.Add(graph);
Console.WriteLine($"Will migrate user '{entry.DisplayName}' with id '{entry.Id}'."); Console.WriteLine($"Will migrate user '{entry.DisplayName}' with id '{entry.Id}'.");
yield return graph;
} }
else else
{ {
Console.WriteLine($"Will NOT migrate user '{entry.DisplayName}' with id '{entry.Id}'."); Console.WriteLine($"Will NOT migrate user '{entry.DisplayName}' with id '{entry.Id}'.");
} }
} }
return await Task.FromResult(graphs);
} }
} }
...@@ -16,10 +16,11 @@ ...@@ -16,10 +16,11 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="Coscine.ApiClient" Version="1.3.1" /> <PackageReference Include="Coscine.ApiClient" Version="1.6.0-issue-2847-repor0007" />
<PackageReference Include="dotNetRdf" Version="3.1.1" /> <PackageReference Include="dotNetRdf" Version="3.1.1" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" /> <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Options" Version="8.0.1" /> <PackageReference Include="Microsoft.Extensions.Options" Version="8.0.2" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
</ItemGroup> </ItemGroup>
</Project> </Project>
...@@ -42,33 +42,36 @@ public abstract class StructuralData<S> ...@@ -42,33 +42,36 @@ public abstract class StructuralData<S>
BasePath = "http://localhost:7206/coscine", BasePath = "http://localhost:7206/coscine",
ApiKeyPrefix = { { "Authorization", "Bearer" } }, ApiKeyPrefix = { { "Authorization", "Bearer" } },
ApiKey = { { "Authorization", _adminToken } }, ApiKey = { { "Authorization", _adminToken } },
Timeout = 300000, // 5 minutes
}; };
_adminApi = new AdminApi(_apiConfiguration); _adminApi = new AdminApi(_apiConfiguration);
} }
/// <summary> /// <summary>
/// Converts the given entries to linked data graphs. /// Asynchronously converts a collection of entries into linked data graphs.
/// </summary> /// </summary>
/// <param name="entries">The entries to convert.</param> /// <param name="entries">A collection of entries of type <typeparamref name="S"/> to convert.</param>
/// <returns>An enumerable collection of graphs representing the linked data.</returns> /// <typeparam name="S">The type of the entries to be converted into linked data graphs.</typeparam>
public abstract Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<S> entries); /// <returns>An asynchronous enumerable (<see cref="IAsyncEnumerable{IGraph}"/>) of graphs, where each graph represents the linked data derived from an entry.</returns>
public abstract IAsyncEnumerable<IGraph> ConvertToLinkedDataAsync(IAsyncEnumerable<S> entries);
/// <summary> /// <summary>
/// Retrieves all entries of type <typeparamref name="S"/>. /// Asynchronously retrieves all entries of type <typeparamref name="S"/>.
/// </summary> /// </summary>
/// <returns>A task that represents the asynchronous operation. The task result contains an enumerable of all entries.</returns> /// <typeparam name="S">The type of the entries to retrieve.</typeparam>
public abstract Task<IEnumerable<S>> GetAll(); /// <returns>An <see cref="IAsyncEnumerable{S}"/> representing an asynchronous stream of entries.</returns>
public abstract IAsyncEnumerable<S> GetAll();
/// <summary> /// <summary>
/// Migrates the data, optionally in a dummy mode where changes are not persisted. /// Asynchronously migrates data, with an option to simulate the migration process without making any changes (dummy mode).
/// </summary> /// </summary>
/// <param name="dummyMode">If set to <c>true</c>, the migration is simulated but not executed.</param> /// <param name="dummyMode">If set to <c>true</c>, the method simulates the migration process without executing any changes to the data store.</param>
/// <returns>A task representing the asynchronous operation.</returns> /// <returns>A task representing the asynchronous migration operation.</returns>
public async Task Migrate(bool dummyMode) public async Task Migrate(bool dummyMode)
{ {
var spacer = new string('-', 35); var spacer = new string('-', 35);
Console.WriteLine($"\n{spacer}\n{typeof(S).Name}\n{spacer}"); Console.WriteLine($"\n{spacer}\n{typeof(S).Name}\n{spacer}");
var graphs = await ConvertToLinkedDataAsync(await GetAll()); var graphs = ConvertToLinkedDataAsync(GetAll());
if (!dummyMode) if (!dummyMode)
{ {
await StoreGraphs(graphs); await StoreGraphs(graphs);
...@@ -76,16 +79,16 @@ public abstract class StructuralData<S> ...@@ -76,16 +79,16 @@ public abstract class StructuralData<S>
} }
/// <summary> /// <summary>
/// Stores the given graphs in the underlying data store. /// Asynchronously stores a collection of RDF graphs in the underlying data store.
/// </summary> /// </summary>
/// <param name="graphs">The graphs to be stored.</param> /// <param name="graphs">An asynchronous enumerable of RDF graphs (<see cref="IAsyncEnumerable{IGraph}"/>) to be stored.</param>
/// <returns>A task representing the asynchronous operation.</returns> /// <returns>A task representing the asynchronous operation of storing the graphs.</returns>
public async Task StoreGraphs(IEnumerable<IGraph> graphs) public async Task StoreGraphs(IAsyncEnumerable<IGraph> graphs)
{ {
var formatEnum = RdfFormat.TextTurtle; var formatEnum = RdfFormat.TextTurtle;
var format = "text/turtle"; var format = "text/turtle";
foreach (var graph in graphs) await foreach (var graph in graphs)
{ {
Console.WriteLine($" ({graph.BaseUri})"); Console.WriteLine($" ({graph.BaseUri})");
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment