diff --git a/src/SQL2Linked/Implementations/ProjectStructuralData.cs b/src/SQL2Linked/Implementations/ProjectStructuralData.cs index 6242f4b266a9b95e70def8931120c5b4419d956a..f3fb09a8251b1ef97b85fc36f4884c19b936e108 100644 --- a/src/SQL2Linked/Implementations/ProjectStructuralData.cs +++ b/src/SQL2Linked/Implementations/ProjectStructuralData.cs @@ -12,27 +12,12 @@ namespace SQL2Linked.Implementations; /// </summary> public class ProjectStructuralData : StructuralData<ProjectAdminDto> { - /// <summary> - /// Asynchronously retrieves all project data, including deleted projects. - /// </summary> - /// <returns>A <see cref="Task"/> that represents the asynchronous operation and returns a collection of <see cref="ProjectAdminDto"/>.</returns> - /// <remarks>This override allows for the inclusion of deleted projects.</remarks> - public override async Task<IEnumerable<ProjectAdminDto>> GetAll() - { - return await RequestUtil.WrapPagedRequest<ProjectAdminDtoPagedResponse, ProjectAdminDto>( - (currentPage) => _adminApi.GetAllProjectsAsync(includeDeleted: true, pageNumber: currentPage, pageSize: 250) - ); - } + public override IAsyncEnumerable<ProjectAdminDto> GetAll() => + PaginationHelper.GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>( + (currentPage) => _adminApi.GetAllProjectsAsync(includeDeleted: true, pageNumber: currentPage, pageSize: 50)); - /// <summary> - /// Converts a collection of project data entries into a set of RDF graphs. - /// Each project is transformed into a graph, with RDF triples representing various properties of the project. - /// </summary> - /// <param name="entries">A collection of <see cref="ProjectAdminDto"/> instances representing project data.</param> - /// <returns>A collection of <see cref="IGraph"/> instances, each representing an RDF graph of a project.</returns> - public override async Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<ProjectAdminDto> entries) + public override async IAsyncEnumerable<IGraph> ConvertToLinkedDataAsync(IAsyncEnumerable<ProjectAdminDto> entries) { - var graphs = new List<IGraph>(); var coscineHandlePrefix = UriHelper.TryCombinePath(RdfUris.HandlePrefix, _pidConfiguration.Prefix) ?? throw new Exception("Could not combine handle prefix with PID prefix"); @@ -42,9 +27,10 @@ public class ProjectStructuralData : StructuralData<ProjectAdminDto> }; AssertToGraphUriNode(coscineGraph, RdfUris.CoscinePrefix, RdfUris.DcatCatalog, RdfUris.CoscineProjects); AssertToGraphUriNode(coscineGraph, RdfUris.CoscinePrefix, RdfUris.DcatCatalog, RdfUris.CoscineResources); - graphs.Add(coscineGraph); - foreach (var entry in entries) + yield return coscineGraph; // yeld coscineGraph first + + await foreach (var entry in entries) { var projectGraphName = UriHelper.TryCombineUri(RdfUris.CoscineProjects, entry.Id) ?? throw new Exception("Could not combine projects prefix with project ID"); @@ -159,8 +145,7 @@ public class ProjectStructuralData : StructuralData<ProjectAdminDto> Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {RdfUris.DcTermsCreated} {entry.CreationDate}'. "); } - graphs.Add(graph); + yield return graph; } - return await Task.FromResult(graphs); } } \ No newline at end of file diff --git a/src/SQL2Linked/Implementations/ResourceStructuralData.cs b/src/SQL2Linked/Implementations/ResourceStructuralData.cs index cb1683459cd21c6d6c92c9823dcbb0ccde297e74..db6c249e65df141a462ae8e102decd2087187f95 100644 --- a/src/SQL2Linked/Implementations/ResourceStructuralData.cs +++ b/src/SQL2Linked/Implementations/ResourceStructuralData.cs @@ -14,31 +14,16 @@ namespace SQL2Linked.Implementations; /// </summary> public class ResourceStructuralData : StructuralData<ResourceAdminDto> { - /// <summary> - /// Asynchronously retrieves all resource data, including deleted resources. - /// </summary> - /// <returns>A <see cref="Task"/> that represents the asynchronous operation and returns a collection of <see cref="ResourceAdminDto"/>.</returns> - /// <remarks>This override allows for the inclusion of deleted resources.</remarks> - public override async Task<IEnumerable<ResourceAdminDto>> GetAll() - { - return await RequestUtil.WrapPagedRequest<ResourceAdminDtoPagedResponse, ResourceAdminDto>( - (currentPage) => _adminApi.GetAllResourcesAsync(includeDeleted: true, pageNumber: currentPage, pageSize: 250) - ); - } + public override IAsyncEnumerable<ResourceAdminDto> GetAll() => + PaginationHelper.GetAllAsync<ResourceAdminDtoPagedResponse, ResourceAdminDto>( + (currentPage) => _adminApi.GetAllResourcesAsync(includeDeleted: true, pageNumber: currentPage, pageSize: 50)); - /// <summary> - /// Converts a collection of resource data entries into a set of RDF graphs. - /// Each resource is transformed into a graph, with RDF triples representing various properties of the resource. - /// </summary> - /// <param name="entries">A collection of <see cref="ResourceAdminDto"/> instances representing resource data.</param> - /// <returns>A collection of <see cref="IGraph"/> instances, each representing an RDF graph of a resource.</returns> - public override async Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<ResourceAdminDto> entries) + public override async IAsyncEnumerable<IGraph> ConvertToLinkedDataAsync(IAsyncEnumerable<ResourceAdminDto> entries) { - var graphs = new List<IGraph>(); var coscineHandlePrefix = UriHelper.TryCombinePath(RdfUris.HandlePrefix, _pidConfiguration.Prefix) ?? throw new Exception("Could not combine handle prefix with PID prefix"); - foreach (var entry in entries) + await foreach (var entry in entries) { var resourceGraphName = UriHelper.TryCombineUri(RdfUris.CoscineResources, entry.Id) ?? throw new Exception("Could not combine resources prefix with resource ID"); @@ -180,9 +165,7 @@ public class ResourceStructuralData : StructuralData<ResourceAdminDto> Console.WriteLine($"For resource '{entry.DisplayName}' will migrate triple '{resourceGraphName} {RdfUris.DcTermsCreated} {entry.DateCreated}'. "); } - graphs.Add(graph); + yield return graph; } - - return await Task.FromResult(graphs); } } diff --git a/src/SQL2Linked/Implementations/ResourceTypeStructuralData.cs b/src/SQL2Linked/Implementations/ResourceTypeStructuralData.cs index db049aeb726692fb5588926c958b40aad4655b78..31c1da309f827e4e095ee319b0495bc277c39352 100644 --- a/src/SQL2Linked/Implementations/ResourceTypeStructuralData.cs +++ b/src/SQL2Linked/Implementations/ResourceTypeStructuralData.cs @@ -13,28 +13,19 @@ namespace SQL2Linked.Implementations; /// </summary> public class ResourceTypeStructuralData : StructuralData<ResourceTypeInformationDto> { - /// <summary> - /// Asynchronously retrieves all resource type data. - /// </summary> - /// <returns>A <see cref="Task"/> that represents the asynchronous operation and returns a collection of <see cref="ResourceTypeInformationDto"/>.</returns> - public override async Task<IEnumerable<ResourceTypeInformationDto>> GetAll() + public override async IAsyncEnumerable<ResourceTypeInformationDto> GetAll() { var resourceTypeApi = new ResourceTypeApi(_apiConfiguration); var resourceTypeInformationsResponse = await resourceTypeApi.GetAllResourceTypesInformationAsync(); - return resourceTypeInformationsResponse.Data; + foreach (var item in resourceTypeInformationsResponse.Data) + { + yield return item; // Yield each item in the list to keep the IAsyncEnumerable logic + } } - /// <summary> - /// Converts a collection of resource type data entries into a set of RDF graphs. - /// Each resource type is transformed into a graph, with RDF triples representing various properties of the resource type. - /// </summary> - /// <param name="entries">A collection of <see cref="ResourceTypeInformationDto"/> instances representing resource type data.</param> - /// <returns>A collection of <see cref="IGraph"/> instances, each representing an RDF graph of a resource type.</returns> - public override async Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<ResourceTypeInformationDto> entries) + public override async IAsyncEnumerable<IGraph> ConvertToLinkedDataAsync(IAsyncEnumerable<ResourceTypeInformationDto> entries) { - var graphs = new List<IGraph>(); - - foreach (var entry in entries) + await foreach (var entry in entries) { var resourceTypeGraphName = UriHelper.TryCombineUri(RdfUris.CoscineResourceTypes, entry.Id) ?? throw new Exception("Could not combine resource types prefix with resource type ID"); @@ -89,9 +80,8 @@ public class ResourceTypeStructuralData : StructuralData<ResourceTypeInformation if (!getTriplesDcatDataService.Any() || !getTriplesDctermsTitle.Any()) { - graphs.Add(graph); + yield return graph; } } - return await Task.FromResult(graphs); } } \ No newline at end of file diff --git a/src/SQL2Linked/Implementations/RoleStructuralData.cs b/src/SQL2Linked/Implementations/RoleStructuralData.cs index dcd81ce0407a6a7a787caa878820632656bbb73a..bb514ced1ed2badcb16616dded0261c388d16475 100644 --- a/src/SQL2Linked/Implementations/RoleStructuralData.cs +++ b/src/SQL2Linked/Implementations/RoleStructuralData.cs @@ -14,29 +14,16 @@ namespace SQL2Linked.Implementations; /// </summary> public class RoleStructuralData : StructuralData<RoleDto> { - /// <summary> - /// Asynchronously retrieves all role data. - /// </summary> - /// <returns>A <see cref="Task"/> that represents the asynchronous operation and returns a collection of <see cref="RoleDto"/>.</returns> - public override async Task<IEnumerable<RoleDto>> GetAll() + public override IAsyncEnumerable<RoleDto> GetAll() { var roleApi = new RoleApi(_apiConfiguration); - return await RequestUtil.WrapPagedRequest<RoleDtoPagedResponse, RoleDto>( - (currentPage) => roleApi.GetRolesAsync(pageNumber: currentPage) - ); + return PaginationHelper.GetAllAsync<RoleDtoPagedResponse, RoleDto>( + (currentPage) => roleApi.GetRolesAsync(pageNumber: currentPage)); } - /// <summary> - /// Converts a collection of role data entries into a set of RDF graphs. - /// Each role is transformed into a graph, with RDF triples representing various properties of the role. - /// </summary> - /// <param name="entries">A collection of <see cref="RoleDto"/> instances representing role data.</param> - /// <returns>A collection of <see cref="IGraph"/> instances, each representing an RDF graph of a role.</returns> - public override async Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<RoleDto> entries) + public override async IAsyncEnumerable<IGraph> ConvertToLinkedDataAsync(IAsyncEnumerable<RoleDto> entries) { - var graphs = new List<IGraph>(); - - foreach (var entry in entries) + await foreach (var entry in entries) { var roleGraphName = UriHelper.TryCombineUri(RdfUris.CoscineRoles, entry.Id) ?? throw new Exception("Could not combine role prefix with role ID"); @@ -76,9 +63,8 @@ public class RoleStructuralData : StructuralData<RoleDto> } if (!getTriplesOrgRole.Any() || !getTriplesDctermsTitle.Any()) { - graphs.Add(graph); + yield return graph; } } - return graphs; } } \ No newline at end of file diff --git a/src/SQL2Linked/Implementations/UserStructuralData.cs b/src/SQL2Linked/Implementations/UserStructuralData.cs index 374205cf49bec15db8636778402a3cadb243be29..eba564f494d6b010d1940dcb0d2bf8cb3b64037b 100644 --- a/src/SQL2Linked/Implementations/UserStructuralData.cs +++ b/src/SQL2Linked/Implementations/UserStructuralData.cs @@ -13,29 +13,15 @@ namespace SQL2Linked.Implementations; /// </summary> public class UserStructuralData : StructuralData<UserDto> { - /// <summary> - /// Asynchronously retrieves all user data only for users who have accepted the ToS. - /// </summary> - /// <returns>A <see cref="Task"/> that represents the asynchronous operation and returns a collection of <see cref="UserDto"/>.</returns> - /// <remarks>This override allows for the retrieval of users who have accepted the ToS.</remarks> - public override async Task<IEnumerable<UserDto>> GetAll() + public override IAsyncEnumerable<UserDto> GetAll() { - return await RequestUtil.WrapPagedRequest<UserDtoPagedResponse, UserDto>( - (currentPage) => _adminApi.GetAllUsersAsync(tosAccepted: true, pageNumber: currentPage, pageSize: 250) - ); + return PaginationHelper.GetAllAsync<UserDtoPagedResponse, UserDto>( + (currentPage) => _adminApi.GetAllUsersAsync(tosAccepted: true, pageNumber: currentPage, pageSize: 50)); } - /// <summary> - /// Converts a collection of user data entries into a set of RDF graphs. - /// Each user is transformed into a graph, with RDF triples representing various properties of the user. - /// </summary> - /// <param name="entries">A collection of <see cref="UserDto"/> instances representing user data.</param> - /// <returns>A collection of <see cref="IGraph"/> instances, each representing an RDF graph of a user.</returns> - public override async Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<UserDto> entries) + public override async IAsyncEnumerable<IGraph> ConvertToLinkedDataAsync(IAsyncEnumerable<UserDto> entries) { - var graphs = new List<IGraph>(); - - foreach (var entry in entries) + await foreach (var entry in entries) { var userGraphName = UriHelper.TryCombineUri(RdfUris.CoscineUsers, entry.Id) ?? throw new Exception("Could not combine users prefix with user ID"); @@ -61,15 +47,14 @@ public class UserStructuralData : StructuralData<UserDto> graph.Retract(getTriplesName); AssertToGraphLiteralNode(graph, userGraphName, RdfUris.FoafName, entry.DisplayName); - graphs.Add(graph); - Console.WriteLine($"Will migrate user '{entry.DisplayName}' with id '{entry.Id}'."); + + yield return graph; } else { Console.WriteLine($"Will NOT migrate user '{entry.DisplayName}' with id '{entry.Id}'."); } } - return await Task.FromResult(graphs); } } diff --git a/src/SQL2Linked/SQL2Linked.csproj b/src/SQL2Linked/SQL2Linked.csproj index 68ba671dd784b66db129f344362e484385e2da16..d2da8b27ea0fca0adaa2050936186390b1747b98 100644 --- a/src/SQL2Linked/SQL2Linked.csproj +++ b/src/SQL2Linked/SQL2Linked.csproj @@ -16,10 +16,11 @@ </PropertyGroup> <ItemGroup> - <PackageReference Include="Coscine.ApiClient" Version="1.3.1" /> + <PackageReference Include="Coscine.ApiClient" Version="1.6.0-issue-2847-repor0007" /> <PackageReference Include="dotNetRdf" Version="3.1.1" /> <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" /> - <PackageReference Include="Microsoft.Extensions.Options" Version="8.0.1" /> + <PackageReference Include="Microsoft.Extensions.Options" Version="8.0.2" /> + <PackageReference Include="System.Linq.Async" Version="6.0.1" /> </ItemGroup> </Project> diff --git a/src/SQL2Linked/StructuralData.cs b/src/SQL2Linked/StructuralData.cs index 70dc878a877359d4a74081a71f12eaf896d8035a..20b9fcced0abeb940048262f403c00b23849bac7 100644 --- a/src/SQL2Linked/StructuralData.cs +++ b/src/SQL2Linked/StructuralData.cs @@ -42,33 +42,36 @@ public abstract class StructuralData<S> BasePath = "http://localhost:7206/coscine", ApiKeyPrefix = { { "Authorization", "Bearer" } }, ApiKey = { { "Authorization", _adminToken } }, + Timeout = 300000, // 5 minutes }; _adminApi = new AdminApi(_apiConfiguration); } /// <summary> - /// Converts the given entries to linked data graphs. + /// Asynchronously converts a collection of entries into linked data graphs. /// </summary> - /// <param name="entries">The entries to convert.</param> - /// <returns>An enumerable collection of graphs representing the linked data.</returns> - public abstract Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<S> entries); + /// <param name="entries">A collection of entries of type <typeparamref name="S"/> to convert.</param> + /// <typeparam name="S">The type of the entries to be converted into linked data graphs.</typeparam> + /// <returns>An asynchronous enumerable (<see cref="IAsyncEnumerable{IGraph}"/>) of graphs, where each graph represents the linked data derived from an entry.</returns> + public abstract IAsyncEnumerable<IGraph> ConvertToLinkedDataAsync(IAsyncEnumerable<S> entries); /// <summary> - /// Retrieves all entries of type <typeparamref name="S"/>. + /// Asynchronously retrieves all entries of type <typeparamref name="S"/>. /// </summary> - /// <returns>A task that represents the asynchronous operation. The task result contains an enumerable of all entries.</returns> - public abstract Task<IEnumerable<S>> GetAll(); + /// <typeparam name="S">The type of the entries to retrieve.</typeparam> + /// <returns>An <see cref="IAsyncEnumerable{S}"/> representing an asynchronous stream of entries.</returns> + public abstract IAsyncEnumerable<S> GetAll(); /// <summary> - /// Migrates the data, optionally in a dummy mode where changes are not persisted. + /// Asynchronously migrates data, with an option to simulate the migration process without making any changes (dummy mode). /// </summary> - /// <param name="dummyMode">If set to <c>true</c>, the migration is simulated but not executed.</param> - /// <returns>A task representing the asynchronous operation.</returns> + /// <param name="dummyMode">If set to <c>true</c>, the method simulates the migration process without executing any changes to the data store.</param> + /// <returns>A task representing the asynchronous migration operation.</returns> public async Task Migrate(bool dummyMode) { var spacer = new string('-', 35); Console.WriteLine($"\n{spacer}\n{typeof(S).Name}\n{spacer}"); - var graphs = await ConvertToLinkedDataAsync(await GetAll()); + var graphs = ConvertToLinkedDataAsync(GetAll()); if (!dummyMode) { await StoreGraphs(graphs); @@ -76,16 +79,16 @@ public abstract class StructuralData<S> } /// <summary> - /// Stores the given graphs in the underlying data store. + /// Asynchronously stores a collection of RDF graphs in the underlying data store. /// </summary> - /// <param name="graphs">The graphs to be stored.</param> - /// <returns>A task representing the asynchronous operation.</returns> - public async Task StoreGraphs(IEnumerable<IGraph> graphs) + /// <param name="graphs">An asynchronous enumerable of RDF graphs (<see cref="IAsyncEnumerable{IGraph}"/>) to be stored.</param> + /// <returns>A task representing the asynchronous operation of storing the graphs.</returns> + public async Task StoreGraphs(IAsyncEnumerable<IGraph> graphs) { var formatEnum = RdfFormat.TextTurtle; var format = "text/turtle"; - foreach (var graph in graphs) + await foreach (var graph in graphs) { Console.WriteLine($" ({graph.BaseUri})");