Update: Now using IAsyncEnumerable to yield results

coscine/issues#2847

src/SQL2Linked/Implementations/ProjectStructuralData.cs

@@ -12,27 +12,12 @@ namespace SQL2Linked.Implementations;
 /// </summary>
 public class ProjectStructuralData : StructuralData<ProjectAdminDto>
 {
-    /// <summary>
-    /// Asynchronously retrieves all project data, including deleted projects.
-    /// </summary>
-    /// <returns>A <see cref="Task"/> that represents the asynchronous operation and returns a collection of <see cref="ProjectAdminDto"/>.</returns>
-    /// <remarks>This override allows for the inclusion of deleted projects.</remarks>
-    public override async Task<IEnumerable<ProjectAdminDto>> GetAll()
-    {
-        return await RequestUtil.WrapPagedRequest<ProjectAdminDtoPagedResponse, ProjectAdminDto>(
-            (currentPage) => _adminApi.GetAllProjectsAsync(includeDeleted: true, pageNumber: currentPage, pageSize: 250)
-        );
-    }
+    public override IAsyncEnumerable<ProjectAdminDto> GetAll() => 
+        PaginationHelper.GetAllAsync<ProjectAdminDtoPagedResponse, ProjectAdminDto>(
+            (currentPage) => _adminApi.GetAllProjectsAsync(includeDeleted: true, pageNumber: currentPage, pageSize: 50));
 
-    /// <summary>
-    /// Converts a collection of project data entries into a set of RDF graphs.
-    /// Each project is transformed into a graph, with RDF triples representing various properties of the project.
-    /// </summary>
-    /// <param name="entries">A collection of <see cref="ProjectAdminDto"/> instances representing project data.</param>
-    /// <returns>A collection of <see cref="IGraph"/> instances, each representing an RDF graph of a project.</returns>
-    public override async Task<IEnumerable<IGraph>> ConvertToLinkedDataAsync(IEnumerable<ProjectAdminDto> entries)
+    public override async IAsyncEnumerable<IGraph> ConvertToLinkedDataAsync(IAsyncEnumerable<ProjectAdminDto> entries)
     {
-        var graphs = new List<IGraph>();
         var coscineHandlePrefix = UriHelper.TryCombinePath(RdfUris.HandlePrefix, _pidConfiguration.Prefix)
             ?? throw new Exception("Could not combine handle prefix with PID prefix");
 
@@ -42,9 +27,10 @@ public class ProjectStructuralData : StructuralData<ProjectAdminDto>
         };
         AssertToGraphUriNode(coscineGraph, RdfUris.CoscinePrefix, RdfUris.DcatCatalog, RdfUris.CoscineProjects);
         AssertToGraphUriNode(coscineGraph, RdfUris.CoscinePrefix, RdfUris.DcatCatalog, RdfUris.CoscineResources);
-        graphs.Add(coscineGraph);
 
-        foreach (var entry in entries)
+        yield return coscineGraph; // yeld coscineGraph first
+
+        await foreach (var entry in entries)
         {
             var projectGraphName = UriHelper.TryCombineUri(RdfUris.CoscineProjects, entry.Id)
                 ?? throw new Exception("Could not combine projects prefix with project ID");
@@ -159,8 +145,7 @@ public class ProjectStructuralData : StructuralData<ProjectAdminDto>
                 Console.WriteLine($"For project '{entry.DisplayName}' will migrate triple '{projectGraphName} {RdfUris.DcTermsCreated} {entry.CreationDate}'. ");
             }
 
-            graphs.Add(graph);
+            yield return graph;
         }
-        return await Task.FromResult(graphs);
     }
 }
\ No newline at end of file