From 7ca18e55360e37bc0098d7b968af5b075f794144 Mon Sep 17 00:00:00 2001
From: Morian Sonnet <morian.sonnet@isea.rwth-aachen.de>
Date: Wed, 15 Jan 2025 16:47:23 +0100
Subject: [PATCH 1/2] Reduce code duplication

---
 src/coscine/resource.py | 34 +++++++++++-----------------------
 1 file changed, 11 insertions(+), 23 deletions(-)

diff --git a/src/coscine/resource.py b/src/coscine/resource.py
index a5bcfb6..7ce6fcb 100644
--- a/src/coscine/resource.py
+++ b/src/coscine/resource.py
@@ -665,7 +665,8 @@ class Resource:
         self,
         path: str,
         handle: BinaryIO,
-        progress: Callable[[int], None] | None = None
+        progress: Callable[[int], None] | None = None,
+        use_put: bool = False
     ):
         """
         Uploads a file-like object to a resource in Coscine.
@@ -705,7 +706,10 @@ class Resource:
                 progress_callback(progress_bar, mon.bytes_read, progress)
         )
         headers = {"Content-Type": monitor.content_type}
-        self.client.post(uri, data=monitor, headers=headers)
+        if use_put:
+            self.client.put(uri, data=monitor, headers=headers)
+        else:
+            self.client.post(uri, data=monitor, headers=headers)
 
     def _upload_blob_s3(self, path: str, handle: BinaryIO) -> None:
         """
@@ -1201,27 +1205,11 @@ class FileObject:
             handle = handle.encode("utf-8")
         if isinstance(handle, bytes):
             handle = BytesIO(handle)
-        uri = self.client.uri(
-            "projects", self.resource.project.id,
-            "resources", self.resource.id,
-            "blobs", self.path
-        )
-        files = {
-            "file": (self.path, handle, "application/octect-stream")
-        }
-        encoder = MultipartEncoder(fields=files)
-        progress_bar = tqdm(
-            desc=self.path, total=encoder.len,
-            unit="B", unit_scale=True, ascii=True,
-            disable=not self.client.verbose
-        )
-        monitor = MultipartEncoderMonitor(
-            encoder,
-            lambda mon:
-                progress_callback(progress_bar, mon.bytes_read, progress)
-        )
-        headers = {"Content-Type": monitor.content_type}
-        self.client.put(uri, data=monitor, headers=headers)
+
+        if self.type.general_type == "rdss3" and self.client.native:
+            self.resource._upload_blob_s3(self.path, handle, progress)
+        else:
+            self.resource._upload_blob(self.path, handle, progress, use_put = True)
 
     def update_metadata(self, metadata: MetadataForm | dict | rdflib.Graph) -> None:
         """
-- 
GitLab


From 6fb52060fd4cb0b3cf59de44de6456347528dcd0 Mon Sep 17 00:00:00 2001
From: Morian Sonnet <morian.sonnet@isea.rwth-aachen.de>
Date: Wed, 15 Jan 2025 17:00:14 +0100
Subject: [PATCH 2/2] Add custom progress callbacks for RDS3 resource

---
 src/coscine/resource.py | 39 ++++++++++++++++++++-------------------
 1 file changed, 20 insertions(+), 19 deletions(-)

diff --git a/src/coscine/resource.py b/src/coscine/resource.py
index 7ce6fcb..eb3ffb3 100644
--- a/src/coscine/resource.py
+++ b/src/coscine/resource.py
@@ -78,20 +78,6 @@ class ResourceQuota:
     def serialize(self) -> dict:
         return self._data
 
-
-def progress_callback(
-    progress_bar: tqdm,
-    bytes_read: int,
-    fn: Callable[[int], None] | None = None
-) -> None:
-    """
-    Updates the progress bar and calls a callback if one has been specified.
-    """
-    progress_bar.update(bytes_read - progress_bar.n)
-    if fn:
-        fn(bytes_read)
-
-
 class ResourceTypeOptions:
     """
     Options and settings regarding the resource type.
@@ -621,7 +607,7 @@ class Resource:
             self.post_metadata(metadata)
         assert isinstance(handle, IOBase)
         if self.type.general_type == "rdss3" and self.client.native:
-            self._upload_blob_s3(path, handle)
+            self._upload_blob_s3(path, handle, progress)
         else:
             self._upload_blob(path, handle, progress)
 
@@ -700,10 +686,16 @@ class Resource:
             unit="B", unit_scale=True, ascii=True,
             disable=not self.client.verbose
         )
+
+        def progress_callback(mon):
+            nonlocal progress_bar, progress
+            progress_bar.update(mon.bytes_read - progress_bar.n)
+            if progress:
+                progress(mon.bytes_read)
+
         monitor = MultipartEncoderMonitor(
             encoder,
-            lambda mon:
-                progress_callback(progress_bar, mon.bytes_read, progress)
+            progress_callback
         )
         headers = {"Content-Type": monitor.content_type}
         if use_put:
@@ -711,7 +703,7 @@ class Resource:
         else:
             self.client.post(uri, data=monitor, headers=headers)
 
-    def _upload_blob_s3(self, path: str, handle: BinaryIO) -> None:
+    def _upload_blob_s3(self, path: str, handle: BinaryIO, progress: Callable[[int], None] | None = None) -> None:
         """
         Works only on rdss3 resources and should not be called
         on other resource types! Bypasses Coscine and uploads
@@ -727,11 +719,20 @@ class Resource:
             aws_secret_access_key=self.type.options.secret_key_write,
             endpoint_url=self.type.options.endpoint
         )
+
+        bytes_read_abs = 0
+        def progress_callback(bytes_read_inc):
+            nonlocal progress_bar, progress, bytes_read_abs
+            progress_bar.update(bytes_read_inc)
+            bytes_read_abs += bytes_read_inc
+            if progress:
+                progress(bytes_read_abs)
+
         s3.upload_fileobj(
             handle,
             self.type.options.bucket_name,
             path,
-            Callback=progress_bar.update
+            Callback=progress_callback
         )
 
     def _fetch_files_recursively(self, path: str = "") -> list[FileObject]:
-- 
GitLab