"ValueError: Invalid endpoint: " when downloading files
Hi @romin.benfer something is up with the file download function.
Here's the output when I try to download a file using resource.file('README.txt').download()
:
{
"name": "ValueError",
"message": "Invalid endpoint: ",
"stack": "---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
Cell In[21], line 2
1 f = resource.file('README.txt')
----> 2 f.download()
File /opt/homebrew/lib/python3.11/site-packages/coscine/resource.py:1116, in FileObject.download(self, path, recursive)
1114 filepath = f\"{path} ({i})\"
1115 with open(filepath, \"wb\") as fp:
-> 1116 self.stream(fp)
File /opt/homebrew/lib/python3.11/site-packages/coscine/resource.py:1123, in FileObject.stream(self, fp)
1119 \"\"\"
1120 Streams file contents.
1121 \"\"\"
1122 if self.resource.type.general_type == \"rdss3\" and self.client.native:
-> 1123 self.stream_s3(fp)
1124 else:
1125 self.stream_blob(fp)
File /opt/homebrew/lib/python3.11/site-packages/coscine/resource.py:1157, in FileObject.stream_s3(self, handle)
1147 \"\"\"
1148 Works only on rdss3 resources and should not be called
1149 on other resource types! Bypasses Coscine and uploads
1150 directly to the underlying s3 storage.
1151 \"\"\"
1152 progress_bar = tqdm(
1153 desc=self.path, total=self.size,
1154 unit=\"B\", unit_scale=True, ascii=True,
1155 disable=not self.client.verbose
1156 )
-> 1157 s3 = boto3.client(
1158 \"s3\",
1159 aws_access_key_id=self.resource.type.options.access_key_write,
1160 aws_secret_access_key=self.resource.type.options.secret_key_write,
1161 endpoint_url=self.resource.type.options.endpoint
1162 )
1163 s3.download_fileobj(
1164 self.resource.type.options.bucket_name,
1165 self.path,
1166 handle,
1167 Callback=progress_bar.update
1168 )
File /opt/homebrew/lib/python3.11/site-packages/boto3/__init__.py:92, in client(*args, **kwargs)
86 def client(*args, **kwargs):
87 \"\"\"
88 Create a low-level service client by name using the default session.
89
90 See :py:meth:`boto3.session.Session.client`.
91 \"\"\"
---> 92 return _get_default_session().client(*args, **kwargs)
File /opt/homebrew/lib/python3.11/site-packages/boto3/session.py:299, in Session.client(self, service_name, region_name, api_version, use_ssl, verify, endpoint_url, aws_access_key_id, aws_secret_access_key, aws_session_token, config)
217 def client(
218 self,
219 service_name,
(...)
228 config=None,
229 ):
230 \"\"\"
231 Create a low-level service client by name.
232
(...)
297
298 \"\"\"
--> 299 return self._session.create_client(
300 service_name,
301 region_name=region_name,
302 api_version=api_version,
303 use_ssl=use_ssl,
304 verify=verify,
305 endpoint_url=endpoint_url,
306 aws_access_key_id=aws_access_key_id,
307 aws_secret_access_key=aws_secret_access_key,
308 aws_session_token=aws_session_token,
309 config=config,
310 )
File /opt/homebrew/lib/python3.11/site-packages/botocore/session.py:997, in Session.create_client(self, service_name, region_name, api_version, use_ssl, verify, endpoint_url, aws_access_key_id, aws_secret_access_key, aws_session_token, config)
980 self._add_configured_endpoint_provider(
981 client_name=service_name,
982 config_store=config_store,
983 )
985 client_creator = botocore.client.ClientCreator(
986 loader,
987 endpoint_resolver,
(...)
995 user_agent_creator=user_agent_creator,
996 )
--> 997 client = client_creator.create_client(
998 service_name=service_name,
999 region_name=region_name,
1000 is_secure=use_ssl,
1001 endpoint_url=endpoint_url,
1002 verify=verify,
1003 credentials=credentials,
1004 scoped_config=self.get_scoped_config(),
1005 client_config=config,
1006 api_version=api_version,
1007 auth_token=auth_token,
1008 )
1009 monitor = self._get_internal_component('monitor')
1010 if monitor is not None:
File /opt/homebrew/lib/python3.11/site-packages/botocore/client.py:159, in ClientCreator.create_client(self, service_name, region_name, is_secure, endpoint_url, verify, credentials, scoped_config, api_version, client_config, auth_token)
146 region_name, client_config = self._normalize_fips_region(
147 region_name, client_config
148 )
149 endpoint_bridge = ClientEndpointBridge(
150 self._endpoint_resolver,
151 scoped_config,
(...)
157 ),
158 )
--> 159 client_args = self._get_client_args(
160 service_model,
161 region_name,
162 is_secure,
163 endpoint_url,
164 verify,
165 credentials,
166 scoped_config,
167 client_config,
168 endpoint_bridge,
169 auth_token,
170 endpoints_ruleset_data,
171 partition_data,
172 )
173 service_client = cls(**client_args)
174 self._register_retries(service_client)
File /opt/homebrew/lib/python3.11/site-packages/botocore/client.py:490, in ClientCreator._get_client_args(self, service_model, region_name, is_secure, endpoint_url, verify, credentials, scoped_config, client_config, endpoint_bridge, auth_token, endpoints_ruleset_data, partition_data)
466 def _get_client_args(
467 self,
468 service_model,
(...)
479 partition_data,
480 ):
481 args_creator = ClientArgsCreator(
482 self._event_emitter,
483 self._user_agent,
(...)
488 user_agent_creator=self._user_agent_creator,
489 )
--> 490 return args_creator.get_client_args(
491 service_model,
492 region_name,
493 is_secure,
494 endpoint_url,
495 verify,
496 credentials,
497 scoped_config,
498 client_config,
499 endpoint_bridge,
500 auth_token,
501 endpoints_ruleset_data,
502 partition_data,
503 )
File /opt/homebrew/lib/python3.11/site-packages/botocore/args.py:137, in ClientArgsCreator.get_client_args(self, service_model, region_name, is_secure, endpoint_url, verify, credentials, scoped_config, client_config, endpoint_bridge, auth_token, endpoints_ruleset_data, partition_data)
134 new_config = Config(**config_kwargs)
135 endpoint_creator = EndpointCreator(event_emitter)
--> 137 endpoint = endpoint_creator.create_endpoint(
138 service_model,
139 region_name=endpoint_region_name,
140 endpoint_url=endpoint_config['endpoint_url'],
141 verify=verify,
142 response_parser_factory=self._response_parser_factory,
143 max_pool_connections=new_config.max_pool_connections,
144 proxies=new_config.proxies,
145 timeout=(new_config.connect_timeout, new_config.read_timeout),
146 socket_options=socket_options,
147 client_cert=new_config.client_cert,
148 proxies_config=new_config.proxies_config,
149 )
151 serializer = botocore.serialize.create_serializer(
152 protocol, parameter_validation
153 )
154 response_parser = botocore.parsers.create_parser(protocol)
File /opt/homebrew/lib/python3.11/site-packages/botocore/endpoint.py:402, in EndpointCreator.create_endpoint(self, service_model, region_name, endpoint_url, verify, response_parser_factory, timeout, max_pool_connections, http_session_cls, proxies, socket_options, client_cert, proxies_config)
384 def create_endpoint(
385 self,
386 service_model,
(...)
397 proxies_config=None,
398 ):
399 if not is_valid_endpoint_url(
400 endpoint_url
401 ) and not is_valid_ipv6_endpoint_url(endpoint_url):
--> 402 raise ValueError(\"Invalid endpoint: %s\" % endpoint_url)
404 if proxies is None:
405 proxies = self._get_proxies(endpoint_url)
ValueError: Invalid endpoint: "
}
I tried this in two separate RDS-S3 resources.
I'm not sure which API function downloads a file... the GetBlob one seems to work fine, but that's as far as I've gotten.