diff --git a/.gitignore b/.gitignore
index 5e6fc369c7c57b2711a6b4034e6560f4181a4d39..c8b7ebec5864a56bc29a47cab049b19d2f47c53e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,9 +3,12 @@ __pycache__/
*.egg-info/
.ipynb_checkpoints
.dist/
-token.txt
.coverage
.mypy_cache/
public/
.idea/
playground.py
+.vscode/
+cache.json
+docs/_build/
+report.md
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 007c870da6e7846f634ab45968d2fed7462375ee..424d8a04bdb8a98c91a39af4abcd64b18d77c711 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,15 +1,8 @@
###############################################################################
# Coscine Python SDK
-# Copyright (c) 2018-2023 RWTH Aachen University
+# Copyright (c) 2020-2023 RWTH Aachen University
# Licensed under the terms of the MIT License
-###############################################################################
-# Coscine, short for Collaborative Scientific Integration Environment, is
-# a platform for research data management (RDM).
# For more information on Coscine visit https://www.coscine.de/.
-#
-# Please note that this python module is open source software primarily
-# developed and maintained by the scientific community. It is not
-# an official service that RWTH Aachen provides support for.
###############################################################################
###############################################################################
@@ -37,7 +30,7 @@ stages:
# the job fails.
###############################################################################
-module analysis with pyroma:
+pyroma package analysis:
stage: prepare deploy
tags:
- "runner:docker"
@@ -56,7 +49,7 @@ module analysis with pyroma:
# Since tabs are used for indentation, some settings have been turned off.
###############################################################################
-static analysis with flake8:
+flake8 static analysis:
stage: quality assurance
tags:
- "runner:docker"
@@ -76,7 +69,7 @@ static analysis with flake8:
# following best practices.
###############################################################################
-static analysis with pylint:
+pylint static analysis:
stage: quality assurance
tags:
- "runner:docker"
@@ -85,8 +78,7 @@ static analysis with pylint:
- python -V
- pip install pylint
script:
- - python -m pylint -f colorized --fail-under=9
- src/coscine
+ - python -m pylint -f colorized --fail-under=9 src/coscine
when: manual
###############################################################################
@@ -95,7 +87,7 @@ static analysis with pylint:
# dependencies in case they are not typed.
###############################################################################
-type checking with mypy:
+mypy type checking:
stage: quality assurance
tags:
- "runner:docker"
@@ -113,7 +105,7 @@ type checking with mypy:
# This job determines the minimum python version required to run the package.
###############################################################################
-determine minimum python version with vermin:
+minimum python version:
stage: prepare deploy
tags:
- "runner:docker"
@@ -122,7 +114,7 @@ determine minimum python version with vermin:
- python -V
- pip install vermin
script:
- - vermin -vvv --no-parse-comments --eval-annotations --target=3.7-
+ - vermin -vvv --no-parse-comments --eval-annotations --target=3.10-
--pessimistic --violations --backport typing src/coscine
when: manual
@@ -147,17 +139,18 @@ analyse security:
# This job tests the sourcecode in the python module.
###############################################################################
-unittest:
+integration testing:
stage: test
tags:
- "runner:docker"
- image: python:3.7
+ image: python:latest
before_script:
- python -V
- pip install .
script:
+ - cd tests/
- COSCINE_API_TOKEN=${COSCINE_API_TOKEN}
- python -m unittest discover src/tests
+ python test.py
when: manual
###############################################################################
@@ -169,7 +162,7 @@ release:
stage: deploy
tags:
- "runner:docker"
- image: "python:3.7"
+ image: python:latest
before_script:
- python -V
- pip --version
@@ -192,14 +185,16 @@ pages:
stage: pages
tags:
- "runner:docker"
- image: "python:3.7"
+ image: python:latest
before_script:
- python -V
- - pip install pdoc
- pip install -r requirements.txt
script:
- - python -m pdoc --docformat numpy --template-dir src/docs/template
- -o ./public ./src/coscine !coscine.cache !coscine.defaults
+ - cd docs
+ - py -m sphinx.ext.apidoc -o . ../src/coscine
+ - make html
+ - cp -rv _build/html ../public
+ - cd ..
- ls -la ./public
artifacts:
paths:
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
deleted file mode 100644
index aa3e9c722e8f4744be6cbadb9411ad0adf081f24..0000000000000000000000000000000000000000
--- a/CONTRIBUTING.md
+++ /dev/null
@@ -1,94 +0,0 @@
-# Contributing to the Coscine Python SDK
-## General
-The Coscine Python SDK is open source software and not developed by
-the Coscine team. Instead it is developed by the userbase of Coscine,
-which is researchers, data stewards and project managers.
-If you belong to that userbase and would like to improve this package,
-you are more than welcome to contribute.
-
-## Contribution Workflow
-The general flow of contributing to this repository is:
-1. Create a fork of the repository and make your changes in your fork.
-In case you are not an RWTH member you may not have full access to
-the [RWTH Gitlab instance] and may not be able to fork projects.
-If you would like to participate anyway, you need to be added to the project.
-Please contact the [Coscine Community Team] and state your intention of
-contributing to the project. They will be able to grant you developer rights,
-which enable you to create merge requests inside of community repositories.
-2. When youβre ready and satisfied with the changes in your fork,
- create a new merge request.
-3. Ensure you provide complete and accurate information in the merge
- requestβs description.
-4. Once youβre ready, mark your MR as ready for review.
-5. The Maintainers of this package will take a look at your contribution
- and incorporate the changes into the repository, provided no fatal flaws
- have been detected and your proposed change is actually useful.
-
-For small suggestions / changes you can also open an issue with
-a feature request.
-
-[RWTH Gitlab instance]: https://git.rwth-aachen.de
-[Coscine Community Team]: mailto:coscine-technical-adaptation@itc.rwth-aachen.de
-
-## Code convention & Style guidelines
-Consider some general guidelines to help making (your) code readable and
-in line with the rest of the code in this package.
-
-- Limit the column size to 80 columns per line
-- Use [numpy DOCstrings](https://numpydoc.readthedocs.io/en/latest/format.html)
- for modules, classes, methods and functions
-- Write excessive type hints
-- Write self descriptive code, use sensible variable/function names
-- Roughly stick to Python's PEP8
-- Try to comply with the minimum python version currently supported
- by the package. This version matches the recommendations given in
- https://devguide.python.org/versions/. The CI Pipeline contains a job
- to determine the minimum python version supported.
-
-Roughly stick to the following template for sourcecode files:
-```python
-###############################################################################
-# Coscine Python SDK
-# Copyright (c) 2018-2023 RWTH Aachen University
-# Licensed under the terms of the MIT License
-###############################################################################
-# Coscine, short for Collaborative Scientific Integration Environment, is
-# a platform for research data management (RDM).
-# For more information on Coscine visit https://www.coscine.de/.
-#
-# Please note that this python module is open source software primarily
-# developed and maintained by the scientific community. It is not
-# an official service that RWTH Aachen provides support for.
-###############################################################################
-
-###############################################################################
-# File description
-###############################################################################
-
-"""
-TODO
-"""
-
-###############################################################################
-# Dependencies
-###############################################################################
-
-from __future__ import annotations
-
-###############################################################################
-# Module globals / Constants
-###############################################################################
-
-###############################################################################
-# Classes / Functions / Scripts
-###############################################################################
-
-###############################################################################
-```
-
-## CI Pipelines
-This repository is configured to work with a variety of static analysis tools
-and automatic documentation generators. For an overview over the different
-CI/CD Pipelines, have a look at [.gitlab-ci.yml](.gitlab-ci.yml).
-Install the tools mentioned in the jobs on your local system to test your
-fork locally. Documentation is provided in the ci configuration file.
diff --git a/LICENSE.txt b/LICENSE.txt
index d55dca6ca2fb7d2589e5800c6c3f8bcd00bb472a..55567c08e1a101c1a394982140ecce91a5cd3102 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -1,6 +1,6 @@
MIT License
-Copyright (c) 2018-2023 RWTH Aachen University
+Copyright (c) 2020-2023 RWTH Aachen University
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
diff --git a/README.md b/README.md
index a8f63a1eef32eba584943b1b5edbccde4da1f520..86793c17838a002b979656e3ac39706a20d5a88d 100644
--- a/README.md
+++ b/README.md
@@ -1,146 +1,57 @@
# Coscine Python SDK
-![Coscine] ![Python]
-[Coscine](https://coscine.de/), short for **Co**llaborative **Sc**ientific
-**In**tegration **E**nvironment, is a platform for Research Data Management.
-
-[Coscine]: ./data/coscine_logo_rgb.png
-[Python]: ./data/python-powered-w-200x80.png
-[Coscine Landing Page]: https://www.coscine.de/
-
-## About
-The *Coscine Python SDK* is an open source python package providing
-a pythonic interface to the *Coscine* REST API.
-### Features:
-- **Project Management**
- - Create, modify or delete projects
- - Add/Invite members to projects and set their roles
- - Download projects and all their content
-- **Resource Management**
- - Create, modify or delete resources
- - Download resources and all of their content
-- **File and Metadata Management**
- - Upload, download and delete files
- - Interact with metadata in an intuitive pythonic manner
- - Fetch S3 access credentials
-
-> **DISCLAIMER**
-> Please note that this python module is developed and maintained
-> by the scientific community and even though Copyright remains with
-> *RWTH Aachen*, it is not an official service that *RWTH Aachen*
-> provides support for. Direct bug reports, feature requests and general
-> questions at this repository via the issues feature.
-
-## Example Code
-
-**Uploading a file to a resource located in a subproject:**
+![python-logo] ![coscine-logo]
+
+[Coscine](https://coscine.de), short for **Co**llaborative **Sc**ientific
+**In**tegration **E**nvironment, is a platform for Research Data Management.
+The Coscine Python SDK is an open source package that provides
+a high-level interface to the Coscine REST API. It allows for
+the automization of research workflows and offers nearly all
+the features that are available in the Coscine web interface
+to python programmers.
+
+## Showcase
+Updating the metadata of an existing file in a resource:
```python
import coscine
from datetime import datetime
-token: str = "My Coscine API Token"
-client = coscine.Client(token)
-project = client.project("My Project").subproject("My Subproject")
+token = "My Coscine API token"
+client = coscine.ApiClient(token)
+project = client.project("My Project")
resource = project.resource("My Resource")
-metadata = resource.metadata_form()
+handle = resource.file("My File.csv")
+metadata = handle.metadata()
metadata["Author"] = "Dr. Akula"
metadata["Created"] = datetime.now()
-metadata["Discipline"] = "Medicine"
-metadata["DAP"] = 0.32
-# ...
-resource.upload("file.txt", "C:/databases/file.txt", metadata)
-```
-
-**Listing all files in all resources of a project:**
-```python
-import coscine
-
-token: str = "My Coscine API Token"
-client = coscine.Client(token)
-for resource in client.project("My Project").resources():
- for file in resource.contents():
- if not file.is_folder:
- print(file.path)
-```
-
-More examples can be found in the online [documentation].
-
-[documentation]: https://coscine.pages.rwth-aachen.de/community-features/coscine-python-sdk/coscine.html
-
-## Installation
-### via the Python Package Index (PyPi)
-The *Coscine Python SDK* is hosted on the [*Python Package Index (PyPi)*].
-You can download and install the package with *pip*:
-```bash
-py -m pip install coscine
+handle.update(metadata)
```
-[*Python Package Index (PyPi)*]: https://pypi.org/project/coscine/
+## Documentation
+Installation instructions and an in-depth guide on using the Python SDK can
+be found in the online [documentation]. The source code itself has been
+heavily annotated with numpy-style DOCstrings. You can generate a local
+copy of the documentation using Sphinx:
-### via Conda
-The package version hosted on the *Python Package Index (PyPi)* is
-automatically mirrored in the community driven packaging for *Conda*.
-You can download and install the package with *conda*:
```bash
-conda install -c conda-forge coscine
-```
-### via Git
-Manual installation:
-```bash
-git clone https://git.rwth-aachen.de/coscine/community-features/coscine-python-sdk.git
-cd ./coscine-python-sdk
-py -m pip install .
+py -m pip install -U sphinx furo myst-parser
+cd docs
+set SPHINXBUILD=py -m sphinx.cmd.build
+py -m sphinx.ext.apidoc -o . ../src/coscine
+make html
```
-## Documentation
-The source code has been thorougly documented with *Python DOCstrings*.
-Documentation can be generated via a variety of tools that take advantage
-of these *DOCstrings*, such as [pydoc] or [pdoc].
-Use the following script to generate documentation with *pdoc*:
-```bash
-cd ./coscine-python-sdk
-py -m pip install pdoc
-py -m pdoc --docformat numpy --template-dir src/docs/template -o ./public ./src/coscine
-```
-The documentation inside of this repository is automatically deployed to
-a [GitLab-Pages instance] for online access.
-
-[GitLab-Pages instance]:https://coscine.pages.rwth-aachen.de/community-features/coscine-python-sdk/coscine.html
-[pydoc]:https://docs.python.org/3/library/pydoc.html
-[pdoc]:https://pypi.org/project/pdoc/
-
## Contact
To report bugs, request features or resolve questions open an issue inside
-of the current git repository.
-Contributions and any help on improving this package are appreciated. To
-contribute source code you may fork the repository and open a merge request
-or simply submit a short and relevant snippet or fix inside of an issue.
-More information on contributing can be found in [CONTRIBUTING.md].
-
-[CONTRIBUTING.md]: ./CONTRIBUTING.md
+of the current git repository. Contributions and any help on improving this
+package are appreciated. To contribute source code you may fork
+the repository and open a merge request or simply submit a short
+and relevant snippet or fix inside of an issue.
## License
-
This project is Open Source Software and licensed under the terms of
-the [MIT License](./LICENSE.txt).
+the [MIT License].
-> **MIT License**
->
-> Copyright (c) 2018-2023 *RWTH Aachen University*
->
-> Permission is hereby granted, free of charge, to any person obtaining a copy
-> of this software and associated documentation files (the "Software"), to deal
-> in the Software without restriction, including without limitation the rights
-> to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-> copies of the Software, and to permit persons to whom the Software is
-> furnished to do so, subject to the following conditions:
->
-> The above copyright notice and this permission notice shall be included in
-> all copies or substantial portions of the Software.
->
-> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-> FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-> AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-> LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-> SOFTWARE.
+[coscine-logo]: ./docs/_static/data/coscine_logo_rgb.png
+[python-logo]: ./docs/_static/data/python-powered-w-200x80.png
+[documentation]: https://coscine.pages.rwth-aachen.de/community-features/coscine-python-sdk/coscine.html
+[MIT License]: ./LICENSE.txt
diff --git a/docs/Makefile b/docs/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..d4bb2cbb9eddb1bb1b4f366623044af8e4830919
--- /dev/null
+++ b/docs/Makefile
@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS ?=
+SPHINXBUILD ?= sphinx-build
+SOURCEDIR = .
+BUILDDIR = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+ @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+ @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
diff --git a/data/coscine_logo_rgb.png b/docs/_static/coscine_logo_rgb.png
similarity index 100%
rename from data/coscine_logo_rgb.png
rename to docs/_static/coscine_logo_rgb.png
diff --git a/data/coscine_logo_white_rgb.png b/docs/_static/coscine_logo_white_rgb.png
similarity index 100%
rename from data/coscine_logo_white_rgb.png
rename to docs/_static/coscine_logo_white_rgb.png
diff --git a/data/coscine_python_sdk_icon_rgb.png b/docs/_static/coscine_python_sdk_icon_rgb.png
similarity index 100%
rename from data/coscine_python_sdk_icon_rgb.png
rename to docs/_static/coscine_python_sdk_icon_rgb.png
diff --git a/data/python-powered-w-200x80.png b/docs/_static/python-powered-w-200x80.png
similarity index 100%
rename from data/python-powered-w-200x80.png
rename to docs/_static/python-powered-w-200x80.png
diff --git a/docs/_templates/index.rst b/docs/_templates/index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..de1c4f130a6fa979d4938ad53d670645b6653c97
--- /dev/null
+++ b/docs/_templates/index.rst
@@ -0,0 +1,20 @@
+.. coscine documentation master file, created by
+ sphinx-quickstart on Thu Oct 26 00:54:17 2023.
+ You can adapt this file completely to your liking, but it should at least
+ contain the root `toctree` directive.
+
+Welcome to coscine's documentation!
+===================================
+
+.. toctree::
+ :maxdepth: 2
+ :caption: Contents:
+
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
diff --git a/docs/api-token.md b/docs/api-token.md
new file mode 100644
index 0000000000000000000000000000000000000000..7b1436c39d334501ecfec88487de4869e9bd6cb6
--- /dev/null
+++ b/docs/api-token.md
@@ -0,0 +1,75 @@
+# Creating an API token π
+You need an API token to use the Coscine API. If you have
+not already, [create a new API token].
+Once you have an API token you are ready to use the API.
+
+### A word of advice β
+The token represents sensible data and grants anyone in possesion of it full
+access to your data in coscine. Do not leak it publicly on online platforms!
+Do not include it within your sourcecode if you intend on uploading it
+to the internet or sharing it among peers. Take precautions
+and follow best practices to avoid corruption, theft or loss of data!
+
+### Using the API token
+There are two simple and safe methods of using the API token without exposing
+it to unintended audiences.
+
+#### Loading the token from a file
+Simply put your API token in a file on your harddrive and read the file when
+initializing the Coscine client. This has the advantage of keeping the token
+out of the sourcecode and offering the user an easy way to switch between
+tokens by changing the filename in the sourcecode.
+```python
+with open("token.txt", "rt") as fp:
+ token = fp.read()
+# You can now use token to intialize the coscine ApiClient!
+```
+This approach comes at the disadvantage of potentially exposing the token by
+accidentially leaking the file together with the sourcecode e. g. on online
+platforms, especially sourcecode hosting platforms such as GitHub or GitLab.
+Therefore precautions must be taken. When using git as a versioning system,
+a `.gitignore` file that inclues any possible token name or file extension
+should be mandatory. This allows you to exclude any filename like `token.txt`.
+An even better way would be to agree upon a common token file extension
+such as `.token` and exclude that file extension. By doing this you can
+safely upload/push your code to the usual suspects, since the `.gitignore`
+makes sure your sensitive files are not included in any of these uploads.
+- [gitignore guide by W3schools.com]
+- [gitignore docs on git-scm.com]
+
+#### Keeping the token in an environment variable
+This method does not rely on any files but instead on environment variables.
+Simply set an environment variable containing your token and use that
+variable from your python program.
+**Set environment variable:**
+```python
+import os
+os.environ["COSCINE_API_TOKEN"] = "My Token Value"
+```
+You would do this only once, and obviously not in your final program/script.
+There are alternative platform-specific methods for creating an environment
+variable (see links given below), but this pythonic approach should work on
+most platforms out of the box. Just don't do this in your actual program!
+Do it via a dedicated script or in an interactive python session (shell).
+**To get the environment variable:**
+```python
+import os
+token = os.getenv("COSCINE_API_TOKEN")
+```
+This is certainly a little more complex for some users, especially users who
+may want to use your program. They can easily share tokens by sending
+a file to colleagues but sharing environment variables requires each user
+to additionally create the environment variable on their local system.
+
+Find out how to temporarily or permanently set environment variables
+on certain Operating Systems:
+- [Windows]
+- [Linux]
+- [Apple]
+
+[create a new API token]: https://docs.coscine.de/en/token
+[gitignore guide by W3schools.com]: https://www.w3schools.com/git/git_ignore.asp
+[gitignore docs on git-scm.com]: https://git-scm.com/docs/gitignore
+[Windows]: http://www.dowdandassociates.com/blog/content/howto-set-an-environment-variable-in-windows-command-line-and-registry/
+[Linux]: https://phoenixnap.com/kb/linux-set-environment-variable
+[Apple]: https://i.kym-cdn.com/photos/images/newsfeed/001/765/157/333.jpg
diff --git a/docs/api.md b/docs/api.md
new file mode 100644
index 0000000000000000000000000000000000000000..b7fd2b5dafb3d75da931931481a52475509ce452
--- /dev/null
+++ b/docs/api.md
@@ -0,0 +1,4 @@
+# Coscine REST API
+The Coscine REST API has its own [dedicated documentation page].
+There you are able to see a list of the various endpoints and
+the data they expect and return.
diff --git a/docs/client.md b/docs/client.md
new file mode 100644
index 0000000000000000000000000000000000000000..d52953b67b02711499a6c1d7b277a6a99669ae01
--- /dev/null
+++ b/docs/client.md
@@ -0,0 +1,77 @@
+# The ApiClient
+The ApiClient communicates with the Coscine REST API and enables
+the user to make requests to and get responses from Coscine.
+
+## Cache
+To speed things up it makes use of a request cache that stores
+responses from Coscine for re-use. As a consequence not every
+function provided by the Coscine Python SDK that makes a request
+to Coscine actually sends a request. Some may only send a request
+once and later on revert to the cached response.
+Other users in Coscine may make changes in the meantime, thereby effectively
+invalidating cached data. To make sure that this does not lead to
+inconsistencies, only constant data is cacheable and even that kind of data
+has a limited lifetime in the cache. Once that lifetime ends, the request
+is automatically refreshed on the next access.
+Since constant data in Coscine rarely changes, the cache may be saved
+to file and re-used in later sessions. This can be disabled by the user.
+
+## Initialization and configuration π’
+All we need to get started after we have installed python and the coscine
+package is these few lines of code:
+```python
+import coscine
+
+token = "Our API Token"
+client = coscine.ApiClient(token)
+```
+
+## Parallelization π
+While it does not come with builtin support for parallel requests,
+the Coscine Python SDK can easily be used with concurrency in mind
+using external measures such as a thread pool. The following snippet
+uses the ThreadPoolExecutor type provided by the standard library module
+`concurrent`. In that snippet the `executor` manages 8 threads to which
+it delegates function calls. We no longer have to wait until one function
+returnes but instead can have multiple functions run at the same time -
+in this case 8, since there are 8 threads. By increasing the number of
+threads we can increase the number of operations that we are able to
+do concurrently. However, computing resources are generally scarce and
+we also should not send too many requests at once to Coscine in order
+to not trigger the rate limiter and get a temporary timeout. A reasonable
+measure for the amount of threads could be a low multiple of the amount
+of processor cores that your current machine has to offer or even the
+exact amount of cores. Anything in the order of 2 to 16 threads will
+suffice.
+
+```python
+from concurrent.futures import Future, ThreadPoolExecutor, wait
+
+emails = ["user@example.org", "user2@example.org", ...]
+with ThreadPoolExecutor(8) as executor: # 8 threads
+ futures = []
+ for email in emails:
+ future = executor.submit(project.invite, email)
+ futures.append(future)
+# The tasks are not running in several threads. We are still in the main
+# thread at this point and can either process something else in the
+# meantime or wait for the threads with our function to finish.
+# Once they are finished we can process the results.
+wait(futures)
+# All futures arrived at this point since we were waiting for them
+for future in futures:
+ function_return_value = future.result()
+ print(function_return_value)
+```
+If you need the results returned by these functions you need to wait
+for their futures. Futures are not returned immediately since the function
+is running in another thread. Therefore the main thread does not get
+the result of the function in time and proceeds executing whatever
+comes next. The result therefore arrives in the future. When we are
+done sending stuff to Coscine and want to process the results, we can
+wait for the remaining futures to appear and then process them.
+
+For more information on launching parallel tasks with python refer to
+[the official documentation].
+
+[the official documentation]: https://docs.python.org/3/library/concurrent.futures.html
diff --git a/docs/conf.py b/docs/conf.py
new file mode 100644
index 0000000000000000000000000000000000000000..7a980ad1d23007040565c58bc6bf6b60a5978141
--- /dev/null
+++ b/docs/conf.py
@@ -0,0 +1,48 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+from coscine.__about__ import __author__, __version__
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+project = 'coscine'
+copyright = '2023, RWTH Aachen University'
+author = __author__
+release = __version__
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = [
+ 'sphinx.ext.autodoc', # extracts python docstrings
+ 'sphinx.ext.napoleon', # ability to use numpy-style docstrings
+ 'sphinx.ext.viewcode', # adds a link to the sourcecode in docs
+ 'sphinx.ext.mathjax', # math formula rendering
+ 'sphinx_copybutton', # inserts buttons to copy sourcecode
+ 'myst_parser' # ability to use markdown in place of .rst
+]
+
+templates_path = ['_templates']
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+html_theme = 'furo'
+html_static_path = ['_static']
+
+# based on html_static_path as the root directory
+html_theme_options = {
+ "light_logo": "coscine_logo_rgb.png",
+ "dark_logo": "coscine_logo_white_rgb.png",
+ "source_edit_link": (
+ "https://git.rwth-aachen.de/coscine/community-features"
+ "/coscine-python-sdk/-/tree/master/docs"
+ )
+}
+html_favicon = "_static/coscine_python_sdk_icon_rgb.png"
diff --git a/docs/coscine.rst b/docs/coscine.rst
new file mode 100644
index 0000000000000000000000000000000000000000..bdb727f9cb71f2e162ae17194d03915c1baae4e2
--- /dev/null
+++ b/docs/coscine.rst
@@ -0,0 +1,61 @@
+coscine package
+===============
+
+Submodules
+----------
+
+coscine.client module
+---------------------
+
+.. automodule:: coscine.client
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+coscine.common module
+---------------------
+
+.. automodule:: coscine.common
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+coscine.exceptions module
+-------------------------
+
+.. automodule:: coscine.exceptions
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+coscine.metadata module
+-----------------------
+
+.. automodule:: coscine.metadata
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+coscine.project module
+----------------------
+
+.. automodule:: coscine.project
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+coscine.resource module
+-----------------------
+
+.. automodule:: coscine.resource
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: coscine
+ :members:
+ :undoc-members:
+ :show-inheritance:
diff --git a/docs/index.md b/docs/index.md
new file mode 100644
index 0000000000000000000000000000000000000000..aff9e0b814ddf05cab903def6ec770a155b14d27
--- /dev/null
+++ b/docs/index.md
@@ -0,0 +1,52 @@
+---
+hide-toc: true
+---
+
+# Coscine Python SDK Docs π
+You use the Coscine Python SDK to interact with the Coscine REST API
+from Python. The SDK provides a user-oriented high-level and
+object-oriented API as well as low-level methods for easily extending
+the functionality and implementing custom routines.
+It is aimed at developers of all levels of experience - this specifically
+includes those who may not have comprehensive programming expertise
+and just want to "get stuff done" (i.e. the vast majority
+of people reading this).
+Therefore this documentation provides copy-pastable examples
+specifically tailored for those kinds of people as well as in-depth
+guides for those, who would like to delve deeper into implementation details.
+
+## Recommendations π€
+*Vim, emacs and nano users may want to scroll past this paragraph...*
+It is recommended to use an advanced text editor or integrated development
+environment (IDE) for writing programs with the Coscine Python SDK.
+Tools like [Visual Studio Code] make use of the docstring annotations
+inside of this packages sourcecode and display a detailed description
+for most of the classes and functions exposed by this package directly
+in the editor. Obviously you could also use a simple text editor like
+notepad, but your experience will be severely impacted by your choice of tool.
+
+## Feedback π
+If you stumble upon an error and believe it is related to
+the Coscine Python SDK you are more than welcome to report it.
+Active collaboration, bug reports and feature requests are very welcome.
+To report a bug or to request a feature simply open an issue on
+the projects [issue tracker].
+To contribute features that you have developed yourself and believe
+to benefit everyone if integrated into the SDK you may also open a
+new merge request.
+
+[issue tracker]: https://git.rwth-aachen.de/coscine/community-features/coscine-python-sdk/-/issues
+
+```{toctree}
+:hidden:
+
+installation
+api-token
+client
+project
+resource
+metadata
+API Reference<genindex>
+modindex
+build
+```
diff --git a/docs/installation.md b/docs/installation.md
new file mode 100644
index 0000000000000000000000000000000000000000..88330d574b0e583e1a29748a648b1dad362f55b5
--- /dev/null
+++ b/docs/installation.md
@@ -0,0 +1,55 @@
+# Installation
+### Installing Python π
+Most users do not have Python installed by default, so we begin with the
+installation of Python itself. To check if you already have Python installed,
+open a Command Prompt (on Windows: Win+r and type cmd).
+Once the command prompt is open, type python --version and press Enter.
+If Python is installed, you will see the version of Python printed to the
+screen. If you do not have Python installed, refer to the installation guides
+in the [Hitchhikers Guide to Python]. You must install Python 3.
+Make sure that you choose an [appropriate version] for installation!
+Once Python is installed, you can install the Coscine Python SDK using pip.
+Other useful links on Python:
+- [https://wiki.python.org/moin/BeginnersGuide](https://wiki.python.org/moin/BeginnersGuide)
+- [https://www.python.org/downloads/](https://www.python.org/downloads/)
+
+In the following snippets, depending on your installation and platform, you may have to substitute `python` with `py` or `python3`.
+
+### Installing the Coscine Python SDK
+#### Via the Python Package Index
+The Coscine Python SDK package is hosted on the Python Package Index (PyPi).
+You can install and update it and all of its dependencies via the Python
+Package Manager (pip):
+```bash
+python -m pip install --upgrade coscine
+```
+The `--upgrade` option updates the package if it is already installed
+on your system.
+
+#### Via Conda
+This module's pypi version is mirrored in conda forge. You can install it
+and all of its dependencies via Conda:
+```bash
+conda install -c conda-forge coscine
+```
+and likewise for updating it you would do:
+```bash
+conda update -c conda-forge coscine
+```
+
+#### Via Git
+You can install the python module with a local copy of the source code,
+which you can grab directly from this repository via git:
+```bash
+git clone https://git.rwth-aachen.de/coscine/community-features/coscine-python-sdk.git
+cd ./coscine-python-sdk
+py setup.py
+```
+The `master` branch should contain the latest stable release version.
+If you want to make sure, refer to one of the git tags.
+Development is done in the `dev` branch - the most recent and not always
+100% stable versions can be found there. It is recommended that you stick
+to the master branch for installations.
+
+[Hitchhikers guide to python]: https://docs.python-guide.org/
+[appropriate version]: https://devguide.python.org/versions/
diff --git a/docs/make.bat b/docs/make.bat
new file mode 100644
index 0000000000000000000000000000000000000000..954237b9b9f2b248bb1397a15c055c0af1cad03e
--- /dev/null
+++ b/docs/make.bat
@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+ set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+ echo.
+ echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+ echo.installed, then set the SPHINXBUILD environment variable to point
+ echo.to the full path of the 'sphinx-build' executable. Alternatively you
+ echo.may add the Sphinx directory to PATH.
+ echo.
+ echo.If you don't have Sphinx installed, grab it from
+ echo.https://www.sphinx-doc.org/
+ exit /b 1
+)
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd
diff --git a/docs/metadata.md b/docs/metadata.md
new file mode 100644
index 0000000000000000000000000000000000000000..0ba55444ea472d9ce6c00575fe00f69f7128e8f2
--- /dev/null
+++ b/docs/metadata.md
@@ -0,0 +1 @@
+# Interacting with Metadata
diff --git a/docs/modules.rst b/docs/modules.rst
new file mode 100644
index 0000000000000000000000000000000000000000..d51f9023014decd9a1d059a5702e766052b13d0c
--- /dev/null
+++ b/docs/modules.rst
@@ -0,0 +1,7 @@
+coscine
+=======
+
+.. toctree::
+ :maxdepth: 4
+
+ coscine
diff --git a/docs/project.md b/docs/project.md
new file mode 100644
index 0000000000000000000000000000000000000000..3a13e6b362d3ca3cf401b93b6438c5ef9cf2a5c7
--- /dev/null
+++ b/docs/project.md
@@ -0,0 +1 @@
+# Interacting with Projects
diff --git a/docs/resource.md b/docs/resource.md
new file mode 100644
index 0000000000000000000000000000000000000000..131d006238540729f4e38a0ad930f8b8a81373fc
--- /dev/null
+++ b/docs/resource.md
@@ -0,0 +1 @@
+# Interacting with Resources
diff --git a/requirements.txt b/requirements.txt
index 95ef843586b79a6486718a9b36516fd05c3dea9a..aa47f4c56a1c8defaefc06af1062cfcfb6ffef16 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,9 +1,12 @@
-appdirs
-colorama
-prettytable
-rdflib
-requests
-requests-toolbelt
-setuptools
-tqdm
-python-dateutil
+boto3 # Amazons S3 SDK
+tqdm # CLI progress bars
+pyshacl # RDF SHACL shapes constraints verification
+rdflib # Deals with RDF
+requests # Deals with HTTP
+requests-toolbelt # Multipart upload monitoring
+tabulate # Pretty prints data in tabular format
+python-dateutil # Parses arbitrary datetime strings
+sphinx # documentation generator
+furo # sphinx documentation theme
+myst-parser # sphinx plugin for markdown support
+sphinx-copybutton # inserts buttons to copy sourcecode in docs
diff --git a/setup.py b/setup.py
index def3ea2b52a9ed03140d4ed0748e5105b140d264..5e94f3e162b9da1a106872f83894e01c2d292f0e 100644
--- a/setup.py
+++ b/setup.py
@@ -1,84 +1,72 @@
###############################################################################
# Coscine Python SDK
-# Copyright (c) 2018-2023 RWTH Aachen University
+# Copyright (c) 2020-2023 RWTH Aachen University
# Licensed under the terms of the MIT License
-###############################################################################
-# Coscine, short for Collaborative Scientific Integration Environment, is
-# a platform for research data management (RDM).
# For more information on Coscine visit https://www.coscine.de/.
-#
-# Please note that this python module is open source software primarily
-# developed and maintained by the scientific community. It is not
-# an official service that RWTH Aachen provides support for.
-###############################################################################
-
-###############################################################################
-# File description
-###############################################################################
-
-"""
-This setup script builds and installs the Coscine Python SDK package.
-For a local installation use:
-cd ./coscine-python-sdk
-py -m pip install .
-
-It is usually not necessary to edit this file, since all package metadata
-is defined in 'src/coscine/__about__.py'.
-"""
-
-###############################################################################
-# Dependencies
###############################################################################
import setuptools
-###############################################################################
-# Module globals / Constants
-###############################################################################
-
-# Serves as a container for package metadata.
-# Dynamically loaded from ./src/coscine/about.py
-about = {}
+PROJECT_URL = (
+ "https://git.rwth-aachen.de/coscine/community-features/coscine-python-sdk"
+)
-# Read package metadata into $about
-with open("src/coscine/__about__.py", "r", encoding="utf-8") as fd:
- exec(fd.read(), about)
+README = ""
-# Use README.md as package description for display on PyPi
-# and similar sourcecode hosting platforms.
+# Read README.md into $description to serve as the long_description on PyPi.
# Prepends local links with the repository url to ensure display of
# images and other media on external platforms such as PyPi.
# Local links in README.md are marked with the prefix './data'.
-with open("README.md", "r", encoding="utf-8") as fd:
- readme = fd.read()
- description = readme.replace("./data", \
- about["__url__"] + "-/raw/master/data")
+with open("README.md", "rt", encoding="utf-8") as fp:
+ README = fp.read()
+ README = README.replace("./data", PROJECT_URL + "/-/raw/master/data")
+
+
+# Read package metadata into $about
+about = {}
+with open("src/coscine/__about__.py", "r", encoding="utf-8") as fp:
+ exec(fp.read(), about)
+__author__ = about["__author__"]
+__version__ = about["__version__"]
-###############################################################################
-# Classes / Functions / Scripts
-###############################################################################
setuptools.setup(
- name = about["__title__"],
- version = about["__version__"],
- description = about["__summary__"],
- long_description = description,
+ name = "coscine",
+ version = __version__,
+ description = (
+ "The Coscine Python SDK provides a pythonic high-level "
+ "interface to the Coscine REST API."
+ ),
+ long_description = README,
long_description_content_type = "text/markdown",
- author = about["__author__"],
- author_email = about["__contact__"],
- license = about["__license__"],
+ author = __author__,
+ author_email = "coscine@itc.rwth-aachen.de",
+ license = "MIT License",
packages = setuptools.find_packages(where="src"),
- keywords = about["__keywords__"],
- package_data = {"coscine": ["data/*"]},
- install_requires = about["__dependencies__"],
- url = about["__url__"],
- project_urls = about["__project_urls__"],
+ keywords = ["Coscine", "RWTH Aachen", "Research Data Management"],
+ install_requires = [
+ "boto3",
+ "pyshacl",
+ "python-dateutil",
+ "rdflib",
+ "requests",
+ "requests-toolbelt",
+ "tabulate",
+ "tqdm"
+ ],
+ url = PROJECT_URL,
+ project_urls = {
+ "Issues": PROJECT_URL + "/-/issues",
+ "Documentation": (
+ "https://coscine.pages.rwth-aachen.de/"
+ "community-features/coscine-python-sdk/"
+ )
+ },
classifiers = [
- "Programming Language :: Python :: 3.7",
- "Programming Language :: Python :: 3.8",
- "Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
+ "Programming Language :: Python :: 3.12",
+ "Programming Language :: Python :: 3.13",
"License :: OSI Approved :: MIT License",
"Operating System :: OS Independent",
"Topic :: Scientific/Engineering",
@@ -86,11 +74,9 @@ setuptools.setup(
"Topic :: Software Development :: Libraries :: Python Modules",
"Intended Audience :: Science/Research",
"Intended Audience :: Developers",
- "Development Status :: 4 - Beta",
+ "Development Status :: 5 - Production/Stable",
"Typing :: Typed"
],
package_dir = {"": "src"},
- python_requires = about["__pyver__"]
+ python_requires = ">=3.10"
)
-
-###############################################################################
diff --git a/src/coscine/__about__.py b/src/coscine/__about__.py
index a9705479e43df5e0bc78506f2fb94b2ea3987579..333f26f9b3e7e1dbd64c78026fbaada40387e3e7 100644
--- a/src/coscine/__about__.py
+++ b/src/coscine/__about__.py
@@ -1,86 +1,9 @@
-###############################################################################
-# Coscine Python SDK
-# Copyright (c) 2018-2023 RWTH Aachen University
-# Licensed under the terms of the MIT License
-###############################################################################
-# Coscine, short for Collaborative Scientific Integration Environment, is
-# a platform for research data management (RDM).
-# For more information on Coscine visit https://www.coscine.de/.
-#
-# Please note that this python module is open source software primarily
-# developed and maintained by the scientific community. It is not
-# an official service that RWTH Aachen provides support for.
-###############################################################################
-
-###############################################################################
-# File description
-###############################################################################
-
-"""
-This file contains package metadata primarily intended for setup.py but
-also accessed by various source files in src/.
-"""
-
-###############################################################################
-# Module globals / Constants
-###############################################################################
-
-# Package title/name as it would appear in PyPi
-__title__ = "coscine"
-
-# Current package version
-# Do not set version to 1.0.0 before update to Coscine API version 2
-__version__ = "0.9.4"
-
-# Short package description
-__summary__ = (
- "The Coscine Python SDK provides a pythonic interface to "
- "the Coscine REST API."
-)
-
-# Package copyright owner
-__author__ = "RWTH Aachen University"
-
-# Coscine contact (Note: This is the official Coscine contact email!)
-# Only for copyright claims, licensing issues or sourcecode hosting!
-# Please do not direct bug reports or feature requests to that address!
-# They are not responsible for the development, they "just" represent Coscine!
-__contact__ = "coscine@itc.rwth-aachen.de"
-
-# Package license
-__license__ = "MIT License"
-
-# Package keywords/tags (must be given as a list!)
-__keywords__ = ["Coscine", "RWTH Aachen", "Research Data Management"]
-
-# Package dependencies (must be given as a list!)
-__dependencies__ = [
- "rdflib",
- "requests",
- "requests-toolbelt",
- "tqdm",
- "colorama",
- "prettytable",
- "appdirs",
- "python-dateutil"
-]
-
-# Python version required
-__pyver__ = ">=3.7"
-
-# Project url (official sourcecode hosting)
-__url__ = (
- "https://git.rwth-aachen.de/coscine/community-features/"
- "coscine-python-sdk/"
-)
-
-# Additional urls for e.g. bug reports and documentation
-__project_urls__ = {
- "Issues": __url__ + "-/issues",
- "Documentation": (
- "https://coscine.pages.rwth-aachen.de/"
- "community-features/coscine-python-sdk/"
- )
-}
-
-###############################################################################
+###############################################################################
+# Coscine Python SDK
+# Copyright (c) 2020-2023 RWTH Aachen University
+# Licensed under the terms of the MIT License
+# For more information on Coscine visit https://www.coscine.de/.
+###############################################################################
+
+__author__ = "Romin Benfer"
+__version__ = "1.0.0-alpha"
diff --git a/src/coscine/__init__.py b/src/coscine/__init__.py
index 7ac1cdbb89572e708317ec8b18a7670dfded7602..4ca0a0c1bf00b11c0c64113e65ad63c2a3d1c279 100644
--- a/src/coscine/__init__.py
+++ b/src/coscine/__init__.py
@@ -1,55 +1,25 @@
###############################################################################
# Coscine Python SDK
-# Copyright (c) 2018-2023 RWTH Aachen University
+# Copyright (c) 2020-2023 RWTH Aachen University
# Licensed under the terms of the MIT License
-###############################################################################
-# Coscine, short for Collaborative Scientific Integration Environment, is
-# a platform for research data management (RDM).
# For more information on Coscine visit https://www.coscine.de/.
-#
-# Please note that this python module is open source software primarily
-# developed and maintained by the scientific community. It is not
-# an official service that RWTH Aachen provides support for.
-###############################################################################
-
-###############################################################################
-# File description
###############################################################################
"""
-# Coscine Python SDK
-
-The Coscine Python SDK is an open source python package providing
-a pythonic interface to the Coscine REST API. It is compatible
-with Python versions 3.7+ and has been released under the terms
-of the MIT License.
-
-Please note that this python module is developed and maintained
-by the scientific community and even though Copyright remains with
-RWTH Aachen, it is not an official service that RWTH Aachen
-provides support for.
-
-.. include:: ../docs/getting_started.md
-.. include:: ../docs/usage.md
-.. include:: ../docs/examples.md
-.. include:: ../../CONTRIBUTING.md
+The Coscine Python SDK provides a high-level interface
+to the Coscine REST API.
"""
-###############################################################################
-# Dependencies
-###############################################################################
-
import logging
-from .client import Client, Settings
-from .project import Project, ProjectMember, ProjectForm
-from .resource import Resource, ResourceForm
-from .object import FileObject, MetadataForm
-from .graph import ApplicationProfile
-from .utils import concurrent
-
-###############################################################################
-
-# Set up default logger to '/dev/null'
-logging.getLogger("coscine").addHandler(logging.NullHandler())
-
-###############################################################################
+from coscine.__about__ import *
+from coscine.exceptions import *
+from coscine.client import *
+from coscine.common import *
+from coscine.metadata import *
+from coscine.project import *
+from coscine.resource import *
+
+# Set up logging to /dev/null like a library is supposed to.
+# This ensures that if no logger was configured, all the logging
+# calls made by this library do not yield any output.
+logging.getLogger(__name__).addHandler(logging.NullHandler())
diff --git a/src/coscine/cache.py b/src/coscine/cache.py
deleted file mode 100644
index db6ad91266febb2c0731e94a898594186d76dc29..0000000000000000000000000000000000000000
--- a/src/coscine/cache.py
+++ /dev/null
@@ -1,178 +0,0 @@
-###############################################################################
-# Coscine Python SDK
-# Copyright (c) 2018-2023 RWTH Aachen University
-# Licensed under the terms of the MIT License
-###############################################################################
-# Coscine, short for Collaborative Scientific Integration Environment, is
-# a platform for research data management (RDM).
-# For more information on Coscine visit https://www.coscine.de/.
-#
-# Please note that this python module is open source software primarily
-# developed and maintained by the scientific community. It is not
-# an official service that RWTH Aachen provides support for.
-###############################################################################
-
-###############################################################################
-# File description
-###############################################################################
-
-"""
-This file contains the implementation of a basic persistent cache for
-nonvolatile Coscine data such as metadata vocabularies. That data rarely
-changes (if ever) thus creating a good opportunity for saving bandwidth and
-speed by storing it locally.
-The data is automatically refreshed every once in a while to make sure it
-is still up to date.
-"""
-
-###############################################################################
-# Dependencies
-###############################################################################
-
-from __future__ import annotations
-import atexit
-import json
-import os
-from json.decoder import JSONDecodeError
-from datetime import datetime, timedelta
-from typing import Any, Union
-from coscine.defaults import APPDIR, CACHEFILE, TIMEFORMAT
-
-###############################################################################
-# Classes / Functions / Scripts
-###############################################################################
-
-class Cache:
- """
- A basic persistent cache implementation for temporary storage of
- nonvolatile Coscine data such as vocabulary entries or
- application profiles. The data is automatically refreshed when it
- exceeds a certain age.
- Data is only loaded from & stored on disk if in
- `coscine.Client.settings` the persistent option is enabled.
- The cache file is stored in `coscine.defaults.APPDIR`.
-
- Attributes
- -----------
- _cache : dict
- A simple dictionary to store the cached data.
- _persistent : bool
- Controls cache persistence - if enabled the cache is saved to
- a file and restored in the next session.
- """
-
- _cache: dict
- _persistent: bool
-
-###############################################################################
-
- def __init__(self, persistent: bool = True) -> None:
- """
- Initializes a Cache instance and attempts to load a previous
- cache copy from a file if the $persistent option is set.
-
- Parameters
- ----------
- persistent : bool
- Enable to save the cache in a file upon program exit and
- to restore it in the next session.
- """
-
- self._persistent = persistent
- if persistent:
- self.load()
- atexit.register(self.save)
- else:
- self._cache = {}
-
-###############################################################################
-
- def save(self) -> None:
- """
- Saves the current session cache into a file if
- the $persistent option is set (see __init__()).
- Any cachefile existing prior is overwritten.
- """
-
- if self._persistent and self._cache:
- path = APPDIR.user_cache_dir
- if not os.path.exists(path):
- os.makedirs(path, exist_ok=True)
- filepath = os.path.join(path, CACHEFILE)
- with open(filepath, "w", encoding="utf-8") as file:
- file.write(json.dumps(self._cache))
-
-###############################################################################
-
- def load(self) -> None:
- """
- Loads a previously stored session cache for use with the
- current session. If no previous session is found, the cache
- is intialized as an empty dict.
- """
- try:
- path = APPDIR.user_cache_dir
- filepath = os.path.join(path, CACHEFILE)
- with open(filepath, "r", encoding="utf-8") as file:
- self._cache = json.loads(file.read())
- except (FileNotFoundError, JSONDecodeError):
- self._cache = {}
-
-###############################################################################
-
- def get(self, key: str) -> Union[Any, None]:
- """
- Attempts to read a cached dataset from the cache via a key. If the
- data is contained within the cache but older than 24 days, it is
- updated.
-
- Parameters
- ----------
- key : str
- The key used to save data in the cache (e.g. a URL)
-
- Returns
- --------
- Any
- The cached data for the given key
- None
- If the key is not present in the cache or if the data is
- likely to be outdated
- """
-
- if key in self._cache:
- last_time = datetime.strptime(self._cache[key]["time"], TIMEFORMAT)
- if (datetime.now() - last_time) < timedelta(days=24):
- return self._cache[key]["data"]
- return None
-
-###############################################################################
-
- def set(self, key: str, data: Any) -> None:
- """
- Sets data inside the cache via a key. If that key is already
- present any data referenced by it is overwritten.
-
- Parameters
- ----------
- key : str
- The key used for saving data in the cache (e.g. a URL)
- data : Any
- The data to store inside the cache
- """
-
- self._cache[key] = {
- "time": datetime.now().strftime(TIMEFORMAT),
- "data": data
- }
-
-###############################################################################
-
- def clear(self) -> None:
- """
- Clears the cache
- """
-
- self._cache.clear()
-
-###############################################################################
diff --git a/src/coscine/client.py b/src/coscine/client.py
index 05fd16c387c8da58f427beedf990ba354ad1819d..76c830e22c8eb530594437c112c3e28955f46e4c 100644
--- a/src/coscine/client.py
+++ b/src/coscine/client.py
@@ -1,720 +1,1064 @@
-###############################################################################
-# Coscine Python SDK
-# Copyright (c) 2018-2023 RWTH Aachen University
-# Licensed under the terms of the MIT License
-###############################################################################
-# Coscine, short for Collaborative Scientific Integration Environment, is
-# a platform for research data management (RDM).
-# For more information on Coscine visit https://www.coscine.de/.
-#
-# Please note that this python module is open source software primarily
-# developed and maintained by the scientific community. It is not
-# an official service that RWTH Aachen provides support for.
-###############################################################################
-
-###############################################################################
-# File description
-###############################################################################
-
-"""
-This file contains the backbone of the Coscine Python SDK - the client class.
-The client class acts as the manager of the SDK and is mainly
-responsible for the communication and exchange of information
-with Coscine servers.
-"""
-
-###############################################################################
-# Dependencies
-###############################################################################
-
-from __future__ import annotations
-from typing import List, Optional, Union
-import urllib.parse
-import logging
-import platform
-import requests
-import colorama
-from coscine.__about__ import __version__
-from coscine.cache import Cache
-from coscine.defaults import BASE_URL, LANGUAGES
-from coscine.project import Project, ProjectForm
-from coscine.resource import ResourceForm
-from coscine.utils import in_cli_mode
-from coscine.vocabulary import VocabularyManager
-
-###############################################################################
-# Module globals / Constants
-###############################################################################
-
-# Coscine REST API Endpoints
-API_ENDPOINTS = (
- "Blob", "Metadata", "Organization", "Project", "Resources",
- "Tree", "User", "Search", "ActivatedFeatures", "Notices"
-)
-
-# Get a logger handle.
-logger = logging.getLogger(__name__)
-
-# A colorful pretty banner to show on startup when run in a cli.
-# Note: The noqa comment disables flake8 linter rule E101 for this variable.
-BANNER: str = rf"""{colorama.Fore.BLUE}
- _
- (_)
- ___ ___ ___ ___ _ _ __ ___
- / __/ _ \/ __|/ __| | '_ \ / _ \
- | (_| (_) \__ \ (__| | | | | __/
- \___\___/|___/\___|_|_| |_|\___|{colorama.Fore.WHITE}
-____________________________________
-
- Coscine Python SDK {colorama.Fore.YELLOW}{__version__}{colorama.Fore.WHITE}
-____________________________________
-""" # noqa: E101
-
-###############################################################################
-# Classes / Functions / Scripts
-###############################################################################
-
-
-class Settings:
- """
- Contains settings for configuring the Coscine Client class.
- """
-
- _language: str
- read_only: bool
- concurrent: bool
- persistent_cache: bool
- verbose: bool
-
-###############################################################################
-
- @property
- def language(self) -> str:
- """Returns the language setting"""
- return self._language
-
-###############################################################################
-
- @language.setter
- def language(self, lang: str) -> None:
- lang = lang.lower()
- if lang in LANGUAGES:
- self._language = lang
- else:
- errormessage: str = (
- f"Invalid value for argument 'lang' -> [{lang}]!\n"
- f"Possible values are {str(LANGUAGES)}."
- )
- raise ValueError(errormessage)
-
-###############################################################################
-
- def __init__(
- self,
- language: str = "en",
- persistent_cache: bool = True,
- concurrent: bool = True,
- read_only: bool = False,
- verbose: bool = True
- ) -> None:
- """
- Parameters
- ----------
- language : str, "en" or "de", default: "en"
- Language preset for input form fields and vocabularies.
- persistent_cache : bool, default: True
- Enable to store the cache in a file on deinitialization of
- the client object. Will attempt to load the cache file
- on initialization if enabled. Leads to a significant speed
- boost when making static requests right after init, but may
- also lead to invalid data when using outdated cache data, in case
- the Coscine API changed recently. However this is mostly avoided
- by performing frequent updates. Useful for applications with
- a short runtime that get run often.
- concurrent : bool, default: True
- If enabled, a ThreadPool is used for bulk requests, speeding up
- up- and downloads of multiple files tremendously.
- read_only : bool, default: False
- Do not perform PUT, POST, DELETE requests
- verbose : bool, default: True
- Print stuff to stdout if running from a command line
- """
- self.concurrent = concurrent
- self.read_only = read_only
- self.persistent_cache = persistent_cache
- self.language = language
- self.verbose = verbose
-
-###############################################################################
-###############################################################################
-###############################################################################
-
-class User:
- """
- The Coscine user associated with the Coscine REST API Token.
- """
-
- _data: dict
- _organization: str
-
- def __init__(self, client: Client) -> None:
- uri = client.uri("User", "User", "user")
- self._data = client.get(uri).json()
- uri = client.uri("Organization", "Organization", "-", "isMember")
- self._organization = client.get(uri).json()["data"][0]["displayName"]
-
- @property
- def id(self) -> str:
- return self._data["id"]
-
- @property
- def name(self) -> str:
- return self._data["displayName"]
-
- @property
- def email(self) -> str:
- return self._data["emailAddress"]
-
- @property
- def organization(self) -> str:
- return self._organization
-
-###############################################################################
-###############################################################################
-###############################################################################
-
-class Client:
- """
- The client class handles the connection with the Coscine server.
- It performs requests to the API and returns the response data.
- All objects of the Python SDK use a handle to the client to
- communicate with Coscine.
- """
-
- cache: Cache
- session: requests.Session
- settings: Settings
- vocabularies: VocabularyManager
- _id: int = 0 # unique identifier for each client instance
-
-###############################################################################
-
- @property
- def version(self) -> str:
- """Returns the current Coscine Python SDK version"""
- return __version__
-
-###############################################################################
-
- def __init__(self, token: str, settings: Settings = Settings()) -> None:
- """
- Initializes an instance of the base class of the Coscine Python SDK.
-
- Parameters
- ----------
- token : str
- A Coscine API access token.
- settings : Settings
- Coscine Python SDK client settings.
- """
-
- if not isinstance(token, str):
- raise TypeError("Invalid token type: Expected string!")
-
- self.settings = settings
- self.cache = Cache(self.settings.persistent_cache)
- self.vocabularies = VocabularyManager(self)
- self.session = requests.Session()
- self.session.headers.update({
- "Authorization": f"Bearer {token}",
- "User-Agent": f"Coscine Python SDK {self.version}"
- })
- if in_cli_mode():
- colorama.init(autoreset=True)
- if self.settings.verbose:
- print(BANNER)
- self._id += 1
- logger.info(
- "Initialized Coscine Python SDK version %s "
- "Client instance (id=%d)", __version__, self._id
- )
- logger.debug(self.sysinfo())
- maintenance = self._maintenance_string()
- if maintenance:
- logger.info(maintenance)
-
-###############################################################################
-
- @staticmethod
- def sysinfo() -> str:
- """
- Constructs system information for better bug reports.
-
- Returns
- -------
- str
- Multiline string containing system and python information.
- """
-
- return f"""
- Platform: {platform.platform()}
- Machine: {platform.machine()}
- Processor: {platform.processor()}
- Python compiler: {platform.python_compiler()}
- Python branch: {platform.python_branch()}
- Python implementation: {platform.python_implementation()}
- Python revision: {platform.python_revision()}
- Python version: {platform.python_version()}
- """.replace("\t", " ")
-
-###############################################################################
-
- @staticmethod
- def uri(api: str, endpoint: str, *args) -> str:
- """
- Constructs a URL for performing a request to the Coscine API.
-
- Parameters
- ----------
- api : str
- The target Coscine API endpoint, e.g. Blob, Metadata, Tree, ...
- endpoint : str
- The subendpoint of `api`.
- *args
- Variable number of arguments of type string to append to the URL.
- Arguments are automatically seperated by a slash '/' and
- special characters are encoded.
-
- Raises
- ------
- ValueError
- If the api / endpoint is invalid.
-
- Returns
- -------
- str
- Encoded URL for communicating with the Coscine servers.
- """
-
- if api not in API_ENDPOINTS:
- raise ValueError(
- "Invalid value for argument 'api'! "
- f"Possible values are {str(API_ENDPOINTS)}."
- )
-
- uri = BASE_URL % (api, endpoint)
- for arg in args:
- if arg is None:
- continue
- uri += "/" + urllib.parse.quote(arg, safe="")
- return uri
-
-###############################################################################
-
- def _maintenance_string(self) -> str:
- """
- Returns the Coscine maintenance notice as a human-readable string
- """
- message: str = ""
- notice = self.get_maintenance()
- if notice["type"] is not None:
- message = (
- f"{notice['type']}\n"
- f"{notice['displayName']}\n"
- f"{notice['body']}\n"
- )
- return message
-
-###############################################################################
-
- def get_maintenance(self) -> dict:
- """
- Returns the maintenance status of the Coscine service
- """
- uri = self.uri("Notices", "Notice", "getMaintenance")
- return self.get(uri).json()
-
-###############################################################################
-
- def _request(self, method: str, uri: str, **kwargs) -> requests.Response:
- """
- Performs a HTTP request to the Coscine Servers.
-
- Parameters
- ----------
- method : str
- HTTP request method (GET, PUT, POST, DELETE).
- uri : str
- Coscine URL generated with Client.uri(...).
- **kwargs
- Additional keyword arguments forwarded to the requests library.
-
- Raises
- ------
- ConnectionError
- If the Coscine servers could not be reached.
- PermissionError
- If the Coscine API token is invalid.
- RuntimeError
- If the request resulted in an error.
-
- Returns
- -------
- requests.Response
- The response of the Coscine server as a requests.Response object.
- """
-
- # Debugging URL
- params = kwargs["params"] if "params" in kwargs else None
- full_url = requests.Request(method, uri, params=params).prepare().url
- if full_url:
- logger.debug("HTTP %s: %s", method, full_url)
- else:
- logger.debug("HTTP %s: %s", method, uri)
-
- # Handling read_only setting
- if self.settings.read_only and method != "GET":
- logger.warning("READ_ONLY mode in effect for %s: %s", method, uri)
- return requests.Response()
-
- try: # performing the request and handle any resulting errors
- response = self.session.request(method, uri, **kwargs)
- response.raise_for_status()
- logger.debug("response: %s", str(response.content))
- return response
- except requests.exceptions.ConnectionError as exc:
- raise ConnectionError("Failed to connect to Coscine!") from exc
- except requests.exceptions.RequestException as exc:
- if exc.response.status_code == 401:
- raise PermissionError("Invalid Coscine API token!") from exc
- raise RuntimeError(
- "Unspecified error occurred when communicating "
- "with the Coscine servers"
- ) from exc
-
-###############################################################################
-
- def get(self, uri: str, **kwargs) -> requests.Response:
- """
- Performs a GET request to the Coscine API.
-
- Parameters
- ----------
- uri : str
- Coscine URL generated with Client.uri(...).
- **kwargs
- Additional keyword arguments forwarded to the requests library.
-
- Examples
- --------
- >>> uri = Client.uri("Project", "Project")
- >>> projects = Client.get(uri).json()
-
- Raises
- ------
- ConnectionError
- If the Coscine servers could not be reached.
- PermissionError
- If the Coscine API token is invalid.
- RuntimeError
- If the request resulted in an error.
-
- Returns
- -------
- requests.Response
- The response of the Coscine server as a requests.Response object.
- """
-
- return self._request("GET", uri, **kwargs)
-
-###############################################################################
-
- def put(self, uri: str, **kwargs) -> requests.Response:
- """
- Performs a PUT request to the Coscine API.
-
- Parameters
- ----------
- uri : str
- Coscine URL generated with Client.uri(...).
- **kwargs
- Additional keyword arguments forwarded to the requests library.
-
- Examples
- --------
- >>> uri = Client.uri("Tree", "Tree", resource.id, filename)
- >>> Client.put(uri, data = metadata)
-
- Raises
- ------
- ConnectionError
- If the Coscine servers could not be reached.
- PermissionError
- If the Coscine API token is invalid.
- RuntimeError
- If the request resulted in an error.
-
- Returns
- -------
- requests.Response
- The response of the Coscine server as a requests.Response object.
- """
-
- return self._request("PUT", uri, **kwargs)
-
-###############################################################################
-
- def post(self, uri: str, **kwargs) -> requests.Response:
- """
- Performs a POST request to the Coscine API.
-
- Parameters
- ----------
- uri : str
- Coscine URL generated with Client.uri(...).
- **kwargs
- Additional arguments forwarded to the requests library.
-
- Examples
- --------
- >>> data = member.data
- >>> data["projectId"] = Client.id
- >>> data["role"]["displayName"] = "Member"
- >>> data["role"]["id"] = ProjectMember.ROLES["Member"]
- >>> uri = Client.uri("Project", "ProjectRole")
- >>> Client.post(uri, data=data)
-
- Raises
- ------
- ConnectionError
- If the Coscine servers could not be reached.
- PermissionError
- If the Coscine API token is invalid.
- RuntimeError
- If the request resulted in an error.
-
- Returns
- -------
- requests.Response
- The response of the Coscine server as a requests.Response object.
- """
-
- return self._request("POST", uri, **kwargs)
-
-###############################################################################
-
- def delete(self, uri: str, **kwargs) -> requests.Response:
- """
- Performs a DELETE request to the Coscine API.
-
- Parameters
- ----------
- uri : str
- Coscine URL generated with Client.uri(...).
- **kwargs
- Additional keyword arguments forwarded to the requests library.
-
- Examples
- --------
- >>> uri = Client.uri("Project", "Project", Client.id)
- >>> Client.delete(uri)
-
- Raises
- ------
- ConnectionError
- If the Coscine servers could not be reached.
- PermissionError
- If the Coscine API token is invalid.
- RuntimeError
- If the request resulted in an error.
-
- Returns
- -------
- requests.Response
- The response of the Coscine server as a requests.Response object.
- """
-
- return self._request("DELETE", uri, **kwargs)
-
-###############################################################################
-
- def static_request(self, uri: str) -> dict:
- """
- Performs a GET request for the given uri. If such a request
- has been performed previously during the current session, the previous
- response is returned. Otherwise a new request is made to Coscine
- and that response is then stored inside the session cache.
-
- Parameters
- -----------
- uri : str
- Request URI
-
- Returns
- -------
- dict
- """
-
- logger.debug("static_request(%s)", uri)
- data = self.cache.get(uri)
- if data is None:
- data = self.get(uri).json()
- self.cache.set(uri, data)
- return data
-
-###############################################################################
-
- def user(self) -> User:
- """
- Returns the user associated with the Coscine API Token used by the
- client.
- """
- return User(self)
-
-###############################################################################
-
- def project_form(self, data: Optional[dict] = None) -> ProjectForm:
- """
- Returns an empty project form.
-
- Parameters
- ----------
- data : dict, default: None
- If data is specified, the form is initialized with that data
- using the InputForm.fill() method.
- """
- form = ProjectForm(self)
- if data:
- form.fill(data)
- return form
-
-###############################################################################
-
- def resource_form(self, data: Optional[dict] = None) -> ResourceForm:
- """
- Returns an empty resource form.
-
- Parameters
- ----------
- data : dict, default: None
- If data is specified, the form is initialized with that data
- using the InputForm.fill() method.
- """
- form = ResourceForm(self)
- if data:
- form.fill(data)
- return form
-
-###############################################################################
-
- def projects(self, toplevel: bool = True) -> List[Project]:
- """
- Retrieves a list of a all projects the creator of
- the Coscine API token is currently a member of.
-
- Parameters
- ----------
- toplevel : bool, default: True
- Retrieve only toplevel projects (no subprojects).
- Set to False if you want to retrieve all projects, regardless
- of hierarchy.
-
- Returns
- -------
- List[Project]
- List of coscine.Project objects
- """
-
- endpoint = ("Project", "Project/-/topLevel")
- uri = self.uri("Project", endpoint[toplevel])
- projects = []
- for project_data in self.get(uri).json():
- projects.append(Project(self, project_data))
- return projects
-
-###############################################################################
-
- def project(
- self,
- display_name: str,
- toplevel: bool = True
- ) -> Project:
- """
- Returns a single project via its displayName
-
- Parameters
- ----------
- display_name : str
- Look for a project with the specified displayName
- toplevel : bool, default: True
- Retrieve only toplevel projects (no subprojects).
- Set to False if you want to retrieve all projects, regardless
- of hierarchy.
-
- Returns
- -------
- Project
- A single coscine project handle
-
- Raises
- ------
- IndexError
- In case more than 1 project matches the specified criteria.
- FileNotFoundError
- In case no project with the specified display_name was found.
- """
- filtered_project_list = list(filter(
- lambda project: project.display_name == display_name,
- self.projects(toplevel)
- ))
- if len(filtered_project_list) == 1:
- return filtered_project_list[0]
- if len(filtered_project_list) == 0:
- raise FileNotFoundError(
- f"Found no project with name '{display_name}'!"
- )
- raise IndexError(
- "Received more than 1 project matching the specified "
- f"criteria (display_name = '{display_name}')!"
- )
-
-###############################################################################
-
- def create_project(self, form: Union[ProjectForm, dict]) -> Project:
- """
- Creates a project using the given ProjectForm.
-
- Parameters
- ----------
- form : ProjectForm or dict
- ProjectForm filled with project metadata or a dict with the
- data generated from a form.
-
- Returns
- -------
- Project
- Project object for the new project.
- """
-
- if isinstance(form, ProjectForm):
- form = form.generate()
- uri = self.uri("Project", "Project")
- return Project(self, self.post(uri, json=form).json())
-
-###############################################################################
-
- def search(self, query: str) -> dict:
- """
- Performs a Coscine search query.
-
- Returns
- --------
- dict
- The search results as a dict
- """
-
- uri = self.uri("Search", f"Search?query={query}")
- results = self.get(uri).json()
- return results
-
-###############################################################################
+###############################################################################
+# Coscine Python SDK
+# Copyright (c) 2020-2023 RWTH Aachen University
+# Licensed under the terms of the MIT License
+# For more information on Coscine visit https://www.coscine.de/.
+###############################################################################
+
+"""
+Provides the Coscine ApiClient.
+"""
+
+import atexit
+import json
+import logging
+import requests
+from requests.compat import quote as urlquote
+from requests.utils import get_encoding_from_headers
+from datetime import date, datetime, timedelta
+from io import BytesIO
+from coscine.__about__ import __version__
+import coscine.exceptions
+from coscine.common import (
+ Discipline, Language, License, Organization, User, Visibility
+)
+from coscine.metadata import (
+ ApplicationProfile, ApplicationProfileInfo, Instance, Vocabulary
+)
+from coscine.project import Project, ProjectRole
+from coscine.resource import ResourceType
+
+
+logger = logging.getLogger(__name__)
+
+###############################################################################
+
+class RequestCache:
+ """
+ Basic requests response cache.
+
+ Parameters
+ ----------
+ enable
+ Set to True to enable caching. If set to False, caching is disabled.
+ cachefile
+ Specifies the path to a file to save the cache to and load the
+ cache from in subsequent sessions.
+ """
+
+ _cachefile: str | None
+ _database: dict
+ _enabled: bool
+ _modified: bool
+
+ def __init__(
+ self,
+ enable: bool = True,
+ cachefile: str | None = None
+ ) -> None:
+ self._enabled = enable
+ self._cachefile = cachefile
+ self._modified = False
+ if enable and cachefile:
+ self._database = self.load()
+ atexit.register(self.save)
+ else:
+ self._database = {}
+
+ def save(self) -> None:
+ """
+ Saves the current session database to a file.
+ """
+ if self._modified:
+ with open(self._cachefile, "w", encoding="utf-8") as fp:
+ json.dump(self._database, fp, indent=2)
+
+ def load(self) -> dict:
+ """
+ Loads a previously stored session database into memory.
+ If no previous session is found, an empty database is
+ returned.
+ """
+ try:
+ with open(self._cachefile, "r", encoding="utf-8") as fp:
+ return json.load(fp)
+ except FileNotFoundError:
+ return {}
+
+ def get(self, url: str) -> requests.Response | None:
+ """
+ Attempts to read a cached dataset from the cache via its key. If the
+ data is contained within the cache but older than 30 days, it
+ requires updating.
+ """
+ if self._enabled and url in self._database:
+ entry = self._database.get(url)
+ timestamp: datetime = datetime.fromisoformat(entry["time"])
+ data: dict = entry["data"]
+ if (datetime.now() - timestamp) < timedelta(days=30):
+ response = requests.Response()
+ response.status_code = 200
+ response.code = 200
+ response.raw = BytesIO(json.dumps(data).encode("utf-8"))
+ return response
+ return None
+
+ def set(self, response: requests.Response) -> None:
+ """
+ (Over-)writes data to the cache session.
+ """
+ if self._enabled:
+ self._modified = True
+ self._database[response.url] = {
+ "time": datetime.now().isoformat(),
+ "data": response.json()
+ }
+
+ def clear(self) -> None:
+ """
+ Clear the current cache session of all entries. When persistent
+ caching is enabled and the cachefile gets overwritten at program
+ exit, the cachefile will also be empty, given no more entries
+ have been added to the session since the call to clear.
+ """
+ self._database.clear()
+
+###############################################################################
+
+class MaintenanceNotice:
+ """
+ Models maintenance notices set in Coscine.
+ """
+
+ _data: dict
+
+ @property
+ def title(self) -> str:
+ """
+ The title or name of the maintenance notice.
+ """
+ return self._data.get("displayName")
+
+ @property
+ def link(self) -> str:
+ """
+ The URL link to the detailed maintenance notice.
+ """
+ return self._data.get("href")
+
+ @property
+ def type(self) -> str:
+ """
+ The type of maintenance.
+ """
+ return self._data.get("type")
+
+ @property
+ def body(self) -> str:
+ """
+ The body or description of the notice.
+ """
+ return self._data.get("body")
+
+ @property
+ def starts_date(self) -> date:
+ """
+ Date when the maintenance goes active.
+ """
+ return date.fromisocalendar(self._data.get("startsDate"))
+
+ @property
+ def ends_date(self) -> date:
+ """
+ Date when the maintenance ends.
+ """
+ return date.fromisocalendar(self._data.get("endsDate"))
+
+ def __init__(self, data: dict) -> None:
+ self._data = data
+
+ def __str__(self) -> str:
+ return self.title
+
+###############################################################################
+
+class SearchResult:
+ """
+ This class models the search results returned by Coscine upon
+ a search request.
+ """
+
+ _data: dict
+
+ @property
+ def uri(self) -> str:
+ """
+ Link to the result (i.e. a project or resource or file).
+ """
+ return self._data.get("uri")
+
+ @property
+ def type(self) -> str:
+ """
+ The search category the result falls into (e.g. project).
+ """
+ return self._data.get("type")
+
+ @property
+ def source(self) -> str:
+ """
+ The source text that matches in some way or another the search query.
+ """
+ return self._data.get("source")
+
+ def __init__(self, data: dict) -> None:
+ self._data = data
+
+ def __str__(self) -> str:
+ return self.uri
+
+###############################################################################
+
+class ApiToken:
+ """
+ This class models the Coscine API token.
+ """
+
+ _data: dict
+
+ @property
+ def id(self) -> str:
+ """
+ Unique Coscine-internal identifier for the API token.
+ """
+ return self._data.get("id")
+
+ @property
+ def name(self) -> str:
+ """
+ The name assigned to the API token by the creator upon creation.
+ """
+ return self._data.get("name")
+
+ @property
+ def created(self) -> date:
+ """
+ Timestamp of when the API token was created.
+ """
+ return date.fromisocalendar(self._data.get("creationDate"))
+
+ @property
+ def expires(self) -> date:
+ """
+ Timestamp of when the API token will expire.
+ """
+ return date.fromisocalendar(self._data.get("expiryDate"))
+
+ @property
+ def expired(self) -> bool:
+ """
+ Evaluates to True if the API token is expired.
+ """
+ return datetime.now() > self.expires
+
+ @property
+ def owner(self) -> str:
+ """
+ Unique Coscine-internal user id of the owner of the API token.
+ """
+ return self._data.get("owner")["id"]
+
+ def __init__(self, data: dict) -> None:
+ self._data = data
+
+ def __str__(self) -> str:
+ return self.name
+
+###############################################################################
+
+class ApiClient:
+ """
+ An ApiClient is responsible for communicating with Coscine and
+ allows you to send requests and receive data from Coscine.
+
+ Parameters
+ -----------
+ token : str
+ To be able to use the Coscine REST API one has to supply their
+ Coscine API token. Every Coscine user can create their own set
+ of API tokens for free in their Coscine user profile.
+ For security reasons Coscine API Tokens are only valid for
+ a certain amount of time after which they are deactivated.
+ language : str
+ Just like in the web interface of Coscine one can select
+ a language preset for the Coscine API. This will localize
+ all multi-language vocabularies and application profiles to the
+ selected language. The language can later be switched on the fly.
+ base_url : str
+ Coscine is Open Source software and hosted on various domains.
+ Via the base_url setting, the API user can specify which instance
+ they would like to connect to. By default this is set to
+ the Coscine instance of RWTH Aachen.
+ verify : bool
+ Whether to verify the SSL server certificate of the Coscine server.
+ By default this is enabled as it provides some form of protection
+ against spoofing, but on test instances or "fresh" installs
+ certificates are often not used. To be able to use
+ the Coscine Python SDK with such instances, the verify setting
+ should be turned off.
+ verbose : bool
+ By disabling the verbose setting one can stop the Python SDK
+ from printing to the command line interface (stdout). The stderr
+ file handle is unaffected by this. This setting is particulary
+ helpful in case you wish to disable the banner on initialization.
+ caching : bool
+ Enabling caching allows the Python SDK to store some of
+ the responses it gets from Coscine in a RequestCache. This cache
+ is always active at runtime but may be saved and loaded to a file
+ by enabling the caching setting. Entries in the cachefiles
+ are valid for a certain amount of time until they are refreshed.
+ With caching enabled the Python SDK is much faster.
+ timeout : float (seconds)
+ The timeout threshold for Coscine to respond to a request.
+ If Coscine does not answer within the specified amount of
+ seconds, an exception is raised. Note that setting a timeout
+ is very important since otherwise your application may hang
+ indefinitely if it never gets a response.
+ """
+
+ BANNER: str = (
+ r" _ " "\n"
+ r" (_) " "\n"
+ r" ___ ___ ___ ___ _ _ __ ___ " "\n"
+ r" / __/ _ \/ __|/ __| | '_ \ / _ \ " "\n"
+ r" | (_| (_) \__ \ (__| | | | | __/ " "\n"
+ r" \___\___/|___/\___|_|_| |_|\___| " "\n"
+ r" ___________________________________ " "\n"
+ f" Coscine Python SDK {__version__} " "\n"
+ r" https://coscine.de/ " "\n"
+ )
+
+ base_url: str
+ cache: RequestCache
+ session: requests.Session
+ verbose: bool
+ verify: bool
+ timeout: float
+
+ _language: str
+
+ @property
+ def language(self) -> str:
+ """
+ The language setting of the ApiClient.
+ This may be set to "en" for english or "de" for german.
+ By default it is set to english but it can be changed
+ on the fly even after the ApiClient has been instantiated.
+ """
+ return self._language
+
+ @language.setter
+ def language(self, value: str) -> None:
+ language = value.lower()
+ if language not in ("de", "en"):
+ raise coscine.exceptions.ValueError(
+ f"Invalid language value: {value}! "
+ "Acceptable values are: 'de' or 'en'."
+ )
+ self._language = language
+
+ @property
+ def version(self) -> str:
+ """
+ Coscine Python SDK version string. For example: "1.0.0"
+ """
+ return __version__
+
+ def __init__(
+ self,
+ token: str,
+ language: str = "en",
+ base_url: str = "https://coscine.rwth-aachen.de",
+ caching: bool = True,
+ cachefile: str = "./cache.json",
+ verbose: bool = True,
+ verify: bool = True,
+ timeout: float = 60.0
+ ) -> None:
+ self.base_url = base_url
+ self.cache = RequestCache(caching, cachefile)
+ self.language = language
+ self.timeout = timeout
+ self.session = requests.Session()
+ self.session.headers.update({
+ "Authorization": f"Bearer {token}",
+ "User-Agent": f"Coscine Python SDK {self.version}"
+ })
+ self.verbose = verbose
+ self.verify = verify
+ if self.verbose:
+ print(ApiClient.BANNER)
+
+ def latest_version(self) -> str:
+ """
+ Retrieves the version string of the latest version of this
+ package hosted on PyPi. Useful for checking whether the currently
+ used version is outdated and if an update should be performed.
+
+ Examples
+ --------
+ >>> if client.version != client.latest_version():
+ >>> print("Module outdated.")
+ >>> print("Run 'py -m pip install --upgrade coscine'.")
+
+ """
+ uri = "https://pypi.org/pypi/coscine/json"
+ data = requests.get(uri).json()
+ version = data["info"]["version"]
+ return version
+
+ def uri(self, *args) -> str:
+ """
+ Constructs a URI for requests to the Coscine REST API.
+ This method creates URLs relative to the ApiClient.base_url
+ and escapes URL arguments for compliance with the HTTP.
+
+ Parameters
+ -----------
+ *args
+ Any number of arguments that should be included in the URI.
+ The arguments do not have to be of type string, but should
+ be str() serializable.
+
+ Examples
+ --------
+ >>> ApiClient.uri("application-profiles", "profiles", profile_uri)
+ """
+ suffix = "/".join((urlquote(str(arg), safe="") for arg in args))
+ return f"{self.base_url}/coscine/api/v2/{suffix}"
+
+ def request(
+ self,
+ method: str,
+ *args,
+ stream: bool = False,
+ cacheable: bool = False,
+ **kwargs
+ ) -> requests.Response:
+ """
+ Sends a request to the Coscine REST API. This method is used
+ internally. As a user of the ApiClient you should use the methods
+ ApiClient.get(), ApiClient.post(), ApiClient.put(),
+ ApiClient.delete(), ApiClient.options()
+ instead of directly calling ApiClient.request().
+
+ Parameters
+ ----------
+ method
+ The HTTP method to use for the request:
+ GET, PUT, POST, DELETE, OPTIONS, etc.
+ *args
+ Any number of arguments to forward to the requests.Request()
+ stream
+ If set to true, the response will be streamed. This means that
+ the response will be split up and arrive in multiple chunks.
+ When attempting to download files, the stream parameter must be
+ set to True. Otherwise it should be left at False.
+ cacheable
+ To speed up requests, certain responses can and should be cached.
+ As a user of the ApiClient you should NOT touch this option!
+ Not all responses are eligible for caching! Only those that return
+ constant data or data that seldomly changes should be cached.
+ Cached responses have a limited lifetime until they are refreshed.
+ This option does not have any effect if ApiClient.caching is False.
+ *kwargs
+ Any number of keyword arguments to forward to requests.Request()
+
+ Raises
+ ------
+ coscine.exceptions.ConnectionError
+ If the request never reaches Coscine or we never get a response.
+ coscine.exceptions.AuthenticationError
+ If the Coscine API token is not valid.
+ coscine.exceptions.RequestRejected
+ If the request reached Coscine but was subsequently rejected
+ by the Server.
+ """
+ try:
+ request = requests.Request(method, *args, **kwargs)
+ request = self.session.prepare_request(request)
+ if request.body:
+ encoding = get_encoding_from_headers(request.headers)
+ body = (
+ request.body.decode(encoding)
+ if encoding else "<binary data>"
+ )
+ logger.debug(body)
+ if cacheable and method == "GET":
+ response = self.cache.get(request.url)
+ if response:
+ return response
+ response = self.session.send(
+ request,
+ stream=stream,
+ verify=self.verify,
+ timeout=self.timeout
+ )
+ logger.debug(response.content.decode("utf-8"))
+ response.raise_for_status()
+ if cacheable and method == "GET":
+ self.cache.set(response)
+ return response
+ except requests.exceptions.ConnectionError as err:
+ raise coscine.exceptions.ConnectionError(
+ "Failed to connect to Coscine! Check your internet "
+ "connection or whether Coscine is currently down."
+ ) from err
+ except requests.exceptions.Timeout as err:
+ raise coscine.exceptions.ConnectionError(
+ "The connection timed out. This may either be "
+ "a ConnectionError or Coscine took a lot of time "
+ "processing the request. You can increase the timeout "
+ "threshold of the SDK in the client settings."
+ ) from err
+ except requests.exceptions.RequestException as err:
+ if err.response.status_code == 401:
+ raise coscine.exceptions.AuthenticationError(
+ "Invalid Coscine API token! The token was rejected "
+ "by Coscine. Check whether it is expired."
+ ) from err
+ raise coscine.exceptions.RequestRejected(
+ "Coscine rejected the request sent by the Coscine Python SDK "
+ "with the following error message: "
+ f"{response.content.decode('utf-8')}."
+ ) from err
+
+ def get(self, *args, **kwargs) -> dict:
+ """
+ Sends a GET request to the Coscine REST API.
+ For a list of exceptions that are raised by this method
+ or a detailed list of parameters, refer to ApiClient.request().
+
+ Parameters
+ ----------
+ Refer to ApiClient.request() for a list of parameters.
+
+ Raises
+ -------
+ Refer to ApiClient.request() for a list of exceptions.
+
+ Returns
+ --------
+ dict
+ The "data": { ... } section of the JSON-response.
+ """
+ response = self.request("GET", *args, **kwargs).json()
+ return response.get("data")
+
+ def get_pages(self, *args, **kwargs) -> list:
+ """
+ Sends a GET request to the Coscine REST API and expects
+ a paginated response.
+
+ Parameters
+ ----------
+ Refer to ApiClient.request() for a list of parameters.
+
+ Raises
+ -------
+ Refer to ApiClient.request() for a list of exceptions.
+
+ Yields
+ -------
+ Returns a generator that yields the paginated responses.
+
+ Examples
+ --------
+ >>> response = ApiClient.get_pages(uri)
+ >>> full_data = [data for page in response for data in page]
+ """
+ if "params" not in kwargs:
+ kwargs["params"] = {}
+ kwargs["params"]["PageNumber"] = 1
+ kwargs["params"]["PageSize"] = 50
+ response = self.request("GET", *args, **kwargs).json()
+ yield response.get("data")
+ while "pagination" in response and response["pagination"]["hasNext"]:
+ kwargs["params"]["PageNumber"] += 1
+ response = self.request("GET", *args, **kwargs).json()
+ yield response.get("data")
+ if kwargs["params"]["PageNumber"] > 20:
+ break # Protection against accidential flooding
+
+ def put(self, *args, **kwargs) -> None:
+ """
+ Sends a PUT request to the Coscine REST API.
+
+ Parameters
+ ----------
+ Refer to ApiClient.request() for a list of parameters.
+
+ Raises
+ -------
+ Refer to ApiClient.request() for a list of exceptions.
+ """
+ self.request("PUT", *args, **kwargs)
+
+ def post(self, *args, **kwargs) -> dict | None:
+ """
+ Sends a POST request to the Coscine REST API.
+
+ Parameters
+ ----------
+ Refer to ApiClient.request() for a list of parameters.
+
+ Raises
+ -------
+ Refer to ApiClient.request() for a list of exceptions.
+ """
+ response = self.request("POST", *args, **kwargs)
+ try:
+ return response.json().get("data")
+ except requests.exceptions.JSONDecodeError as err:
+ return None
+
+ def delete(self, *args, **kwargs) -> None:
+ """
+ Sends a DELETE request to the Coscine REST API.
+
+ Parameters
+ ----------
+ Refer to ApiClient.request() for a list of parameters.
+
+ Raises
+ -------
+ Refer to ApiClient.request() for a list of exceptions.
+ """
+ self.request("DELETE", *args, **kwargs)
+
+ def options(self, *args, **kwargs) -> None:
+ """
+ Sends an OPTIONS request to the Coscine REST API.
+
+ Parameters
+ ----------
+ Refer to ApiClient.request() for a list of parameters.
+
+ Raises
+ -------
+ Refer to ApiClient.request() for a list of exceptions.
+ """
+ self.request("OPTIONS", *args, **kwargs)
+
+ def projects(self, toplevel: bool = True) -> list[Project]:
+ """
+ Retrieves a list of all Coscine projects that the creator of
+ the Coscine API token is currently a member of.
+
+ Parameters
+ -----------
+ toplevel
+ If set to True, only toplevel projects are retrieved.
+ Set it to False to include all (sub-)projects in the results.
+ """
+ uri = self.uri("projects")
+ response = self.get_pages(uri, params={"TopLevel": toplevel})
+ return [Project(self, data) for page in response for data in page]
+
+ def project(
+ self,
+ key: str,
+ property: property = Project.display_name,
+ toplevel: bool = True
+ ) -> Project:
+ """
+ Returns a single Coscine Project via one of its properties.
+
+ Parameters
+ ----------
+ key
+ The value of the property to filter by.
+ property
+ The property/attribute of the project to filter by.
+ toplevel
+ If set to True, only toplevel projects are searched.
+ Set it to False to include all (sub-)projects in the search.
+
+ Raises
+ ------
+ coscine.exceptions.IndistinguishableError
+ In case more than 1 project was found via the selected property.
+ coscine.exceptions.NotFoundError
+ In case the project could not be found via the selected property.
+ """
+ results = list(filter(
+ lambda project: project.match(property, key),
+ self.projects(toplevel)
+ ))
+ if len(results) > 1:
+ raise coscine.exceptions.IndistinguishableError(
+ "Found more than 1 project with a property matching "
+ f"the key '{key}'. "
+ "Certain project properties such as the name "
+ "allow for duplicates among other projects. "
+ "Use a different (unique) property to filter by!"
+ )
+ elif len(results) == 0:
+ raise coscine.exceptions.NotFoundError(
+ f"Failed to find a project via the key '{key}'! "
+ "Maybe you are looking for a subproject and have "
+ "set the toplevel argument to False? Also check whether "
+ "you are filtering by the correct property."
+ )
+ return results[0]
+
+ def create_project(
+ self,
+ name: str,
+ display_name: str,
+ description: str,
+ start_date: date,
+ end_date: date,
+ principal_investigators: str,
+ disciplines: list[Discipline],
+ organizations: list[Organization],
+ visibility: Visibility,
+ keywords: list[str] = [],
+ grant_id: str = ""
+ ) -> Project:
+ """
+ Creates a new Coscine project.
+
+ Parameters
+ ----------
+ name
+ The project's name.
+ display_name
+ The project's display name (how it appears in the web interface).
+ description
+ The project description.
+ start_date
+ Date when the project starts.
+ end_date
+ Date when the project ends.
+ principal_investigators
+ The project PIs.
+ disciplines
+ List of associated scientific disciplines.
+ organizations
+ List of organizations partaking in the project.
+ visibility
+ Project metadata visibility (relevant for search).
+ keywords
+ List of project keywords (relevant for search).
+ grant_id
+ The projects grant ID.
+ """
+ data: dict = {
+ "name": name,
+ "displayName": display_name,
+ "description": description,
+ "startDate": start_date.isoformat(),
+ "endDate": end_date.isoformat(),
+ "principleInvestigators": principal_investigators,
+ "disciplines": [{
+ "id": discipline.id,
+ "uri": discipline.uri,
+ "displayNameEn": discipline.name
+ } for discipline in disciplines
+ ],
+ "organizations": [{
+ "uri": organization.uri
+ } for organization in organizations
+ ],
+ "visibility": {
+ "id": visibility.id
+ },
+ "keywords": keywords,
+ "grantId": grant_id
+ }
+ uri = self.uri("projects")
+ return Project(self, self.post(uri, json=data))
+
+ def search(self, query: str, category: str = None) -> list[SearchResult]:
+ """
+ Sends a search request to Coscine and returns the results.
+
+ Parameters
+ -----------
+ query
+ The search query
+ category
+ The search can optionally be restricted to one of these
+ categories: "metadata", "project" or "resource"
+ """
+ uri = self.uri("search")
+ parameters = {
+ "Query": query,
+ "category": category
+ }
+ response = self.get(uri, params=parameters)
+ return [SearchResult(data) for data in response]
+
+ def maintenances(self) -> list[MaintenanceNotice]:
+ """
+ Retrieves the list of current active maintenance notices for Coscine.
+ """
+ uri = self.uri("maintenances")
+ response = self.get(uri)
+ return [MaintenanceNotice(data) for data in response]
+
+ def visibilities(self) -> list[Visibility]:
+ """
+ Retrieves the list of visibility options available in Coscine.
+ """
+ uri = self.uri("visibilities")
+ response = self.get(uri, cacheable=True)
+ return [Visibility(data) for data in response]
+
+ def visibility(self, name: str) -> Visibility:
+ """
+ Returns the visibility that matches the name.
+ Valid names are:
+ * "Project Members"
+ * "Public"
+ """
+ results = list(filter(
+ lambda visibility: visibility.name == name,
+ self.visibilities()
+ ))
+ if len(results) > 1:
+ raise coscine.exceptions.IndistinguishableError(
+ f"Found more than 1 visibility with the name '{name}'!"
+ )
+ elif len(results) == 0:
+ raise coscine.exceptions.NotFoundError(
+ f"Failed to find a visibility with the name '{name}'!"
+ )
+ return results[0]
+
+ def disciplines(self) -> list[Discipline]:
+ """
+ Retrieves the list of scientific disciplines available in Coscine.
+ """
+ uri = self.uri("disciplines")
+ response = self.get_pages(uri, cacheable=True)
+ return [Discipline(data) for page in response for data in page]
+
+ def discipline(self, name: str) -> Discipline:
+ """
+ Returns the discipline that matches the name.
+ Valid names would be for example:
+ * "Jurisprudence 113"
+ * "Materials Science 406"
+ * "Medicine 205"
+ * "Computer Science 409"
+ * ...
+ """
+ results = list(filter(
+ lambda discipline: discipline.name == name,
+ self.disciplines()
+ ))
+ if len(results) > 1:
+ raise coscine.exceptions.IndistinguishableError(
+ f"Found more than 1 discipline with the name '{name}'!"
+ )
+ elif len(results) == 0:
+ raise coscine.exceptions.NotFoundError(
+ f"Failed to find a discipline with the name '{name}'!"
+ )
+ return results[0]
+
+ def api_tokens(self) -> list[ApiToken]:
+ """
+ Retrieves the list of Coscine API tokens that have been created
+ by the owner of the same API token that was used to initialize
+ the Coscine Python SDK ApiClient.
+ """
+ uri = self.uri("self", "api-tokens")
+ response = self.get_pages(uri)
+ return [ApiToken(data) for page in response for data in page]
+
+ def self(self) -> User:
+ """
+ Returns the owner of the Coscine API token that was used to
+ initialize the ApiClient.
+ """
+ if hasattr(self, "_self"):
+ return self._self
+ userdata = self.get(self.uri("self"))
+ self._self = User(userdata)
+ return self._self
+
+ def users(self, query: str) -> list[User]:
+ """
+ Searches for users.
+ """
+ response = self.get_pages(self.uri("users"), params={"SearchTerm": query})
+ return [User(data) for page in response for data in page]
+
+ def application_profile(self, profile_uri: str) -> ApplicationProfile:
+ """
+ Retrieves a specific application profile via its uri.
+
+ Parameters
+ ----------
+ profile_uri
+ The uri of the application profile,
+ e.g. https://purl.org/coscine/ap/base
+ """
+ # Since we are parsing the ApplicationProfile right away as ttl
+ # there is not point in fetching it in any other format or letting
+ # the user decide the format.
+ format: str = "text/turtle"
+ uri = self.uri("application-profiles", "profiles", profile_uri)
+ data = self.get(uri, cacheable=True, params={"format": format})
+ return ApplicationProfile(self, data)
+
+ def application_profiles(self) -> list[ApplicationProfileInfo]:
+ """
+ Retrieves the list of all application profiles that are currently
+ available in Coscine.
+ """
+ uri = self.uri("application-profiles", "profiles")
+ response = self.get_pages(uri, cacheable=True)
+ return [
+ ApplicationProfileInfo(data)
+ for page in response for data in page
+ ]
+
+ def vocabulary(self, class_uri: str) -> Vocabulary:
+ """
+ Retrieves the vocabulary for the class.
+
+ Parameters
+ ----------
+ class_uri
+ The instance class uri, e.g. http://purl.org/dc/dcmitype
+ """
+ uri = self.uri("vocabularies", "instances")
+ params = {"Class": class_uri, "Language": self.language}
+ response = self.get_pages(uri, cacheable=True, params=params)
+ instances = [
+ Instance(data)
+ for page in response for data in page
+ ]
+ return Vocabulary(instances)
+
+ def languages(self) -> list[Language]:
+ """
+ Retrieves all languages available in Coscine.
+ """
+ uri = self.uri("languages")
+ response = self.get_pages(uri, cacheable=True)
+ return [Language(data) for page in response for data in page]
+
+ def organization(self, ror_uri: str) -> Organization:
+ """
+ Looks up an organization based on its ror uri.
+ """
+ uri = self.uri("organizations", ror_uri)
+ response = self.get(uri, cacheable=True)
+ return Organization(response)
+
+ def licenses(self) -> list[License]:
+ """
+ Retrieves a list of all licenses available in Coscine.
+ """
+ uri = self.uri("licenses")
+ response = self.get_pages(uri, cacheable=True)
+ return [License(data) for page in response for data in page]
+
+ def license(self, name: str) -> License:
+ """
+ Returns the license that matches the name.
+ """
+ results = list(filter(
+ lambda license: license.name == name,
+ self.licenses()
+ ))
+ if len(results) > 1:
+ raise coscine.exceptions.IndistinguishableError(
+ f"Found more than 1 licenses with the name '{name}'!"
+ )
+ elif len(results) == 0:
+ raise coscine.exceptions.NotFoundError(
+ f"Failed to find a license with the name '{name}'!"
+ )
+ return results[0]
+
+ def roles(self) -> list[ProjectRole]:
+ """
+ """
+ response = self.get(self.uri("roles"))
+ return [ProjectRole(item) for item in response]
+
+ def role(self, name: str) -> ProjectRole:
+ """
+ Returns the role that matches the name.
+ """
+ results = list(filter(
+ lambda role: role.name == name,
+ self.roles()
+ ))
+ if len(results) > 1:
+ raise coscine.exceptions.IndistinguishableError(
+ f"Found more than 1 role with the name '{name}'!"
+ )
+ elif len(results) == 0:
+ raise coscine.exceptions.NotFoundError(
+ f"Failed to find a role with the name '{name}'!"
+ )
+ return results[0]
+
+ def resource_types(self) -> list[ResourceType]:
+ """
+ Retrieves a list of all resource types available in Coscine.
+ """
+ uri = self.uri("resource-types", "types")
+ response = self.get_pages(uri, cacheable=True)
+ return [ResourceType(data) for page in response for data in page]
+
+ def resource_type(self, name: str) -> ResourceType:
+ """
+ Returns the ResourceType that matches the name.
+ Here name refers to the resource specificType,
+ e.g. "rdsrwth" instead of the general type "rds".
+ Mapping between specificType -> generalType:
+ * "rdss3rwth" -> "rdss3"
+ * "linked" -> "linked"
+ * "rdss3wormrwth" -> "rdss3worm"
+ * "rdsrwth" -> "rds"
+ * "rdstudo" -> "rds"
+ * "gitlab" -> "gitlab"
+ * "rdss3nrw" -> "rdss3"
+ * "rdss3ude" -> "rdss3"
+ * "rdsude" -> "rds"
+ * "rdss3tudo" -> "rdss3"
+ * "rdsnrw" -> "rds"
+ """
+ results = list(filter(
+ lambda rtype: rtype.specificType == name,
+ self.resource_types()
+ ))
+ if len(results) > 1:
+ raise coscine.exceptions.IndistinguishableError(
+ f"Found more than 1 resource types with the name '{name}'!"
+ )
+ elif len(results) == 0:
+ raise coscine.exceptions.NotFoundError(
+ f"Failed to find a resource type with the name '{name}'!"
+ )
+ return results[0]
+
+ def validate_pid(self, pid: str) -> bool:
+ """
+ Checks the given PID for validity.
+ """
+ prefix, id = pid.split("/")
+ uri = self.uri("pids", prefix, id)
+ try:
+ response = self.get(uri)
+ return response.get("isValid")
+ except coscine.exceptions.CoscineException:
+ return False
diff --git a/src/coscine/common.py b/src/coscine/common.py
new file mode 100644
index 0000000000000000000000000000000000000000..7c661d77c2e472097b726602a2acc23734d157cc
--- /dev/null
+++ b/src/coscine/common.py
@@ -0,0 +1,275 @@
+###############################################################################
+# Coscine Python SDK
+# Copyright (c) 2020-2023 RWTH Aachen University
+# Licensed under the terms of the MIT License
+# For more information on Coscine visit https://www.coscine.de/.
+###############################################################################
+
+"""
+Provides common classes shared among multiple modules.
+"""
+
+###############################################################################
+
+class Language:
+ """
+ Models the languages available in Coscine.
+ """
+
+ _data: dict
+
+ @property
+ def id(self) -> str:
+ """
+ Unique and constant Coscine internal identifier for
+ the respective language option.
+ """
+ return self._data.get("id")
+
+ @property
+ def name(self) -> str:
+ """
+ The full name of the language option.
+ """
+ return self._data.get("displayName")
+
+ @property
+ def abbreviation(self) -> str:
+ """
+ The abbreviated name of the language option.
+ """
+ return self._data.get("abbreviation")
+
+ def __init__(self, data: dict) -> None:
+ self._data = data
+
+ def __str__(self) -> str:
+ return self.name
+
+###############################################################################
+
+class AcademicTitle:
+ """
+ Models the Academic Titles available in Coscine.
+ """
+
+ _data: dict
+
+ @property
+ def id(self) -> str:
+ """
+ Unique and constant Coscine internal identifier for
+ the respective Academic Title.
+ """
+ return self._data.get("id")
+
+ @property
+ def name(self) -> str:
+ """
+ The name of the Academic Title, e.g. "Prof." or "Dr."
+ """
+ return self._data.get("displayName")
+
+ def __init__(self, data: dict) -> None:
+ self._data = data
+
+ def __str__(self) -> str:
+ return self.name
+
+###############################################################################
+
+class Discipline:
+ """
+ Models the disciplines available in Coscine.
+ """
+
+ _data: dict
+
+ @property
+ def id(self) -> str:
+ """
+ The Coscine-internal unique identifier for the discipline.
+ """
+ return self._data.get("id")
+
+ @property
+ def uri(self) -> str:
+ """
+ The uri of the discipline.
+ """
+ return self._data.get("uri")
+
+ @property
+ def name(self) -> str:
+ """
+ The human-readable name of the discipline.
+ """
+ return self._data.get("displayNameEn")
+
+ def __init__(self, data: dict) -> None:
+ self._data = data
+
+ def __str__(self) -> str:
+ return self.name
+
+###############################################################################
+
+class License:
+ """
+ Models the licenses available in Coscine.
+ """
+
+ _data: dict
+
+ @property
+ def id(self) -> str:
+ """
+ The Coscine-internal unique identifier for the license.
+ """
+ return self._data.get("id") or ""
+
+ @property
+ def name(self) -> str:
+ """
+ The human-readable name of the license.
+ """
+ return self._data.get("displayName") or ""
+
+ def __init__(self, data: dict) -> None:
+ self._data = data
+
+ def __str__(self) -> str:
+ return self.name
+
+###############################################################################
+
+class Organization:
+ """
+ Models organization information for organizations in Coscine.
+ """
+
+ _data: dict
+
+ @property
+ def uri(self) -> str:
+ """
+ The organization's ror uri.
+ """
+ return self._data.get("uri")
+
+ @property
+ def name(self) -> str:
+ """
+ The full name of the organization.
+ """
+ return self._data.get("displayName")
+
+ @property
+ def email(self) -> str:
+ """
+ Contact email address of the organization.
+ """
+ return self._data.get("email")
+
+ def __init__(self, data: dict) -> None:
+ self._data = data
+
+ def __str__(self) -> str:
+ return self.name
+
+###############################################################################
+
+class User:
+ """
+ This class provides an interface around userdata in Coscine.
+ """
+
+ _data: dict
+
+ @property
+ def id(self) -> str:
+ """
+ The unique Coscine-internal user id for a user.
+ """
+ return self._data.get("id")
+
+ @property
+ def display_name(self) -> str:
+ """
+ The full name of a Coscine user as displayed
+ in the Coscine web interface.
+ """
+ return self._data.get("displayName")
+
+ @property
+ def first_name(self) -> str:
+ """
+ The first name of a Coscine user.
+ """
+ return self._data.get("givenName")
+
+ @property
+ def last_name(self) -> str:
+ """
+ The family name of a Coscine user.
+ """
+ return self._data.get("familyName")
+
+ @property
+ def email(self) -> str | list[str] | None:
+ """
+ The email address or list of email addresses of a user.
+ In case the user has not associated an email address with their
+ account 'None' is returned.
+ """
+ if "email" in self._data:
+ return self._data.get("email")
+ else:
+ return self._data.get("emails")
+
+ @property
+ def title(self) -> AcademicTitle | None:
+ """
+ The academic title of a user.
+ In case the user has not set a title in their user profile
+ 'None' is returned.
+ """
+ title = self._data.get("title")
+ if title is not None:
+ title = AcademicTitle(title)
+ return title
+
+ def __init__(self, data: dict) -> None:
+ self._data = data
+
+ def __str__(self) -> str:
+ return self.display_name
+
+###############################################################################
+
+class Visibility:
+ """
+ Models the visibility settings available in Coscine.
+ """
+
+ _data: dict
+
+ @property
+ def id(self) -> str:
+ """
+ Coscine-internal identifier for the visibility setting.
+ """
+ return self._data.get("id")
+
+ @property
+ def name(self) -> str:
+ """
+ Human-readable name of the visibility setting.
+ """
+ return self._data.get("displayName")
+
+ def __init__(self, data: dict) -> None:
+ self._data = data
+
+ def __str__(self) -> str:
+ return self.name
diff --git a/src/coscine/data/project.json b/src/coscine/data/project.json
deleted file mode 100644
index deeb1d66d51b0289554582df705ee2b64051392a..0000000000000000000000000000000000000000
--- a/src/coscine/data/project.json
+++ /dev/null
@@ -1,89 +0,0 @@
-[{
- "path": "projectName",
- "name": { "de": "Projektname", "en": "Project Name" },
- "order": 0,
- "class": null,
- "minCount": 1,
- "maxCount": 1,
- "datatype": "xsd:string"
-},{
- "path": "displayName",
- "name": { "de": "Anzeigename", "en": "Display Name" },
- "order": 1,
- "class": null,
- "minCount": 1,
- "maxCount": 1,
- "datatype": "xsd:string"
-},{
- "path": "description",
- "name": { "de": "Projektbeschreibung", "en": "Project Description" },
- "order": 2,
- "class": null,
- "minCount": 1,
- "maxCount": 1,
- "datatype": "xsd:string"
-},{
- "path": "principleInvestigators",
- "name": { "de": "Principal Investigators", "en": "Principal Investigators" },
- "order": 3,
- "class": null,
- "minCount": 1,
- "maxCount": 1,
- "datatype": "xsd:string"
-},{
- "path": "startDate",
- "name": { "de": "Projektstart", "en": "Project Start" },
- "order": 4,
- "class": null,
- "minCount": 1,
- "maxCount": 1,
- "datatype": "xsd:date"
-},{
- "path": "endDate",
- "name": { "de": "Projektende", "en": "Project End" },
- "order": 5,
- "class": null,
- "minCount": 1,
- "maxCount": 1,
- "datatype": "xsd:date"
-},{
- "path": "disciplines",
- "name": { "de": "Disziplin", "en": "Discipline" },
- "order": 6,
- "class": "Discipline",
- "minCount": 1,
- "maxCount": 42,
- "datatype": "xsd:string"
-},{
- "path": "organizations",
- "name": { "de": "Teilnehmende Organisationen", "en": "Participating Organizations" },
- "order": 7,
- "class": null,
- "minCount": 1,
- "maxCount": 42,
- "datatype": "xsd:string"
-},{
- "path": "keywords",
- "name": { "de": "ProjektschlagwΓΆrter", "en": "Project Keywords" },
- "order": 8,
- "class": null,
- "minCount": 0,
- "maxCount": 1,
- "datatype": "xsd:string"
-},{
- "path": "visibility",
- "name": { "de": "Sichtbarkeit der Metadaten", "en": "Metadata Visibility" },
- "order": 9,
- "class": "Visibility",
- "minCount": 1,
- "maxCount": 1,
- "datatype": "xsd:string"
-},{
- "path": "grantId",
- "name": { "de": "Grant ID", "en": "Grant ID" },
- "order": 10,
- "class": null,
- "minCount": 0,
- "maxCount": 1,
- "datatype": "xsd:string"
-}]
diff --git a/src/coscine/data/resource.json b/src/coscine/data/resource.json
deleted file mode 100644
index d22267356dc2015a6386c6a2c98b22413b8b958d..0000000000000000000000000000000000000000
--- a/src/coscine/data/resource.json
+++ /dev/null
@@ -1,89 +0,0 @@
-[{
- "path": "type",
- "name": { "de": "Ressourcentyp", "en": "Resource Type" },
- "order": 0,
- "class": "type",
- "minCount": 1,
- "maxCount": 1,
- "datatype": "xsd:string"
-},{
- "path": "resourceTypeOption",
- "name": { "de": "RessourcengrΓΆΓe", "en": "Resource Size" },
- "order": 1,
- "class": null,
- "minCount": 1,
- "maxCount": 1,
- "datatype": "xsd:integer"
-},{
- "path": "resourceName",
- "name": { "de": "Ressourcenname", "en": "Resource Name" },
- "order": 2,
- "class": null,
- "minCount": 1,
- "maxCount": 1,
- "datatype": "xsd:string"
-},{
- "path": "displayName",
- "name": { "de": "Anzeigename", "en": "Display Name" },
- "order": 3,
- "class": null,
- "minCount": 1,
- "maxCount": 1,
- "datatype": "xsd:string"
-},{
- "path": "description",
- "name": { "de": "Ressourcenbeschreibung", "en": "Resource Description" },
- "order": 4,
- "class": null,
- "minCount": 1,
- "maxCount": 1,
- "datatype": "xsd:string"
-},{
- "path": "disciplines",
- "name": { "de": "Disziplin", "en": "Discipline" },
- "order": 5,
- "class": "Disciplines",
- "minCount": 1,
- "maxCount": 42,
- "datatype": "xsd:string"
-},{
- "path": "keywords",
- "name": { "de": "RessourcenschlagwΓΆrter", "en": "Resource Keywords" },
- "order": 6,
- "class": null,
- "minCount": 0,
- "maxCount": 1,
- "datatype": "xsd:string"
-},{
- "path": "visibility",
- "name": { "de": "Sichtbarkeit der Metadaten", "en": "Metadata Visibility" },
- "order": 7,
- "class": "Visibility",
- "minCount": 1,
- "maxCount": 1,
- "datatype": "xsd:string"
-},{
- "path": "license",
- "name": { "de": "Lizenz", "en": "License" },
- "order": 8,
- "class": "License",
- "minCount": 0,
- "maxCount": 1,
- "datatype": "xsd:string"
-},{
- "path": "usageRights",
- "name": { "de": "Interne Regeln zur Nachnutzung", "en": "Internal Rules for Reuse" },
- "order": 9,
- "class": null,
- "minCount": 0,
- "maxCount": 1,
- "datatype": "xsd:string"
-},{
- "path": "applicationProfile",
- "name": { "de": "Applikationsprofil", "en": "Application Profile" },
- "order": 10,
- "class": "applicationProfile",
- "minCount": 1,
- "maxCount": 1,
- "datatype": "xsd:string"
-}]
diff --git a/src/coscine/defaults.py b/src/coscine/defaults.py
deleted file mode 100644
index 184ba58c43dd2a3138eb73a9a66978e0bfa6d478..0000000000000000000000000000000000000000
--- a/src/coscine/defaults.py
+++ /dev/null
@@ -1,54 +0,0 @@
-###############################################################################
-# Coscine Python SDK
-# Copyright (c) 2018-2023 RWTH Aachen University
-# Licensed under the terms of the MIT License
-###############################################################################
-# Coscine, short for Collaborative Scientific Integration Environment, is
-# a platform for research data management (RDM).
-# For more information on Coscine visit https://www.coscine.de/.
-#
-# Please note that this python module is open source software primarily
-# developed and maintained by the scientific community. It is not
-# an official service that RWTH Aachen provides support for.
-###############################################################################
-
-###############################################################################
-# File description
-###############################################################################
-
-"""
-This file defines default and constant data internally used by
-multiple modules to avoid redefinitions.
-"""
-
-###############################################################################
-# Dependencies
-###############################################################################
-
-from __future__ import annotations
-from typing import List
-from appdirs import AppDirs
-from coscine.__about__ import __version__
-
-###############################################################################
-# Module globals / Constants
-###############################################################################
-
-# The languages supported by the Coscine Python SDK
-LANGUAGES: List[str] = ["de", "en"]
-
-# The time format for parsing Coscine date strings with datetime
-# Note: This is no longer used - we use dateutil.parser instead!
-# This format is still used for coscine-python-sdk cache timestamps.
-TIMEFORMAT: str = "%Y-%m-%dT%H:%M:%S"
-
-# Build default application directory paths
-APPDIR: AppDirs = AppDirs("coscine-python-sdk", "Coscine")
-
-# The name for the persistent cache file storage
-CACHEFILE: str = f"{__version__}-cache.json"
-
-# The base URL for the Coscine service server instance
-BASE_URL: str = "https://coscine.rwth-aachen.de/coscine/api/Coscine.Api.%s/%s"
-
-###############################################################################
diff --git a/src/coscine/exceptions.py b/src/coscine/exceptions.py
new file mode 100644
index 0000000000000000000000000000000000000000..f9a3be6c627d0bf96b09683895569e04bfb8e86c
--- /dev/null
+++ b/src/coscine/exceptions.py
@@ -0,0 +1,66 @@
+###############################################################################
+# Coscine Python SDK
+# Copyright (c) 2020-2023 RWTH Aachen University
+# Licensed under the terms of the MIT License
+# For more information on Coscine visit https://www.coscine.de/.
+###############################################################################
+
+"""
+The Coscine Python SDK ships with its own set of exceptions.
+All exceptions raised by the Coscine Python SDK are derived from
+a common base exception class called "CoscineException".
+"""
+
+
+class CoscineException(Exception):
+ """
+ Coscine Python SDK base exception.
+ Inherited by all other Coscine Python SDK exceptions.
+ """
+
+class ValueError(CoscineException):
+ """
+ Indicates that an invalid value has been provided by the user.
+ """
+
+class TypeError(CoscineException):
+ """
+ Indicates that an invalid or unexpected type has been provided
+ by the user.
+ """
+
+class ConnectionError(CoscineException):
+ """
+ Failed to communicate with the Coscine Servers.
+ """
+
+class AuthenticationError(CoscineException):
+ """
+ Failed to authenticate with the API token supplied by the user.
+ """
+
+class IndistinguishableError(CoscineException):
+ """
+ Two or more instances match the property provided by the user but
+ the Coscine Python SDK expected just a single instance to match.
+ """
+
+class NotFoundError(CoscineException):
+ """
+ The droids you were looking for have not been found.
+ Move along!
+ """
+
+class RequestRejected(CoscineException):
+ """
+ The request has reached the Coscine servers but has
+ been rejected for whatever reason there may be. This
+ exception is most likely thrown in case of ill-formatted
+ requests.
+ """
+
+class NotImplementedError(CoscineException):
+ """
+ The feature has not been implemented (yet) or it has been
+ removed.
+ """
diff --git a/src/coscine/form.py b/src/coscine/form.py
deleted file mode 100644
index 0afe7a02ebf7b6ab11e3e34d6051676cadf8dd9b..0000000000000000000000000000000000000000
--- a/src/coscine/form.py
+++ /dev/null
@@ -1,816 +0,0 @@
-###############################################################################
-# Coscine Python SDK
-# Copyright (c) 2018-2023 RWTH Aachen University
-# Licensed under the terms of the MIT License
-###############################################################################
-# Coscine, short for Collaborative Scientific Integration Environment, is
-# a platform for research data management (RDM).
-# For more information on Coscine visit https://www.coscine.de/.
-#
-# Please note that this python module is open source software primarily
-# developed and maintained by the scientific community. It is not
-# an official service that RWTH Aachen provides support for.
-###############################################################################
-
-###############################################################################
-# File description
-###############################################################################
-
-"""
-This file provides base class for all input forms defined by
-the Coscine Python SDK.
-"""
-
-###############################################################################
-# Dependencies
-###############################################################################
-
-from __future__ import annotations
-from typing import Any, TYPE_CHECKING, Union, Tuple, Iterator, List, Dict
-from datetime import datetime
-import urllib.parse
-from prettytable import PrettyTable
-if TYPE_CHECKING:
- from coscine.client import Client
- from coscine.vocabulary import Vocabulary
-
-###############################################################################
-# Module globals/constants
-###############################################################################
-
-# TODO:
-# can we use rdflib namespace xsd for types?
-# -> https://rdflib.readthedocs.io/en/stable/rdf_terms.html
-# from rdflib.term import _castLexicalToPython
-# https://rdflib.readthedocs.io/en/stable/_modules/rdflib/term.html#_castLexicalToPython
-XSD_TYPES: Dict[str, dict] = {
- "integer": {
- "type": int,
- "values": [
- "byte", "int", "integer", "long", "unsignedShort", "unsignedByte"
- "negativeInteger", "nonNegativeInteger", "nonPositiveInteger",
- "positiveInteger", "short", "unsignedLong", "unsignedInt",
- ]
- },
- "decimal": {
- "type": float,
- "values": ["double", "float", "decimal"]
- },
- "boolean": {
- "type": bool,
- "values": ["boolean"]
- },
- "datetime": {
- "type": datetime,
- "values": [
- "date", "dateTime", "duration", "gDay", "gMonth",
- "gMonthDay", "gYear", "gYearMonth", "time"
- ]
- },
- "string": {
- "type": str,
- "values": [
- "ENTITIES", "ENTITY", "ID", "IDREF", "IDREFS", "language",
- "Name", "NCName", "NMTOKEN", "NMTOKENS", "normalizedString",
- "QName", "string", "token"
- ]
- }
-}
-
-###############################################################################
-# Classes / Functions / Scripts
-###############################################################################
-
-class FormField:
- """
- The FormField class defines a field such as "Display Name"
- inside an InputForm with all of its properties.
- """
-
- client: Client
- _data: Dict
-
- def __init__(self, client: Client, data: Dict) -> None:
- self.client = client
- self._data = data
-
- @property
- def name(self) -> str:
- """
- The name of the field e.g. "Project Name"
- """
- return self._data["name"][self.client.settings.language]
-
- @property
- def path(self) -> str:
- """
- The unique Coscine internal name of the field e.g. "projectName"
- """
- return self._data["path"]
-
- @property
- def order(self) -> int:
- """
- The order of appearance of the field inside of the form
- """
- return int(self._data["order"])
-
- @property
- def vocabulary(self) -> str:
- """
- Name of the vocabulary instance for the field, may be None
- """
- return self._data["class"]
-
- @property
- def min_count(self) -> int:
- """
- Minimum amount of values required for a successful generate()
- """
- return int(self._data["minCount"])
-
- @property
- def max_count(self) -> int:
- """
- Maximum allowed amount of values
- """
- return int(self._data["maxCount"])
-
- @property
- def datatype(self) -> str:
- """
- Expected datatype of the field as a string identifier, e.g. "xsd:int"
- """
- return self._data["datatype"]
-
- @property
- def selection(self) -> List[str]:
- """
- Preset selection of values - no other value except for these values
- can be specified. Think of it as a class defined in the profile.
- """
- if "in" in self._data and self._data["in"] is not None:
- return self._data["in"]
- return []
-
-###############################################################################
-###############################################################################
-###############################################################################
-
-class FormValue:
- """
- An InputForm specific representation of a value
- """
-
- _form: InputForm
- _key: str
- properties: FormField
- # In case of form.properties(key).maxValues > 1 it is a list,
- # otherwise a scalar object
- _container: Union[List[Any], Any]
-
-###############################################################################
-
- def __init__(
- self,
- form: InputForm,
- key: str,
- obj: Any = None,
- raw: bool = False
- ) -> None:
- """
- Specifying the associated InputForm and key may seem redundant from
- the point of view of the InputForm, but ultimately it allows the form
- value to access the vocabulary and the field properties.
- """
- self._form = form
- self._key = key
- self.properties = form.properties(key)
- self.set_value(obj, raw)
-
-###############################################################################
-
- def __bool__(self) -> bool:
- if isinstance(self._container, list):
- return self._container[0] is not None
- return self._container is not None
-
-###############################################################################
-
- def _set_list_value(self, values: List[Any], raw: bool = False) -> None:
- if isinstance(values, (list, tuple)):
- self._container = []
- for entry in values:
- self.append(entry, raw)
- else:
- # Let's be generous and do the conversion if scalar type to list
- # of size 1 for the users ourselves
- self._container = []
- self.append(values, raw)
-
-###############################################################################
-
- def _set_scalar_value(self, value: Any, raw: bool = False) -> None:
- if isinstance(value, (list, tuple)):
- # Be very generous and automatically convert lists
- # of size 1 to an atomic value.
- if len(value) > 1:
- raise TypeError(
- "Did not expect list type exceeding "
- f"MaxValues property for key '{self._key}'."
- )
- value = value[0]
- self._container = value if raw else self._objectify(value)
-
-###############################################################################
-
- def set_value(self, value: Any, raw: bool = False) -> None:
- """
- Sets the FormValue equal to the specified value.
-
- Parameters
- ----------
- value : Any
- The value to set the FormField equal to. Could be
- a list of values.
- raw : bool
- If set to True, the value is not interpreted for controlled
- fields and its type is not checked. Use raw when dealing with
- values in Coscines internal format.
- """
- if self.properties.max_count > 1:
- self._set_list_value(value, raw)
- else:
- self._set_scalar_value(value, raw)
-
-###############################################################################
-
- def append(self, value: Any, raw: bool = False) -> None:
- """
- Appends a value to a FormValue allowing multiple values.
-
- Parameters
- ----------
- value : Any
- The value to append to the FormField.
- raw : bool
- If set to True, the value is not interpreted for controlled
- fields and its type is not checked. Use raw when dealing with
- values in Coscines internal format.
-
- Raises
- ------
- TypeError
- In case of appending a value to a scalar FormField.
- IndexError
- When exceeding the maxValues setting for the FormField.
- """
- if not isinstance(self._container, list):
- raise TypeError(
- "Attempting to append a value to scalar "
- f"FormField '{self._key}'. "
- "See 'MaxCount' property in application profile."
- )
- if len(self._container) < self.properties.max_count:
- if not raw:
- value = self._objectify(value)
- self._container.append(value)
- else:
- raise IndexError(f"Exceeding maxValues for field {self._key}.")
-
-###############################################################################
-
- def type_format(self) -> str:
- """
- Returns the type format of the internal type.
- See InputForm.type_format() as this is internally called.
-
- Returns
- -------
- str
- The type format as a string, e.g. "%s" for string types
- or "%d" for integers. This function is handy when dealing with
- datetime objects, as it returns the expected date format.
- """
- return self._form.type_format(self._key)
-
-###############################################################################
-
- def _is_valid_type(self, key: str, value: Any) -> bool:
- if self._form.is_controlled(self._key) and not isinstance(value, str):
- raise TypeError(
- "Values specified for controlled fields "
- "must be of type string to make it possible to "
- "look up their raw representation in the vocabulary: "
- f"Expected 'str', got '{str(type(value))}'"
- )
- if self._form.datatype(key) is None:
- return True
- datatype = self._form.datatype(key)
- return datatype is not None and isinstance(value, datatype)
-
-###############################################################################
-
- def _objectify(self, value: Any) -> Any:
- """
- Checks if the type of the value matches the expecte type and
- in case of controlled fields, converts the human-readable value to
- the unique internal Coscine representation via
- the associated vocabulary.
- """
- if value is None:
- return None
-
- if not self._is_valid_type(self._key, value):
- raise TypeError(
- f"Value of type {str(type(value))} specified "
- f"for key {self._key} does not match expected "
- f"type {self._form.datatype(self._key)}!"
- )
-
- if self._form.has_vocabulary(self._key):
- if not self._form.vocabulary(self._key).contains(value):
- suggestions = self._form.vocabulary(self._key).suggest(value)
- raise ValueError(
- f"Invalid value '{value}' for vocabulary "
- f"controlled key '{self._key}'! "
- f"Perhaps you meant {suggestions}?"
- )
- return self._form.vocabulary(self._key).lookup_key(value)
- if self._form.has_selection(self._key):
- if value not in self._form.selection(self._key):
- raise ValueError(
- f"Invalid value '{value}' for selection "
- f"controlled key '{self._key}'!"
- )
- return value
-
-###############################################################################
-
- def _serialize_value(self, obj: Any, raw: bool = False) -> str:
- """
- Serializes a single value according to its type / format
- -> This function converts a pythonic value to a string.
- """
- serialized_value: str = ""
- if obj and self._form.has_vocabulary(self._key):
- if raw:
- serialized_value = obj
- else:
- serialized_value = str(
- self._form.vocabulary(self._key).lookup_value(obj)
- )
- elif isinstance(obj, datetime):
- try:
- serialized_value = obj.strftime(self.type_format())
- except ValueError:
- serialized_value = str(obj) # Omit format if error
- elif isinstance(obj, dict):
- serialized_value = str(obj)
- elif isinstance(obj, bool):
- serialized_value = str(obj).lower()
- elif obj is None:
- serialized_value = ""
- else:
- serialized_value = str(obj)
- return serialized_value
-
-###############################################################################
-
- def _serialize_list_raw(self) -> List:
- serialized_values = []
- for value in self._container:
- serialized_values.append(self._serialize_value(value, True))
- return serialized_values
-
-###############################################################################
-
- def _serialize_list(self) -> str:
- serialized_values = []
- for value in self._container:
- serialized_values.append(self._serialize_value(value))
- return ",".join(serialized_values)
-
-###############################################################################
-
- def __str__(self) -> str:
- if self.properties.max_count > 1:
- return self._serialize_list()
- return self._serialize_value(self._container)
-
-###############################################################################
-
- def raw(self) -> Any:
- """
- Returns the raw value of the FormValue.
- """
- if self.properties.max_count > 1:
- return self._serialize_list_raw()
- return self._serialize_value(self._container, True)
-
-###############################################################################
-###############################################################################
-###############################################################################
-
-class InputForm:
- """
- Coscine InputForm base class
- """
-
- client: Client
- _fields: List[FormField]
- _values: Dict[str, FormValue]
- _vocabularies: Dict[str, Vocabulary]
-
-###############################################################################
-
- def __init__(self, client: Client) -> None:
- """
- Parameters
- ----------
- client : Client
- Coscine Python SDK Client handle
- """
- super().__init__()
- self.client = client
- self._fields = []
- self._values = {}
- self._vocabularies = {}
-
-###############################################################################
-
- def __setitem__(self, key: str, value: Any) -> None:
- self.set_value(key, value)
-
-###############################################################################
-
- def __getitem__(self, key: str) -> FormValue:
- return self.get_value(key)
-
-###############################################################################
-
- def __delitem__(self, key: str) -> None:
- del self._values[self.path(key)]
-
-###############################################################################
-
- def __iter__(self) -> Iterator[str]:
- return iter(self.keys())
-
-###############################################################################
-
- def __len__(self) -> int:
- return len(self._fields)
-
-###############################################################################
-
- def __str__(self) -> str:
- return self.as_str()
-
-###############################################################################
-
- def clear(self) -> None:
- """Removes all values of the InputForm"""
- self._values.clear()
-
-###############################################################################
-
- def path(self, key: str) -> str:
- """
- Returns the unique identifier for a multilanguage key.
- Logical example:
- path("Display Name") == path("Anzeigename") == "__XXX__"
- ->> True
- where "__XXX__" is of some fixed internal value.
-
- Parameters
- -----------
- key : str
- Multilanguage form key. Language depends on the
- coscine.Client language setting.
-
- Returns
- -------
- str
- Unique identifier of a form field.
- """
- return self.properties(key).path
-
-###############################################################################
-
- def properties(self, key: str) -> FormField:
- """
- Returns the form field properties corresponding to the given key.
-
- Parameters
- ----------
- key : str
- Form key
-
- Returns
- -------
- FormField
- Object containing the field properties for the given key
- """
- return self._fields[self.index_of(key)]
-
-###############################################################################
-
- def index_of(self, key: str) -> int:
- """
- Returns the order/index of the given key.
- """
- for index, item in enumerate(self.keys()):
- if item == key:
- return index
- raise KeyError(f"Key '{key}' not in InputForm!")
-
-###############################################################################
-
- def name_of(self, path: str) -> str:
- """
- Returns the key name for the unique path depending on the client
- language setting.
- """
- for field in self._fields:
- if field.path == path:
- return field.name
- raise KeyError(f"Path '{path}' not in InputForm!")
-
-###############################################################################
-
- def keys(self) -> List[str]:
- """
- Returns a list of keys in their respective order based on the
- language setting of the client class instance used to initialize
- the InputForm.
- """
- return [field.name for field in self._fields]
-
-###############################################################################
-
- def values(self) -> List[FormValue]:
- """
- Returns a list of values in their respective order based on the
- language setting of the client class instance used to initialize
- the InputForm.
- """
- return [self.get_value(key) for key in self.keys()]
-
-###############################################################################
-
- def items(self) -> List[Tuple[str, FormValue]]:
- """
- Returns a list of key, value pairs in their respective order based
- on the language setting of the client class instance
- used to initialize the InputForm.
- """
- return list(zip(self.keys(), self.values()))
-
-###############################################################################
-
- @staticmethod
- def _xsd_typeof(xsd_type: str) -> type:
- if xsd_type is None: return str # Be more permissive
- if xsd_type.startswith("xsd:"):
- xsd_type = xsd_type[4:]
- for item in XSD_TYPES.values():
- if xsd_type in item["values"]:
- return item["type"]
- return str
-
-###############################################################################
-
- def xsd_type(self, key: str) -> str:
- """
- Returns the xsd type identifier for a given field.
- """
- datatype = self.properties(key).datatype
- if datatype and datatype.startswith("http:"):
- datatype = f"xsd:{urllib.parse.urlparse(datatype)[-1]}"
- return datatype
-
-###############################################################################
-
- def datatype(self, key: str) -> type:
- """
- Returns the datatype for a given field (which may be None).
- """
- if self.is_controlled(key):
- return str
- else:
- datatype: str = self.xsd_type(key)
- return self._xsd_typeof(datatype)
-
-###############################################################################
-# FIXME: Lots of xsd types still unhandled, especially datetime formats
- def type_format(self, key: str) -> str:
- """
- Returns the format for a datatype of a field.
- This is especially useful for instances of type datetype.
- """
- datatype = self.datatype(key)
- if datatype == str:
- return "%s"
- if datatype in (bool, int):
- return "%d"
- if datatype == float:
- return "%f"
- if datatype == datetime:
- if self.xsd_type(key) == "xsd:dateTime":
- return "%Y-%m-%dT%H:%M:%S"
- else:
- return "%Y-%m-%d"
- return "Invalid Format"
-
-###############################################################################
-
- def is_typed(self, key: str) -> bool:
- """
- Returns whether a value is expecting a certain datatype.
- """
- return self.datatype(key) is not None
-
-###############################################################################
-
- def is_required(self, key: str) -> bool:
- """
- Determines whether a key is a required one.
- """
- return self.properties(key).min_count > 0
-
-###############################################################################
-
- def has_vocabulary(self, key: str) -> bool:
- """
- Determines whether a key is controlled by a vocabulary.
- """
- return self.properties(key).vocabulary is not None
-
-###############################################################################
-
- def has_selection(self, key: str) -> bool:
- """
- Determines whether a key is controlled by a selection.
- """
- return len(self.properties(key).selection) > 0
-
-###############################################################################
-
- def is_controlled(self, key: str) -> bool:
- """
- Determines whether a key is controlled by a vocabulary or selection.
- """
- return self.has_vocabulary(key) or self.has_selection(key)
-
-###############################################################################
-
- def vocabulary(self, key: str) -> Vocabulary:
- """
- Returns the vocabulary for the given controlled key. Make sure
- the key is actually controlled to avoid an exception.
-
- Raises
- -------
- KeyError
- In case the specified key is not controlled by a vocabulary.
- """
- if self.has_vocabulary(key):
- return self._vocabularies[self.path(key)]
- raise KeyError(f"Key '{key}' is not controlled by a vocabulary!")
-
-###############################################################################
-
- def selection(self, key: str) -> List[str]:
- """
- Returns the selection for the given controlled key. Make sure
- the key is actually controlled to avoid an exception.
-
- Raises
- -------
- KeyError
- In case the specified key is not controlled by a selection.
- """
-
- if self.has_selection(key):
- return self.properties(key).selection
- raise KeyError(f"Key '{key}' is not controlled by a selection!")
-
-###############################################################################
-
- def set_value(self, key: str, value: Any, raw: bool = False) -> None:
- """
- Sets the value for a given key.
-
- Parameters
- ----------
- key : str
- The key to associate the value with
- value : Any
- The typed value, either in raw format or human readable
- raw : bool, default: False
- Treat the value as a raw value (from Coscine) and do not apply
- vocabulary search or type checking on it
- """
- self._values[self.path(key)] = FormValue(self, key, value, raw)
-
-###############################################################################
-
- def get_value(self, key: str) -> FormValue:
- """
- Returns the FormValue for the specified key
-
- Parameters
- ----------
- key : str
- The key to get the value from.
- """
- if key not in self.keys():
- raise KeyError(f"Key '{key}' not in InputForm!")
- if self.path(key) in self._values:
- return self._values[self.path(key)]
- return FormValue(self, key) # Yields an empty FormValue
-
-###############################################################################
-
- def fill(self, data: Dict) -> None:
- """
- Fill in an InputForm from a dictionary
-
- Parameters
- ----------
- data : dict
- Python dictionary containing key value pairs in human-readable
- form (not data from Coscine!).
- """
- for key, value in data.items():
- self[key] = value
-
-###############################################################################
-
- def parse(self, data: Dict) -> None:
- """
- Parses data from Coscine into an InputForm.
-
- Parameters
- ----------
- data : dict
- Python dictionary (output of json.loads) containing
- the data from Coscine.
- """
-
-###############################################################################
-
- def generate(self) -> dict:
- """
- Generates Coscine formatted json-ld from an InputForm.
- """
- return {}
-
-###############################################################################
-
- def as_str(self, format: str = "str") -> str:
- """
- Returns a string in the specified format
-
- Parameters
- -----------
- format : str, default "str"
- Format of the string, possible options are: str, csv, json, html
-
- Returns
- --------
- str
- A string with the form values in the specified format.
- """
-
- supported_formats = ["str", "csv", "json", "html"]
- if format not in supported_formats:
- raise ValueError(f"Unsupported format '{format}'!")
- table = PrettyTable(
- ("C", "Type", "Property", "Value"),
- align="l"
- )
- rows = []
- for key in self.keys():
- value = str(self.get_value(key))
- name: str = key
- name = name + "*" if self.is_required(key) else name
- controlled: str = ""
- if self.is_controlled(key):
- controlled = "V" if self.has_vocabulary(key) else "S"
- datatype: str = str(self.datatype(key).__name__)
- if self.properties(key).max_count > 1:
- datatype = f"[{datatype}]"
- rows.append((controlled, datatype, name, value))
- table.max_width["Value"] = 50
- table.add_rows(rows)
- if format == "csv":
- return table.get_csv_string()
- if format == "json":
- return table.get_json_string()
- if format == "html":
- return table.get_html_string()
- return table.get_string()
-
-###############################################################################
diff --git a/src/coscine/graph.py b/src/coscine/graph.py
deleted file mode 100644
index d62e929669c0b6a551e240df2e779b5ed327d399..0000000000000000000000000000000000000000
--- a/src/coscine/graph.py
+++ /dev/null
@@ -1,212 +0,0 @@
-###############################################################################
-# Coscine Python SDK
-# Copyright (c) 2018-2023 RWTH Aachen University
-# Licensed under the terms of the MIT License
-###############################################################################
-# Coscine, short for Collaborative Scientific Integration Environment, is
-# a platform for research data management (RDM).
-# For more information on Coscine visit https://www.coscine.de/.
-#
-# Please note that this python module is open source software primarily
-# developed and maintained by the scientific community. It is not
-# an official service that RWTH Aachen provides support for.
-###############################################################################
-
-###############################################################################
-# File description
-###############################################################################
-
-"""
-This file provides a simple wrapper around Coscine application profiles.
-It abstracts the interaction with rdf graphs using rdflib and
-provides a more intuitive, Coscine-specific interface to these graphs.
-"""
-
-###############################################################################
-# Dependencies
-###############################################################################
-
-from __future__ import annotations
-from typing import TYPE_CHECKING, Dict, List
-import rdflib
-if TYPE_CHECKING:
- from coscine.client import Client
-
-###############################################################################
-# Classes / Functions / Scripts
-###############################################################################
-
-class ApplicationProfile:
- """
- The ApplicationProfile class serves as a wrapper around Coscine
- application profiles. Coscine application profiles are served
- as rdf graphs which are difficult to interact with, without 3rd
- party libraries.
- The ApplicationProfile class abstracts this interaction.
-
- Attributes
- ----------
- graph : rdflib.Graph
- An rdf graph parsed with the help of rdflib
- RDFTYPE : str
- A variable frequently used by methods of this class.
- """
-
- client: Client
- graph: rdflib.Graph
- RDFTYPE: str = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"
-
-###############################################################################
-
- def __init__(self, client: Client, graph: str, format: str = "json-ld") -> None:
- """
- Initializes an instance of the ApplicationProfile wrapper class.
-
- Parameters
- ----------
- graph : str
- A Coscine application profile rdf graph in json-ld text format.
- format : str (optional), default: "json-ld"
- The format of the graph
- """
-
- self.client = client
- self.graph = rdflib.Graph()
- self.graph.bind("sh", "http://www.w3.org/ns/shacl#")
- self.graph.bind("dcterms", "http://purl.org/dc/terms/")
- self.graph.parse(data=graph, format=format)
- self._recursive_import()
-
-###############################################################################
-
- def _recursive_import(self):
- """
- Imports parent application profiles
- TODO: Import after importing (actually recurse)
- Currently we just iterate over owl:imports once,
- but after importing new profiles, new owl:imports
- may have appeared in the graph...
- """
- query = "SELECT ?url WHERE { ?_ owl:imports ?url . }"
- for result in self.graph.query(query):
- url = str(result[0])
- profile = self.client.vocabularies.application_profile(url)
- self.graph.parse(data=str(profile), format="ttl")
-
-###############################################################################
-
- def __str__(self) -> str:
- """
- Serializes the application profile rdf graph used internally
- for easy output to stdout.
- """
- # Explicitly return as str because pylint does not understand
- # rdflib's ambiguous return type for this method.
- return str(self.graph.serialize(format="ttl"))
-
-###############################################################################
-
- def target(self) -> str:
- """
- Returns a str indicating the target class of the application profile.
- This may for example be "engmeta" in case of an engmeta profile.
- """
-
- query = \
- """
- SELECT ?target WHERE {
- ?target a sh:NodeShape .
- }
- """
- result = self.query(query)
- return str(result[0][0]) # Force string
-
-###############################################################################
-
- def query(self, query: str, **kwargs) -> List[List[object]]:
- """
- Performs a SPARQL query against the application profile and
- returns the result as a list of rows.
- """
-
- items = []
- results = self.graph.query(query, **kwargs)
- for row in results:
- item = []
- for val in row:
- value = val.toPython() if val is not None else None
- item.append(value)
- items.append(item)
- return items
-
-###############################################################################
-
- def items(self) -> List[dict]:
- """
- Returns all items contained within the application profile as a list
- of key value pairs in their order of appearance.
- """
-
- query = \
- """
- SELECT ?path ?name ?order ?class ?minCount ?maxCount ?datatype
- (lang(?name) as ?lang)
- (GROUP_CONCAT(?in; SEPARATOR="~,~") as ?ins)
- WHERE {
- ?_ sh:path ?path ;
- sh:name ?name .
- OPTIONAL { ?_ sh:order ?order . } .
- OPTIONAL { ?_ sh:class ?class . } .
- OPTIONAL { ?_ sh:minCount ?minCount . } .
- OPTIONAL { ?_ sh:maxCount ?maxCount . } .
- OPTIONAL { ?_ sh:datatype ?datatype . } .
- OPTIONAL { ?_ sh:in/rdf:rest*/rdf:first ?in . } .
- }
- GROUP BY ?name
- ORDER BY ASC(?order)
- """
- items: Dict[str, Dict] = {}
- results = self.query(query)
- for result in results:
- lang: str = str(result[7])
- path: str = str(result[0])
- selection: str = str(result[8])
- name: str = str(result[1])
- properties = {
- "path": result[0],
- "name": {
- "de": name,
- "en": name
- },
- "order": result[2],
- "class": result[3],
- "minCount": result[4] if result[4] else 0,
- "maxCount": result[5] if result[5] else 42,
- "datatype": result[6],
- "in": selection.split("~,~") if selection else None
- }
- if path not in items:
- items[path] = properties
- else: # Apply name in different language
- items[path]["name"][lang] = name
-
- return list(items.values())
-
-###############################################################################
-
- def length(self) -> int:
- """
- Returns the number of fields contained within the application profile.
- """
-
- query = \
- """
- SELECT ?path WHERE {
- ?_ sh:path ?path ;
- sh:order ?order .
- }
- """
- results = self.query(query)
- return len(results)
-
-###############################################################################
diff --git a/src/coscine/metadata.py b/src/coscine/metadata.py
new file mode 100644
index 0000000000000000000000000000000000000000..38c685df28920fe67a9d07cdc92f55bdea669946
--- /dev/null
+++ b/src/coscine/metadata.py
@@ -0,0 +1,1000 @@
+###############################################################################
+# Coscine Python SDK
+# Copyright (c) 2020-2023 RWTH Aachen University
+# Licensed under the terms of the MIT License
+# For more information on Coscine visit https://www.coscine.de/.
+###############################################################################
+
+"""
+Provides functions and classes around the handling of metadata.
+"""
+
+from __future__ import annotations
+from typing import TYPE_CHECKING, TypeAlias, Tuple
+if TYPE_CHECKING:
+ from coscine.client import ApiClient
+ from coscine.resource import Resource
+from datetime import date, datetime, time
+from dateutil.parser import parse as dateutil_parse
+from requests.compat import urlparse
+from tabulate import tabulate
+from threading import Lock
+from decimal import Decimal
+import random
+from string import ascii_letters
+import rdflib
+import pyshacl
+import coscine.exceptions
+
+###############################################################################
+
+# Type Alias according to PEP 613 for supported metadata value types
+FormType: TypeAlias = bool | date | datetime | int | float | str | time
+
+# XML Schema datatype to Python native type lookup table
+XSD_TYPES: dict[str, type] = {
+ "any": str,
+ "anyURI": str,
+ "anyType": str,
+ "byte": int,
+ "int": int,
+ "integer": int,
+ "long": int,
+ "unsignedShort": int,
+ "unsignedByte": int,
+ "negativeInteger": int,
+ "nonNegativeInteger": int,
+ "nonPositiveInteger": int,
+ "positiveInteger": int,
+ "short": int,
+ "unsignedLong": int,
+ "unsignedInt": int,
+ "double": float,
+ "float": float,
+ "decimal": Decimal,
+ "boolean": bool,
+ "date": date,
+ "dateTime": datetime,
+ "duration": datetime,
+ "gDay": datetime,
+ "gMonth": datetime,
+ "gMonthDay": datetime,
+ "gYear": datetime,
+ "gYearMonth": datetime,
+ "time": time,
+ "ENTITIES": str,
+ "ENTITY": str,
+ "ID": str,
+ "IDREF": str,
+ "IDREFS": str,
+ "language": str,
+ "Name": str,
+ "NCName": str,
+ "NMTOKEN": str,
+ "NMTOKENS": str,
+ "normalizedString": str,
+ "QName": str,
+ "string": str,
+ "token": str
+}
+
+def xsd_to_python(xmltype: str) -> type:
+ """
+ Converts an XMLSchema XSD datatype string to a native
+ Python datatype class instance.
+ """
+ if xmltype.startswith("http://www.w3.org/2001/XMLSchema#"):
+ xmltype = urlparse(xmltype)[-1]
+ if xmltype not in XSD_TYPES:
+ raise coscine.exceptions.ValueError(
+ "Failed to convert XMLSchema XSD datatype "
+ f"to native Python type: Unsupported type '{xmltype}'!"
+ )
+ return XSD_TYPES[xmltype]
+ return str
+
+###############################################################################
+
+class Instance:
+ """
+ A (vocabulary) instance is an entry inside of a vocabulary.
+ It maps from a human-readable name to a unique uniform
+ resource identifier.
+ """
+
+ _data: dict
+
+ @property
+ def graph_uri(self) -> str:
+ """
+ """
+ return self._data.get("graphUri")
+
+ @property
+ def instance_uri(self) -> str:
+ """
+ """
+ return self._data.get("instanceUri")
+
+ @property
+ def type_uri(self) -> str:
+ """
+ """
+ return self._data.get("typeUri")
+
+ @property
+ def subclass_of(self) -> str:
+ """
+ """
+ return self._data.get("subClassOfUri")
+
+ @property
+ def name(self) -> str:
+ """
+ """
+ return self._data.get("displayName")
+
+ def __init__(self, data: dict) -> None:
+ self._data = data
+
+ def __str__(self) -> str:
+ return self.name
+
+###############################################################################
+
+class Vocabulary:
+ """
+ The Vocabulary contains all instances of a class and provides
+ an interface to easily check whether a term is contained in
+ the set of instances and to query the respective instance.
+ """
+
+ _data: list[Instance]
+
+ def __init__(self, data: list[Instance]) -> None:
+ self._data = data
+
+ def __str__(self) -> str:
+ return "\n".join(self.keys())
+
+ def __contains__(self, key: str) -> bool:
+ return key in self.keys()
+
+ def __getitem__(self, key: str) -> str:
+ for entry in self._data:
+ if entry.name == key:
+ return entry.instance_uri
+ raise KeyError(f"Key {key} is not contained in vocabulary!")
+
+ def __iter__(self):
+ for key in self.keys():
+ yield key
+
+ def graph(self) -> rdflib.Graph:
+ """
+ """
+ graph = rdflib.Graph()
+ for entry in self._data:
+ graph.add((
+ rdflib.URIRef(entry.instance_uri),
+ rdflib.RDF.type,
+ rdflib.URIRef(entry.type_uri)
+ ))
+ graph.add((
+ rdflib.URIRef(entry.instance_uri),
+ rdflib.RDFS.label,
+ rdflib.Literal(entry.name)
+ ))
+ if entry.subclass_of:
+ graph.add((
+ rdflib.URIRef(entry.instance_uri),
+ rdflib.RDFS.subClassOf,
+ rdflib.URIRef(entry.subclass_of)
+ ))
+ return graph
+
+ def keys(self) -> list[str]:
+ """
+ Returns the list of keys that are contained inside
+ of the vocabulary. This equals the set of names of the
+ class instances.
+ """
+ return [entry.name for entry in self._data]
+
+ def resolve(self, value: str) -> FormType:
+ """
+ This method takes a value and return its corresponding key.
+ It can be considered the reverse of Vocabulary[key] -> value,
+ namely Vocabulary[value] -> key but that cannot be expressed
+ in Python, hence this method.
+ """
+ for entry in self._data:
+ if entry.instance_uri == value:
+ return entry.name
+ raise KeyError(f"Value {value} is not contained in vocabulary!")
+
+###############################################################################
+
+class FormField:
+ """
+ A FormField represents a MetadataField that has been specified
+ in an application profile. The FormField has numerous properties
+ which restrict the range of values that can be assigned to
+ a metadata field. It is thus very important for the validation
+ of metadata and ensures the consistency of metadata.
+ """
+
+ client: ApiClient
+
+ _data: dict
+ _values: list[FormType]
+
+ @property
+ def path(self) -> str:
+ """
+ The path of the FormField, acting as a unique identifier.
+ """
+ return self._data.get("path") or ""
+
+ @property
+ def name(self) -> str:
+ """
+ The human-readable name of the field, as displayed in
+ the Coscine web interface.
+ """
+ return self._data.get("name") or ""
+
+ @property
+ def order(self) -> int:
+ """
+ The order of appearance of the field. The metadata fields are
+ often displayed in a list in some sort of user interface. This
+ property simply states at which position the field should appear.
+ """
+ return int(self._data.get("order") or 1)
+
+ @property
+ def class_uri(self) -> str:
+ """
+ In case the field is controlled by a vocabulary, the class_uri
+ specifies the link to the instances of the vocabulary. These
+ can then be fetched via ApiClient.instances(class_uri)
+ """
+ return self._data.get("class") or ""
+
+ @property
+ def min_count(self) -> int:
+ """
+ The minimum count of values that the field must receive.
+ If the count is greater than 0, the field is a required one,
+ as it will always need a value.
+ """
+ return self._data.get("minCount") or 0
+
+ @property
+ def max_count(self) -> int:
+ """
+ The maximum amount of values that can be given to the field.
+ """
+ return self._data.get("maxCount") or 128
+
+ @property
+ def min_length(self) -> int:
+ """
+ Specifies the minumum required length of the value.
+ For values of type string this would equal the minimum
+ string length.
+ """
+ return self._data.get("minLength") or 0
+
+ @property
+ def max_length(self) -> int:
+ """
+ Specifies the maximum permissible length of the value.
+ For values of type string this would equal the maximum
+ string length.
+ """
+ return self._data.get("maxLength") or 4096
+
+ @property
+ def literals(self) -> list[rdflib.Literal]:
+ """
+ The field as rdflib.Literal ready for use with rdflib.
+ The literal has the appropriate datatype as specified in
+ the SHACL application profile. This should be used as Coscine
+ is very strict with its verification: There is apparently
+ a difference between xsd:int and xsd:integer, I kid you not!
+ """
+ # See also: https://rdflib.readthedocs.io/en/stable/rdf_terms.html
+ return [
+ rdflib.Literal(value, datatype=self.xsd_type)
+ for value in self.values
+ ]
+
+ @property
+ def identifiers(self) -> list[rdflib.Literal] | list[rdflib.URIRef]:
+ """
+ """
+ if self.has_vocabulary:
+ return [rdflib.URIRef(serial) for serial in self.serial]
+ return self.literals
+
+ @property
+ def xsd_type(self) -> str | None:
+ """
+ The string representation of the xsd:datatype.
+ For example: http://www.w3.org/2001/XMLSchema#int
+ """
+ return self._data.get("datatype") or None
+
+ @property
+ def datatype(self) -> type:
+ """
+ Restricts the datatype of values that can be assigned
+ to the field.
+ """
+ return xsd_to_python(self._data.get("datatype") or "")
+
+ @property
+ def vocabulary(self) -> Vocabulary:
+ """
+ In the case that the field has a value for the class_uri property,
+ it is controlled by a vocabulary.
+ """
+ if not self.has_vocabulary:
+ raise coscine.exceptions.NotFoundError(
+ f"Field {self.name} is not controlled by a vocabulary!"
+ )
+ return self.client.vocabulary(self.class_uri)
+
+ @property
+ def selection(self) -> list[str]:
+ """
+ Some fields have a predefined selection of values that the user
+ can choose from. In that case other values are not permitted.
+ """
+ return self._data.get("selection", "").split("~,~") or []
+
+ @property
+ def language(self) -> str:
+ """
+ The language setting of the field. This influences the field
+ name and the values of fields controlled by a vocabulary or
+ selection.
+ """
+ return self._data.get("language") or "en"
+
+ @property
+ def has_vocabulary(self) -> bool:
+ """
+ Evaluates to True if the field values are controlled by a vocabulary.
+ """
+ return bool(self._data["class"])
+
+ @property
+ def has_selection(self) -> bool:
+ """
+ Evaluates to True if the field values are controlled
+ by a predefined selection of values.
+ """
+ return bool(self._data["selection"])
+
+ @property
+ def is_controlled(self) -> bool:
+ """
+ Evaluates to True if the field is either controlled by
+ a vocabulary or a selection.
+ """
+ return self.has_vocabulary or self.has_selection
+
+ @property
+ def is_required(self) -> bool:
+ """
+ Evaluates to True if the field must be assigned a value before
+ it can be sent to Coscine alongside the other metadata.
+ """
+ return self.min_count > 0
+
+ @property
+ def serial(self) -> list[FormType]:
+ """
+ Serializes the metadata value to Coscine format. That means
+ that for vocabulary controlled fields, the human-readable
+ value is translated to the machine-readable unique identifier.
+ This property can also be set with the metadata value received
+ by the Coscine API, which is already in machine-readable format
+ and will be translated to human-readable internally.
+ """
+ return self.serialize()
+
+ @serial.setter
+ def serial(self, value: str) -> None:
+ self.values = [self.deserialize(value)]
+
+ @property
+ def values(self) -> list[FormType]:
+ """
+ This is the value of the metadata field in human-readable
+ form. For the machine-readable form that is sent to Coscine
+ use the property FormValue.serial!
+ Setting a value can only be done by using the appropriate datatype.
+ If the FormField.max_count is greater than 1, you may assign a list
+ of values to the field.
+ """
+ return self._values
+
+ @values.setter
+ def values(self, value: FormType | list[FormType]) -> None:
+ if isinstance(value, list) or isinstance(value, tuple):
+ self._values = []
+ for item in value:
+ self.append(item)
+ else:
+ self.validate(value)
+ self._values = [value]
+
+ def validate(self, value: FormType) -> None:
+ """
+ Validates whether the value matches the specification of the
+ FormField. Does not return anything but instead raises all
+ sorts of exceptions.
+ """
+ if type(value) != self.datatype:
+ raise coscine.exceptions.TypeError(
+ f"While setting value for field {self.name}: "
+ f"Expected type {self.datatype} but got {type(value)}!"
+ )
+ if self.is_controlled:
+ if self.has_vocabulary and value not in self.vocabulary:
+ raise coscine.exceptions.ValueError(
+ f"The field '{self.name}' is controlled by a vocabulary. "
+ f"The value '{value}' that you have provided did not "
+ "match any of the entries in the vocabulary!"
+ )
+ elif self.has_selection and value not in self.selection:
+ raise coscine.exceptions.ValueError(
+ f"The field '{self.name}' is controlled by a selection. "
+ f"The value '{value}' that you have provided did not "
+ "match any of the entries in the selection!"
+ )
+
+ def append(self, value: FormType, serialized: bool = False) -> None:
+ """
+ If the field accepts a list of values, one can use the append
+ method to add another value to the end of that list.
+ """
+ if serialized:
+ value = self.deserialize(value)
+ self.validate(value)
+ self._values.append(value)
+
+ def serialize(self) -> str:
+ """
+ """
+ if self.has_vocabulary:
+ return [str(self.vocabulary[value]) for value in self.values]
+ return [str(value) for value in self.values]
+
+ def deserialize(self, value: str) -> FormType:
+ """
+ Unmarshals the value and returns the pythonic representation.
+ """
+ if self.has_vocabulary:
+ return self.vocabulary.resolve(value)
+ elif self.datatype == datetime:
+ return dateutil_parse(value)
+ elif self.datatype == date:
+ return dateutil_parse(value).date()
+ elif self.datatype == time:
+ return dateutil_parse(value).time()
+ elif self.datatype == int:
+ return int(value)
+ elif self.datatype == float:
+ return float(value)
+ elif self.datatype == Decimal:
+ return Decimal(value)
+ elif self.datatype == bool:
+ return bool(value)
+ else:
+ return value
+
+ def __init__(self, client: ApiClient, data: dict) -> None:
+ self.client = client
+ self._data = data
+ self._values = []
+
+ def clear(self) -> None:
+ self._values = []
+
+###############################################################################
+
+class ApplicationProfileInfo:
+ """
+ Many different application profiles are available in Coscine.
+ To be able to get information on a specific application profile or all
+ application profiles, the ApplicationProfileInfo datatype is
+ provided.
+ """
+
+ _data: dict
+
+ @property
+ def uri(self) -> str:
+ """
+ The uri of the application profile.
+ """
+ return self._data.get("uri") or ""
+
+ @property
+ def name(self) -> str:
+ """
+ The human-readable name of the application profile.
+ """
+ return self._data.get("displayName") or ""
+
+ @property
+ def description(self) -> str:
+ """
+ A description of the application profile.
+ """
+ return self._data.get("description") or ""
+
+ def __init__(self, data: dict) -> None:
+ self._data = data
+
+ def __str__(self) -> str:
+ return self.name
+
+###############################################################################
+
+class ApplicationProfile(ApplicationProfileInfo):
+ """
+ An application profile defines how metadata can be specified.
+
+ Parameters
+ ----------
+ client
+ A Coscine Python SDK ApiClient for access to settings and requests.
+ data
+ ApplicationProfileInfo data as received by Coscine.
+ """
+
+ _data: dict
+
+ client: ApiClient
+ graph: rdflib.Graph
+ lock = Lock()
+
+ @property
+ def definition(self) -> str:
+ """
+ The actual application profile in text/turtle format.
+ """
+ return self._data.get("definition").get("content")
+
+ def __init__(self, client: ApiClient, data: dict) -> None:
+ super().__init__(data)
+ self.client = client
+ self.graph = rdflib.Graph()
+ self.graph.bind("sh", "http://www.w3.org/ns/shacl#")
+ self.graph.bind("dcterms", "http://purl.org/dc/terms/")
+ self.graph.parse(data=self.definition, format="ttl")
+ self._resolve_imports() # FIXME See method definition
+
+ def __str__(self) -> str:
+ return str(self.graph.serialize(format="ttl"))
+
+ def _resolve_imports(self) -> int:
+ """
+ Returns the number of resolved owl:imports.
+ FIXME We are not doing this recursive atm
+ """
+ count: int = 0
+ for row in self.query("SELECT ?url WHERE { ?_ owl:imports ?url . }"):
+ profile = self.client.application_profile(str(row[0]))
+ self.graph.parse(data=str(profile), format="ttl")
+ count += 1
+ return count
+
+ def query(self, query: str, **kwargs) -> list[list]:
+ """
+ Performs a SPARQL query on the application profile and
+ returns the results as a list of rows, with each row
+ containing as many columns as selected in the SPARQL query.
+
+ Warnings
+ ---------
+ Note that rdflib SPARQL queries are NOT thread-safe! Under the
+ hood pyparsing is invoked, which leads to a lot of trouble if
+ used in a multithreaded context. To avoid any problems the
+ Coscine Python SDK employs a lock on this function - only one
+ thread can use it at any given time.
+ TODO: Open pull request at rdflib and make rdflib itself thread-safe.
+
+ Parameters
+ ----------
+ query
+ A SPARQL query string.
+ **kwargs
+ Any number of keyword arguments to pass onto rdflib.query()
+ """
+ with self.lock:
+ results = self.graph.query(query, **kwargs)
+ results: list[list[rdflib.query.Result]]
+ items: list[list] = []
+ for row in results:
+ values = [
+ column.toPython()
+ if column else None
+ for column in row
+ ]
+ items.append(values)
+ return items
+
+ def fields(self) -> list[FormField]:
+ """
+ Returns the list of metadata fields with their properties as specified
+ in the application profile.
+ """
+ fields = []
+ for result in self.query(
+ "SELECT ?path ?name ?order ?class ?minCount ?maxCount\n"
+ "?minLength ?maxLength ?datatype\n"
+ "(GROUP_CONCAT(?in; SEPARATOR=\"~,~\") as ?ins)\n"
+ "(lang(?name) as ?lang)\n"
+ "WHERE {\n"
+ " ?_ sh:path ?path ;\n"
+ " sh:name ?name .\n"
+ " OPTIONAL { ?_ sh:order ?order . } .\n"
+ " OPTIONAL { ?_ sh:class ?class . } .\n"
+ " OPTIONAL { ?_ sh:minCount ?minCount . } .\n"
+ " OPTIONAL { ?_ sh:maxCount ?maxCount . } .\n"
+ " OPTIONAL { ?_ sh:minLength ?minLength . } .\n"
+ " OPTIONAL { ?_ sh:maxLength ?maxLength . } .\n"
+ " OPTIONAL { ?_ sh:datatype ?datatype . } .\n"
+ " OPTIONAL { ?_ sh:in/rdf:rest*/rdf:first ?in . } .\n"
+ "}\n"
+ "GROUP BY ?name\n"
+ "ORDER BY ASC(?order)\n"
+ ):
+ if result[10] == self.client.language:
+ data: dict = {
+ "path": result[0],
+ "name": result[1],
+ "order": result[2],
+ "class": result[3],
+ "minCount": result[4],
+ "maxCount": result[5],
+ "minLength": result[6],
+ "maxLength": result[7],
+ "datatype": result[8],
+ "selection": result[9],
+ "language": result[10]
+ }
+ fields.append(FormField(self.client, data))
+ return fields
+
+###############################################################################
+
+class FileMetadata:
+ """
+ The existing metadata to a file as returned by the Coscine API.
+ This metadata is by default in machine-readable format and not
+ human-readable.
+ """
+
+ _data: dict
+
+ @property
+ def path(self) -> str:
+ """
+ Path/Identifier of the metadata field.
+ """
+ return self._data.get("path") or ""
+
+ @property
+ def type(self) -> str:
+ """
+ Datatype of the value as a string.
+ """
+ return self._data.get("type") or ""
+
+ @property
+ def version(self) -> str:
+ """
+ Current metadata version string. The version is a Unix timestamp.
+ """
+ return self._data.get("version") or ""
+
+ @property
+ def versions(self) -> list[str]:
+ """
+ List of all metadata version strings. Versions are unix timestamps.
+ """
+ return list(self._data.get("availableVersions")) or []
+
+ @property
+ def definition(self) -> str:
+ """
+ The actual metadata in rdf turtle format.
+ """
+ turtle = self._data.get("definition")["content"]
+ return turtle
+
+ @property
+ def is_latest(self) -> bool:
+ """
+ Returns True if the current metadata is the newest metadata
+ for the file.
+ """
+ return self.version == max(self.versions)
+
+ def graph(self) -> rdflib.Graph:
+ """
+ The metadata parsed as rdflib graph.
+ """
+ return rdflib.Graph().parse(data=self.definition, format="ttl")
+
+ def fixed_graph(self, resource: Resource) -> rdflib.Graph:
+ """
+ """
+ graph = rdflib.Graph().parse(data=self.definition, format="ttl")
+ for s, p, o in graph.triples((None, None, None)):
+ base: str = (
+ f"https://purl.org/coscine/resources/{resource.id}/"
+ f"{self.path}/@type=metadata&version={self.version}"
+ )
+ graph.remove((s, p, o))
+ graph.add((rdflib.URIRef(base), p, o))
+ return graph
+
+ def items(self) -> list[dict[str, str]]:
+ """
+ Returns the list of metadata values in the format:
+ >>> [{
+ >>> "path": "...",
+ >>> "value": "...",
+ >>> "datatype": "..."
+ >>> }]
+ """
+ results = self.graph().query(
+ "SELECT ?property (str(?value) as ?value) "
+ "(datatype(?value) as ?type)\n"
+ "WHERE {\n"
+ " ?_ ?property ?value .\n"
+ " FILTER( ?property != rdf:type )\n"
+ "}\n"
+ )
+ items = []
+ for row in results:
+ path = str(row[0])
+ value = str(row[1])
+ datatype = str(row[2])
+ item = {
+ "path": path,
+ "value": value,
+ "datatype": datatype
+ }
+ items.append(item)
+ return items
+
+ def __init__(self, data: dict) -> None:
+ self._data = data
+
+ def __str__(self) -> str:
+ return self.path
+
+###############################################################################
+
+class MetadataForm:
+ """
+ The metadata form makes the meatadata fields that have been
+ defined in an application profile accessible to users.
+ """
+
+ _fields: list[FormField]
+ resource: Resource
+
+ @property
+ def client(self) -> ApiClient:
+ return self.resource.client
+
+ def __init__(self, resource: Resource) -> None:
+ self.resource = resource
+ self._fields = self.resource.application_profile.fields()
+
+ def __str__(self) -> str:
+ entries = []
+ for key in self.keys():
+ field = self.field(key)
+ entries.append((
+ field.is_required,
+ field.is_controlled,
+ field.datatype.__name__,
+ f"{field.min_count} - {field.max_count}",
+ key,
+ "\n".join([str(v) for v in field.values])
+ ))
+ headers: list[str] = [
+ "Required", "Controlled", "Type",
+ "Range", "Field", "Value"
+ ]
+ return tabulate(entries, headers=headers, disable_numparse=True)
+
+ def __setitem__(self, key: str, values: FormType | list[FormType]) -> None:
+ self.field(key).values = values
+
+ def __getitem__(self, key: str) -> list[FormType]:
+ return self.field(key).values
+
+ def __delitem__(self, key: str) -> None:
+ self.field(key).clear()
+
+ def __contains__(self, key: str) -> bool:
+ return key in self.keys()
+
+ def __iter__(self):
+ for key in self.keys():
+ yield key
+
+ def clear(self) -> None:
+ """
+ Clears all values.
+ """
+ for field in self.fields():
+ field.clear()
+
+ def fields(self) -> list[FormField]:
+ """
+ The list of metadata fields that can be filled in as defined
+ in the application profile.
+ """
+ return self._fields
+
+ def field(self, key: str) -> FormField:
+ """
+ Looks up a metadata field via its name.
+ """
+ for field in self._fields:
+ if field.name == key:
+ return field
+ raise KeyError(f"The field {key} is not part of the form!")
+
+ def path(self, path: str) -> FormField:
+ """
+ Looks up a metadata field via its path.
+ """
+ for field in self._fields:
+ if field.path == path:
+ return field
+ raise KeyError(f"The field path {path} is not part of the form!")
+
+ def keys(self) -> list[str]:
+ """
+ Returns the list of names of all metadata fields.
+ """
+ return [field.name for field in self._fields]
+
+ def values(self) -> list[list[FormType]]:
+ """
+ Returns the list of values of all metadata fields.
+ """
+ return [field.values for field in self._fields]
+
+ def items(self) -> list[Tuple[str, FormType]]:
+ """
+ Returns key, value pairs for all metadata fields
+ """
+ return zip(self.keys(), self.values())
+
+ def graph(self) -> rdflib.Graph:
+ """
+ """
+ root = rdflib.BNode()
+ graph = rdflib.Graph()
+ graph.add((
+ root,
+ rdflib.RDF.type,
+ rdflib.URIRef(self.resource.application_profile.uri)
+ ))
+ for field in self._fields:
+ if field.values:
+ for value in field.identifiers:
+ graph.add((
+ root,
+ rdflib.URIRef(field.path),
+ value
+ ))
+ return graph
+
+ def validate(self) -> bool:
+ """
+ """
+ ontologies = rdflib.Graph()
+ for field in self.fields():
+ if field.has_vocabulary:
+ ontologies += field.vocabulary.graph()
+ graph = self.graph() + ontologies
+ conforms, results_graph, results_text = pyshacl.validate(
+ graph,
+ shacl_graph=self.resource.application_profile.graph,
+ ont_graph=ontologies,
+ debug=False,
+ inference="rdfs",
+ abort_on_first=False,
+ allow_infos=False,
+ allow_warnings=False,
+ meta_shacl=True,
+ advanced=False,
+ js=False
+ )
+ if not conforms:
+ raise coscine.exceptions.ValueError(results_text)
+ return conforms
+
+ def test(self) -> None:
+ """
+ Auto-fills the MetadataForm with a set of predefined values.
+ Every field is filled in.
+ """
+ def generate_value(field: FormField):
+ length = field.min_length if field.min_length > 1 else 8
+ if length > field.max_length:
+ length = field.max_length
+ sample_data = {
+ datetime: datetime.now(),
+ date: datetime.now().date(),
+ time: datetime.now().time(),
+ int: random.randint(1, 16),
+ float: random.random() * 123.0,
+ Decimal: Decimal(random.randint(1, 42)),
+ bool: True,
+ str: "".join(random.choices(ascii_letters, k=length))
+ }
+ if field.has_vocabulary:
+ length = len(field.vocabulary.keys()) - 1
+ return field.vocabulary.keys()[random.randint(0, length)]
+ elif field.has_selection:
+ length = len(field.selection) - 1
+ return field.selection[random.randint(0, length)]
+ elif field.datatype == "str" and field.min_length > 0:
+ return "X" * field.min_length
+ elif field.datatype == "str" and field.max_length < 4096:
+ return "X" * (field.max_length - 1)
+ else:
+ return sample_data[field.datatype]
+
+ for field in self.fields():
+ values = []
+ if field.min_count > 1:
+ for _ in range(field.min_count):
+ values.append(generate_value(field))
+ else:
+ values.append(generate_value(field))
+ field.values = values
+
+ def generate(self, path: str) -> dict:
+ """
+ Prepares and validates metadata for sending to Coscine.
+ Requires the file path of the file in Coscine as an argument.
+
+ Parameters
+ ----------
+ path
+ The path in Coscine to the FileObject that you would like
+ to attach metadata to.
+ """
+ return {
+ "path": path,
+ "definition": {
+ "content": self.graph().serialize(format="ttl"),
+ "type": "text/turtle"
+ }
+ }
+
+ def parse(self, data: FileMetadata) -> None:
+ """
+ Parses existing metadata that was received from Coscine.
+ """
+ for item in data.items():
+ self.path(item["path"]).append(item["value"], True)
diff --git a/src/coscine/object.py b/src/coscine/object.py
deleted file mode 100644
index 646a24ec48a7a5c07b42243455b10d3f62706fbe..0000000000000000000000000000000000000000
--- a/src/coscine/object.py
+++ /dev/null
@@ -1,551 +0,0 @@
-###############################################################################
-# Coscine Python SDK
-# Copyright (c) 2018-2023 RWTH Aachen University
-# Licensed under the terms of the MIT License
-###############################################################################
-# Coscine, short for Collaborative Scientific Integration Environment, is
-# a platform for research data management (RDM).
-# For more information on Coscine visit https://www.coscine.de/.
-#
-# Please note that this python module is open source software primarily
-# developed and maintained by the scientific community. It is not
-# an official service that RWTH Aachen provides support for.
-###############################################################################
-
-###############################################################################
-# File description
-###############################################################################
-
-"""
-Implements classes and routines for manipulating Metadata and interacting
-with files and file-like data in Coscine.
-"""
-
-###############################################################################
-# Dependencies
-###############################################################################
-
-from __future__ import annotations
-from datetime import datetime
-from typing import List, Optional, TYPE_CHECKING, Callable, Union
-import os
-import posixpath
-import logging
-import dateutil.parser
-from prettytable.prettytable import PrettyTable
-from coscine.graph import ApplicationProfile
-from coscine.form import InputForm, FormField
-from coscine.utils import HumanBytes, ProgressBar, parallelizable
-if TYPE_CHECKING:
- from coscine.client import Client
- from coscine.resource import Resource
-
-###############################################################################
-# Module globals / Constants
-###############################################################################
-
-logger = logging.getLogger(__name__)
-
-###############################################################################
-# Classes / Functions / Scripts
-###############################################################################
-
-class MetadataForm(InputForm):
- """
- The MetadataForm supports parsing coscine.Object metadata, generating
- metadata and manipulating it in the same way one would manipulate
- a python dict.
- """
-
- profile: ApplicationProfile
-
-###############################################################################
-
- def __init__(self, client: Client, graph: ApplicationProfile) -> None:
- """
- Initializes an instance of type MetadataForm.
-
- Parameters
- ----------
- client : Client
- Coscine Python SDK client handle
- graph : ApplicationProfile
- Coscine application profile rdf graph
- """
-
- super().__init__(client)
- self.profile = graph
- self._fields = [
- FormField(client, item) for item in self.profile.items()
- ]
- # Query vocabularies for controlled fields
- for field in self._fields:
- if field.vocabulary:
- instance = client.vocabularies.instance(field.vocabulary)
- self._vocabularies[field.path] = instance
-
-###############################################################################
-
- def generate(self) -> dict:
- """
- Generates and validates metadata for sending to Coscine.
- """
-
- metadata = {}
- metadata[ApplicationProfile.RDFTYPE] = [{
- "type": "uri",
- "value": self.profile.target()
- }]
-
- # Collect missing required fields
- missing = []
-
- # Set metadata
- for key, values in self.items():
- if not values:
- if self.is_required(key):
- missing.append(key)
- continue
-
- properties = self.properties(key)
- metadata[properties.path] = []
- raw_value = values.raw()
- if not isinstance(raw_value, list):
- raw_value = [raw_value]
- for value in raw_value:
- entry = {
- "value": value,
- "datatype": (
- properties.datatype if properties.datatype
- else properties.vocabulary
- ),
- "type": "uri" if properties.vocabulary else "literal"
- }
- if not entry["datatype"]:
- del entry["datatype"]
- metadata[properties.path].append(entry)
-
- # Check for missing required fields
- if len(missing) > 0:
- raise ValueError(missing)
-
- return metadata
-
-###############################################################################
-
- def parse(self, data: Union[dict, None]) -> None:
- if data is None:
- return
- for path, values in data.items():
- if path == ApplicationProfile.RDFTYPE:
- continue
- try:
- key = self.name_of(path)
- self.set_value(key, [v["value"] for v in values], True)
- except KeyError:
- continue
-
-###############################################################################
-###############################################################################
-###############################################################################
-
-class FileActions:
- """
- Provides FileObject Action URLs that provide direct access to a FileObject
- without requiring authentication/credentials. Once generated,
- the URLs are only valid for a limited time.
- """
-
- _data: dict
-
- def __init__(self, data: dict) -> None:
- self._data = data
-
- @property
- def download(self) -> str:
- """
- Returns the direct download URL for an object of type file.
- The URL is valid for 24 hours and requires no authentication.
- """
- if "Download" in self._data:
- return self._data["Download"]["Url"]
- return ""
-
- @property
- def delete(self) -> str:
- """
- Returns the direct download URL for an object of type file.
- The URL is valid for 24 hours and requires no authentication.
- """
- return self._data["Delete"]["Url"] if "Delete" in self._data else ""
-
- @property
- def update(self) -> str:
- """
- Returns the direct download URL for an object of type file.
- The URL is valid for 24 hours and requires no authentication.
- """
- return self._data["Update"]["Url"] if "Update" in self._data else ""
-
-###############################################################################
-###############################################################################
-###############################################################################
-
-class FileObject:
- """
- Objects in Coscine represent file-like data. We could have called it
- 'File', but in case of linked data we are not actually dealing with files
- themselves, but with links to files.Thus we require a more general
- datatype.
- """
-
- client: Client
- resource: Resource
- data: dict
- actions: FileActions
- _cached_metadata: Optional[dict] # None in case of no metadata
- _metadata_cache_is_invalid: bool
-
- @property
- def has_metadata(self) -> bool:
- """
- Evaluates whether the FileObject has metadata attached to it
- """
- return bool(self._cached_metadata)
-
- @property
- def name(self) -> str:
- """
- FileObject name
- """
- return self.data["Name"]
-
- @property
- def size(self) -> int:
- """
- FileObject size in bytes
- """
- return int(self.data["Size"])
-
- @property
- def type(self) -> str:
- """
- FileObject type, e.g. 'file' or 'folder'
- """
- return self.data["Kind"]
-
- @property
- def modified(self) -> datetime:
- """
- Last time the FileObject was modified
- """
- if not self.data["Modified"]:
- return datetime(1900, 1, 1)
- return dateutil.parser.parse(self.data["Modified"])
-
- @property
- def created(self) -> datetime:
- """
- Time the FileObject was created
- """
- if not self.data["Created"]:
- return datetime(1900, 1, 1)
- return dateutil.parser.parse(self.data["Created"])
-
- @property
- def provider(self) -> str:
- """
- Resource type
- """
- return self.data["Provider"]
-
- @property
- def filetype(self) -> str:
- """
- FileObject file type, e.g. '.png'
- """
- return os.path.splitext(self.data["Name"])[1]
-
- @property
- def path(self) -> str:
- """
- FileObject path including folders (in case of S3 resources)
- """
- return self.data["Path"]
-
- @property
- def is_folder(self) -> bool:
- """
- Evaluates to true if the FileObject resembles a folder (only S3)
- """
- return bool(self.data["IsFolder"])
-
- CHUNK_SIZE: int = 4096
-
-###############################################################################
-
- def __init__(
- self,
- resource: Resource, data: dict,
- metadata: Optional[dict] = None
- ) -> None:
- """
- Initializes the Coscine FileObject.
-
- Parameters
- ----------
- resource : Resource
- Coscine resource handle.
- data : dict
- data of the file-like object.
- """
-
- # Configuration
- self.client = resource.client
- self.resource = resource
- self.data = data
- self.actions = FileActions(data["Action"])
- self._cached_metadata = metadata
- self._metadata_cache_is_invalid = False
-
-###############################################################################
-
- def __str__(self) -> str:
- table = PrettyTable(("Property", "Value"))
- rows = [
- ("Name", self.name),
- ("Size", HumanBytes.format(self.size)),
- ("Type", self.type),
- ("Path", self.path),
- ("Folder", self.is_folder)
- ]
- table.max_width["Value"] = 50
- table.add_rows(rows)
- return table.get_string(title=f"Object {self.name}")
-
-###############################################################################
-
- def metadata(self, force_update: bool = False) -> Union[dict, None]:
- """
- Retrieves the metadata of the file-like object.
-
- Parameters
- ----------
- force_update : bool, default: False
- Normally, metadata is cached and that cache is only updated
- when an action is performed via the Coscine Python SDK, that
- would invalidate that cache (e.g. update()).
- By setting force_update to True, metadata is always fetched
- from Coscine, ignoring the cache. Keep in mind that fetching
- the metadata from every file in a resource will result in a
- seperate request for each file, while with caching it only
- fetches all metadata once.
-
- Returns
- -------
- dict
- Metadata as a python dictionary.
- None
- If no metadata has been set for the file-like object.
- """
-
- if not (force_update or self._metadata_cache_is_invalid):
- if self._cached_metadata:
- for key in self._cached_metadata:
- return self._cached_metadata[key]
- else:
- return None
- uri = self.client.uri("Tree", "Tree", self.resource.id)
- args = {"path": self.path}
- data = self.client.get(uri, params=args).json()
- metadata = data["data"]["metadataStorage"]
- if not metadata:
- return None
- metadata = metadata[0]
- for key in metadata:
- self._cached_metadata = metadata
- self._metadata_cache_is_invalid = False
- return metadata[key]
- return None
-
-###############################################################################
-
- @parallelizable
- def update(self, metadata: Union[MetadataForm, dict]) -> None:
- """
- Updates the metadata of the file-like object.
-
- Parameters
- ----------
- metadata : MetadataForm or dict
- MetadataForm or JSON-LD formatted metadata dict
-
- Raises
- ------
- TypeError
- If argument `metadata` has an unexpected type.
- """
-
- if isinstance(metadata, MetadataForm):
- metadata = metadata.generate()
- elif not isinstance(metadata, dict):
- raise TypeError("Expected MetadataForm or dict.")
- logger.info("Updating metadata of FileObject '%s'...", self.path)
- uri = self.client.uri("Tree", "Tree", self.resource.id)
- self.client.put(uri, params={"path": self.path}, json=metadata)
- self._metadata_cache_is_invalid = True # Invalidate cached metadata
-
-###############################################################################
-
- @parallelizable
- def content(self) -> bytes:
- """
- Retrieves the content/data of the object. In case of linked data
- this would be the link, not the actual file itself. It is impossible
- to get the contents of linked data objects with this python module.
- In case of rds or rds-s3 data this would return the file contents.
- Be aware that for very large files this will consume a considerable
- amount of RAM!
-
- Returns
- -------
- bytes
- A raw byte-array containing the Coscine file-object's data.
- """
-
- uri = self.client.uri("Blob", "Blob", self.resource.id)
- return self.client.get(uri, params={"path": self.path}).content
-
-###############################################################################
-
- @parallelizable
- def download(
- self,
- path: str = "./",
- callback: Optional[Callable[[int], None]] = None,
- preserve_path: Optional[bool] = False
- ) -> None:
- """
- Downloads the file-like object to the local harddrive.
-
- Parameters
- ----------
- path : str, default: "./"
- The path to the download location on the harddrive.
- callback : function(chunksize: int)
- A callback function to be called during downloading chunks.
- preserve_path : bool, default: False
- Preserve the folder structure, i.e. if the file object is located
- in a subfolder, create said subfolder if it does not exist yet
- on the local harddrive and put the file in there.
- """
-
- logger.info("Downloading FileObject '%s'...", self.path)
- if self.is_folder:
- if preserve_path:
- path = posixpath.join(path, self.path)
- os.makedirs(path, exist_ok=True)
- else:
- raise TypeError(
- "FileObject is a folder! "
- "Set 'preserve_path=True' to download folders."
- )
- else:
- if "Action" in self.data:
- uri = self.data["Action"]["Download"]["Url"]
- response = self.client.get(uri, stream=True)
- else:
- uri = self.client.uri("Blob", "Blob", self.resource.id)
- args = {"path": self.path}
- response = self.client.get(uri, params=args, stream=True)
- if preserve_path:
- path = posixpath.join(path, self.path)
- else:
- path = os.path.join(path, self.name)
- with open(path, 'wb') as file:
- progress_bar = ProgressBar(
- self.client.settings.verbose,
- self.size, self.name, callback
- )
- for chunk in response.iter_content(chunk_size=self.CHUNK_SIZE):
- file.write(chunk)
- progress_bar.update(len(chunk))
-
-###############################################################################
-
- @parallelizable
- def delete(self) -> None:
- """
- Deletes the file-like object on the Coscine server.
- """
-
- logger.info("Deleting FileObject '%s'...", self.path)
- uri = self.client.uri("Blob", "Blob", self.resource.id)
- self.client.delete(uri, params={"path": self.path})
-
-###############################################################################
-
- def objects(self, path: str = "") -> List[FileObject]:
- """
- Returns a list of FileObjects resembling the contents of
- the current object in case the current object is a folder.
- Irrelevant for anything other than S3 resources.
-
- Parameters
- -----------
- path : str, default: None
- Path to a directory. Irrelevant for anything other than S3.
-
- Returns
- -------
- List[FileObject]
- A list of file objects.
-
- Raises
- -------
- TypeError
- In case the current FileObject is not a folder or not part of an
- S3 resource.
- """
- if self.is_folder:
- fullpath = posixpath.join(self.path, path)
- return self.resource.objects(fullpath)
- raise TypeError(f"FileObject '{self.path}' is not a directory!")
-
-###############################################################################
-
- def object(self, path: str) -> Optional[FileObject]:
- """
- Returns a FileObject resembling a file inside of
- the current object in case the current object is a folder.
- Irrelevant for anything other than S3 resources.
-
- Parameters
- -----------
- path : str
- Path to the file or folder.
-
- Raises
- -------
- TypeError
- In case the current FileObject is not a folder or not part of an
- S3 resource.
- """
- if self.is_folder:
- fullpath = posixpath.join(self.path, path)
- return self.resource.object(fullpath)
- raise TypeError(f"FileObject '{self.path}' is not a directory!")
-
-###############################################################################
-
- def form(self, force_update: bool = False) -> MetadataForm:
- """
- Returns a MetadataForm to interact with the metadata of the FileObject.
- """
-
- graph = self.resource.application_profile()
- form = MetadataForm(self.client, graph)
- form.parse(self.metadata(force_update))
- return form
-
-###############################################################################
diff --git a/src/coscine/project.py b/src/coscine/project.py
index 91bb234351fb9d7701db37ca2ddf99aa5e927b02..8f5078ee0a81ed2a5b6b73bfa726a3b3ac1a4016 100644
--- a/src/coscine/project.py
+++ b/src/coscine/project.py
@@ -1,661 +1,684 @@
-###############################################################################
-# Coscine Python SDK
-# Copyright (c) 2018-2023 RWTH Aachen University
-# Licensed under the terms of the MIT License
-###############################################################################
-# Coscine, short for Collaborative Scientific Integration Environment, is
-# a platform for research data management (RDM).
-# For more information on Coscine visit https://www.coscine.de/.
-#
-# Please note that this python module is open source software primarily
-# developed and maintained by the scientific community. It is not
-# an official service that RWTH Aachen provides support for.
-###############################################################################
-
-###############################################################################
-# File description
-###############################################################################
-
-"""
-This file defines the project object for the representation of
-Coscine projects. It provides a simple interface to interact with Coscine
-projects from python.
-"""
-
-###############################################################################
-# Dependencies
-###############################################################################
-
-from __future__ import annotations
-from typing import Optional, TYPE_CHECKING, Union, List
-import json
-import os
-from datetime import datetime
-import logging
-import dateutil.parser
-from prettytable.prettytable import PrettyTable
-from coscine.resource import Resource, ResourceForm
-from coscine.form import InputForm
-from coscine.utils import parallelizable
-if TYPE_CHECKING:
- from coscine.client import Client
-
-###############################################################################
-# Module globals / Constants
-###############################################################################
-
-logger = logging.getLogger(__name__)
-
-###############################################################################
-# Classes / Functions / Scripts
-###############################################################################
-
-class ProjectForm(InputForm):
- """
- An InputForm for Coscine project metadata. Use this form
- to generate or edit project metadata such as the project name
- and start date.
- """
-
-###############################################################################
-
- def __init__(self, client: Client) -> None:
- """
- Generates a new instance of type ProjectForm.
-
- Parameters
- ----------
- client : Client
- Coscine Python SDK Client handle
- """
- super().__init__(client)
- self._fields = client.vocabularies.builtin("project")
- vocabularies = {
- "disciplines": client.vocabularies.disciplines(True),
- "visibility": client.vocabularies.visibility(True)
- }
- for item in self._fields:
- if item.vocabulary:
- self._vocabularies[item.path] = vocabularies[item.path]
-
-###############################################################################
-
- @staticmethod
- def _parse_organizations(data: List):
- return [value["url"] for value in data]
-
-###############################################################################
-
- def parse(self, data: dict) -> None:
- for path, value in data.items():
- if path in ["id", "slug", "parentId"]: # ignored paths
- continue
- if path == "organizations":
- value = self._parse_organizations(value)
- try:
- key = self.name_of(path)
- self.set_value(key, value, True)
- except KeyError:
- continue
-
-###############################################################################
-
- def generate(self) -> dict:
- metadata = {}
- missing = []
-
- for key, value in self.items():
- if not value:
- if self.is_required(key):
- missing.append(key)
- continue
- properties = self.properties(key)
- metadata[properties.path] = value.raw()
-
- # The organizations field is normally controlled, but we omit
- # actively controlling it, because doing so is too complicated.
- # Because we handle things differently here, some special care
- # is needed when preparing it for Coscine:
- if properties.path == "organizations":
- metadata[properties.path] = [{
- "displayName": url, "url": url
- } for url in value.raw()]
- if missing:
- raise ValueError(missing)
- return metadata
-
-###############################################################################
-# Class definition
-###############################################################################
-
-class Project:
- """
- Python representation of a Coscine Project
- """
-
- client: Client
- data: dict
- parent: Optional[Project]
-
-###############################################################################
-
- def __init__(
- self,
- client: Client,
- data: dict,
- parent: Optional[Project] = None
- ) -> None:
- """
- Initializes a Coscine project object.
-
- Parameters
- ----------
- client : Client
- Coscine client handle
- data : dict
- Project data received from Coscine.
- parent : Project, default: None
- Optional parent project.
- """
-
- self.client = client
- self.data = data
- self.parent = parent
-
-###############################################################################
-
- @property
- def id(self) -> str:
- """
- Project ID
- """
- return self.data["id"]
-
- @property
- def name(self) -> str:
- """
- Project Name
- """
- return self.data["projectName"]
-
- @property
- def display_name(self) -> str:
- """
- Project Display Name
- """
- return self.data["displayName"]
-
- @property
- def description(self) -> str:
- """
- Project Description
- """
- return self.data["description"]
-
- @property
- def principle_investigators(self) -> str:
- """
- Project PIs
- """
- return self.data["principleInvestigators"]
-
- @property
- def start_date(self) -> datetime:
- """
- Project Start Date as datetime object
- """
- return dateutil.parser.parse(self.data["startDate"])
-
- @property
- def end_date(self) -> datetime:
- """
- Project End Date as datetime object
- """
- return dateutil.parser.parse(self.data["endDate"])
-
- @property
- def disciplines(self) -> List[str]:
- """
- Project Disciplines
- """
- lang = {
- "en": "displayNameEn",
- "de": "displayNameDe"
- }[self.client.settings.language]
- return [k[lang] for k in self.data["disciplines"]]
-
- @property
- def organizations(self) -> List[str]:
- """
- Project Associated Organizations
- """
- return [k["displayName"] for k in self.data["organizations"]]
-
- @property
- def visibility(self) -> str:
- """
- Project Visibility in Coscine
- """
- return self.data["visibility"]["displayName"]
-
-###############################################################################
-
- def __str__(self) -> str:
- table = PrettyTable(("Property", "Value"))
- rows = [
- ("ID", self.id),
- ("Name", self.name),
- ("Display Name", self.display_name),
- ("Description", self.description),
- ("Principle Investigators", self.principle_investigators),
- ("Disciplines", "\n".join(self.disciplines)),
- ("Organizations", "\n".join(self.organizations)),
- ("Start Date", self.start_date),
- ("End Date", self.end_date),
- ("Visibility", self.visibility)
- ]
- table.max_width["Value"] = 50
- table.add_rows(rows)
- return table.get_string(title=f"Project {self.display_name}")
-
-###############################################################################
-
- def subprojects(self) -> List[Project]:
- """
- Retrieves a list of a all projects the creator of the Coscine API token
- is currently a member of.
-
- Returns
- -------
- list
- List of coscine.Project objects
- """
-
- uri = self.client.uri("Project", "SubProject", self.id)
- projects = []
- for data in self.client.get(uri).json():
- projects.append(Project(self.client, data, self))
- return projects
-
-###############################################################################
-
- def subproject(self, display_name: str) -> Optional[Project]:
- """
- Returns a single subproject via its displayName
-
- Parameters
- ----------
- display_name : str
- Look for a project with the specified displayName
-
- Returns
- -------
- Project
- A single coscine project handle or None if no match found
-
- Raises
- ------
- IndexError
- """
-
- filtered_project_list = list(filter(
- lambda project: project.display_name == display_name,
- self.subprojects()
- ))
- if len(filtered_project_list) == 1:
- return filtered_project_list[0]
- if len(filtered_project_list) == 0:
- return None
- raise IndexError("Too many projects matching the specified criteria!")
-
-###############################################################################
-
- def create_subproject(self, form: ProjectForm) -> Project:
- """
- Creates a subproject inside of the current project
- """
- metadata = form.generate()
- metadata["ParentId"] = self.id
- return self.client.create_project(metadata)
-
-###############################################################################
-
- def delete(self) -> None:
- """
- Deletes the project on the Coscine servers.
- """
-
- uri = self.client.uri("Project", "Project", self.id)
- self.client.delete(uri)
-
-###############################################################################
-
- def resources(self) -> List[Resource]:
- """
- Retrieves a list of Resources of the current project.
-
- Returns
- -------
- list[Resource]
- list of resources matching the supplied filter.
- """
-
- uri = self.client.uri("Project", "Project", self.id, "resources")
- resources = []
- for data in self.client.get(uri).json():
- resources.append(Resource(self, data))
- return resources
-
-###############################################################################
-
- def resource(self, display_name: str) -> Resource:
- """
- Retrieves a certain resource of the current project
- identified by its displayName.
-
- Parameters
- ----------
- display_name : str
- The display name of the resource.
-
- Returns
- --------
- A single Coscine Resource handle
-
- Raises
- -------
- IndexError
- FileNotFoundError
- In case no resource with the specified display_name was found.
- """
-
- resources = list(filter(
- lambda resource: resource.display_name == display_name,
- self.resources()
- ))
- if len(resources) == 1:
- return resources[0]
- if len(resources) == 0:
- raise FileNotFoundError("Could not find resource!")
- raise IndexError("Too many resources matching the specified criteria!")
-
-###############################################################################
-
- def download(self, path: str = "./", metadata: bool = False) -> None:
- """
- Downloads the project to the location referenced by 'path'.
-
- Parameters
- ----------
- path : str
- Download location on the harddrive
- Default: current directory './'
- metadata : bool, default: False
- If enabled, project metadata is downloaded and put in
- a hidden file '.metadata.json'.
- """
-
- logger.info("Downloading project '%s' (%s)...", self.name, self.id)
- path = os.path.join(path, self.display_name)
- if not os.path.isdir(path):
- os.mkdir(path)
- for resource in self.resources():
- resource.download(path=path, metadata=metadata)
- if metadata:
- data = json.dumps(self.data, indent=4)
- metadata_path: str = os.path.join(path, ".metadata.json")
- with open(metadata_path, "w", encoding="utf-8") as file:
- file.write(data)
-
-###############################################################################
-
- def members(self) -> List[ProjectMember]:
- """
- Retrieves a list of all members of the current project
-
- Returns
- --------
- list[ProjectMember]
- List of project members as ProjectMember objects.
- """
-
- uri = self.client.uri("Project", "ProjectRole", self.id)
- data = self.client.get(uri).json()
- members = [ProjectMember(self, m) for m in data]
- return members
-
-###############################################################################
-
- @parallelizable
- def invite(self, email: str, role: str = "Member") -> None:
- """
- Invites a person to a project via their email address
-
- Parameters
- ----------
- email : str
- The email address to send the invite to
- role : str, "Member", "Guest" or "Owner", default: "Member"
- The role for the new project member
- """
-
- if role not in ProjectMember.ROLES:
- raise ValueError(f"Invalid role {role}.")
-
- logger.info("Inviting %s as %s to project %s.", email, role, self.name)
- uri = self.client.uri("Project", "Project", "invitation")
- data = {
- "projectId": self.data["id"],
- "role": ProjectMember.ROLES[role],
- "email": email
- }
-
- try:
- self.client.post(uri, json=data)
- except RuntimeError:
- logger.warning("User %s has pending invites.", email)
-
-###############################################################################
-
- @parallelizable
- def add_member(self, member: ProjectMember, role: str = "Member"):
- """
- Adds a project member of another project to the current project.
-
- Parameters
- ----------
- member : ProjectMember
- Member of another Coscine project
- role : str, "Member", "Guest" or "Owner", default: "Member"
-
- Raises
- ------
- ValueError
- In case the specified role is unsupported
- """
-
- if role not in ProjectMember.ROLES:
- raise ValueError(f"Invalid role for member '{member.name}'!")
- data = member.data
- data["projectId"] = self.id
- data["role"]["displayName"] = role
- data["role"]["id"] = ProjectMember.ROLES[role]
- uri = self.client.uri("Project", "ProjectRole")
- self.client.post(uri, json=data)
- logger.info(
- "Added '%s' as a project member with "
- "role '%s' to project '%s'.",
- member.name, role, self.name
- )
-
-###############################################################################
-
- def form(self) -> ProjectForm:
- """
- Returns the project metadata form of the project. That form can
- then be used to edit the project metadata.
-
- Examples
- ---------
- >>> metadata = Project.form()
- >>> metadata["Display Name"] = "Different display name"
- >>> Project.update(metadata)
- """
-
- form = ProjectForm(self.client)
- form.parse(self.data)
- return form
-
-###############################################################################
-
- def update(self, form: Union[ProjectForm, dict]) -> Project:
- """
- Updates a project using the given ProjectForm
-
- Parameters
- ----------
- form : ProjectForm or dict
- ProjectForm containing updated data or dict generated from a form.
- """
-
- if isinstance(form, ProjectForm):
- form = form.generate()
- uri = self.client.uri("Project", "Project", self.id)
- response = self.client.post(uri, json=form)
- logger.info(
- "Updated project metadata for '%s' (%s).",
- self.name, self.id
- )
- updated_project_handle = self
- if response.ok:
- # Update project properties
- # The response unfortunately does not return
- # the new project data - bummer!
- updated_project_handle = list(filter(
- lambda p: p.id == self.id,
- self.client.projects(False)
- ))[0]
- return updated_project_handle
-
-###############################################################################
-
- @parallelizable
- def create_resource(self, form: Union[ResourceForm, dict]) -> Resource:
- """
- Creates a resource within the current project using the supplied
- resource form.
-
- Parameters
- ----------
- resourceForm : ResourceForm
- Form to generate the resource with.
- metadataPreset : MetadataPresetForm
- optional application profile configuration.
- Currently not supported.
- """
- if isinstance(form, ResourceForm):
- form = form.generate()
- uri = self.client.uri("Resources", "Resource", "project", self.id)
- logger.info("Creating resource in project '%s'.", self.name)
- return Resource(self, self.client.post(uri, json=form).json())
-
-###############################################################################
-###############################################################################
-
-class ProjectMember:
- """
- Python representation of a Coscine project member.
- """
-
- ROLES = {
- "Owner": "be294c5e-4e42-49b3-bec4-4b15f49df9a5",
- "Member": "508b6d4e-c6ac-4aa5-8a8d-caa31dd39527",
- "Guest": "9184a442-4419-4e30-9fe6-0cfe32c9a81f"
- }
-
- data: dict
- client: Client
- project: Project
-
-###############################################################################
-
- @property
- def name(self) -> str:
- """Name of the Project Member"""
- return self.data["user"]["displayName"]
-
- @property
- def email(self) -> str:
- """E-Mail address of the Project Member"""
- return self.data["user"]["emailAddress"]
-
- @property
- def id(self) -> str:
- """Project Member ID"""
- return self.data["user"]["id"]
-
- @property
- def role(self) -> str:
- """Role of the Project Member within the Project"""
- return self.data["role"]["displayName"]
-
-###############################################################################
-
- def __init__(self, project: Project, data: dict) -> None:
- """
- Initializes a project member for a given project.
-
- Parameters
- ----------
- project : Project
- Coscine Python SDK project handle.
- data: dict
- User data as dict, retrieved via
- client.uri("Project", "ProjectRole", self.id).
- """
-
- self.project = project
- self.client = self.project.client
- self.data = data
-
-###############################################################################
-
- @parallelizable
- def set_role(self, role: str) -> None:
- """
- Sets the role of a project member
-
- Parameters
- ----------
- role : str
- The new role of the member ('Owner', 'Guest' or 'Member').
- """
-
- if role not in ProjectMember.ROLES:
- raise ValueError(f"Invalid role {role}.")
-
- uri = self.client.uri("Project", "ProjectRole")
- self.data["role"]["id"] = ProjectMember.ROLES[role]
- self.data["role"]["displayName"] = role
- logger.info("Setting role of '%s' to '%s'...", self.name, role)
- self.client.post(uri, json=self.data)
-
-###############################################################################
-
- @parallelizable
- def remove(self) -> None:
- """
- Removes a project member from their associated project.
- """
-
- uri = self.client.uri(
- "Project", "ProjectRole", "project", self.project.id, "user",
- self.id, "role", self.data["role"]["id"]
- )
- logger.info(
- "Removing member '%s' from project '%s'...",
- self.name, self.project.name
- )
- self.client.delete(uri)
-
-###############################################################################
+###############################################################################
+# Coscine Python SDK
+# Copyright (c) 2020-2023 RWTH Aachen University
+# Licensed under the terms of the MIT License
+# For more information on Coscine visit https://www.coscine.de/.
+###############################################################################
+
+"""
+"""
+
+from __future__ import annotations
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+ from coscine.client import ApiClient
+from os import mkdir
+from os.path import isdir, join
+from datetime import date
+from dateutil.parser import parse as dateutil_parse
+from tabulate import tabulate
+from textwrap import wrap
+from coscine.common import Discipline, License, User, Visibility
+from coscine.resource import (
+ Resource, ResourceType, ResourceQuota
+)
+import coscine.exceptions
+
+###############################################################################
+
+class ProjectQuota:
+ """
+ Projects have a set of storage space quotas. This class models
+ the quota data returned by Coscine.
+ """
+
+ _data: dict
+
+ @property
+ def project_id(self) -> str:
+ """
+ The ID of the associated project.
+ """
+ return self._data.get("projectId") or ""
+
+ @property
+ def total_used(self) -> int:
+ """
+ The total used storage space in bytes.
+ """
+ return int(self._data.get("totalUsed").get("value") * 1024**3)
+
+ @property
+ def total_reserved(self) -> int:
+ """
+ The total reserved storage space in bytes.
+ """
+ return int(self._data.get("totalReserved").get("value") * 1024**3)
+
+ @property
+ def allocated(self) -> int:
+ """
+ The allocated storage space in bytes.
+ """
+ return int(self._data.get("allocated").get("value") * 1024**3)
+
+ @property
+ def maximum(self) -> int:
+ """
+ The maximum available storage space in bytes.
+ """
+ value = self._data.get("maximum").get("value") or 0
+ return value * int( * 1024**3)
+
+ @property
+ def resource_type(self) -> ResourceType:
+ """
+ The associated resource type.
+ """
+ return ResourceType(self._data.get("resourceType"))
+
+ @property
+ def resource_quotas(self) -> list[ResourceQuota]:
+ """
+ The list of used resource quotas for the project.
+ """
+ return [
+ ResourceQuota(data)
+ for data in self._data.get("resourceQuotas")
+ ]
+
+ def __init__(self, data: dict) -> None:
+ self._data = data
+
+###############################################################################
+
+class ProjectRole:
+ """
+ Models roles that can be assumed by project members
+ within a Coscine project.
+ """
+
+ _data: dict
+
+ @property
+ def id(self) -> str:
+ """
+ Unique and constant Coscine-internal identifier of the role.
+ """
+ return self._data.get("id") or ""
+
+ @property
+ def name(self) -> str:
+ """
+ Name of the role.
+ """
+ return self._data.get("displayName") or ""
+
+ @property
+ def description(self) -> str:
+ """
+ Description for the role.
+ """
+ return self._data.get("description") or ""
+
+ def __init__(self, data: dict) -> None:
+ self._data = data
+
+ def __str__(self) -> str:
+ return self.name
+
+###############################################################################
+
+class ProjectMember:
+ """
+ This class models the members of a Coscine project.
+ """
+
+ _data: dict
+
+ @property
+ def id(self) -> str:
+ """
+ Unique Coscine-internal project member identifier.
+ """
+ return self._data.get("id") or ""
+
+ @property
+ def user(self) -> User:
+ """
+ The user in Coscine that represents the project member.
+ """
+ return User(self._data.get("user"))
+
+ @property
+ def role(self) -> ProjectRole:
+ """
+ The role of the member within the project.
+ """
+ return ProjectRole(self._data.get("role"))
+
+ def __init__(self, data: dict) -> None:
+ self._data = data
+
+ def __str__(self) -> str:
+ return f"{self.user.display_name} as {self.role}"
+
+###############################################################################
+
+class ProjectInvitation:
+ """
+ Models external user invitations via email in Coscine.
+ """
+
+ _data: dict
+
+ @property
+ def id(self) -> str:
+ """
+ Unique Coscine-internal identifier for the invitation.
+ """
+ return self._data.get("id") or ""
+
+ @property
+ def expires(self) -> date:
+ """
+ Timestamp of when the invitation expires.
+ """
+ value = self._data.get("expirationDate") or ""
+ return dateutil_parse(value).date()
+
+ @property
+ def email(self) -> str:
+ """
+ The email address of the invited user.
+ """
+ return self._data.get("email") or ""
+
+ @property
+ def issuer(self) -> User:
+ """
+ The user in Coscine who sent the invitation.
+ """
+ return User(self._data.get("issuer"))
+
+ @property
+ def project_id(self) -> str:
+ """
+ Project ID of the project the invitation applies to.
+ """
+ return self._data.get("project").get("id") or ""
+
+ @property
+ def role(self) -> ProjectRole:
+ """
+ Role assigned to the invited user.
+ """
+ return ProjectRole(self._data.get("role"))
+
+ def __init__(self, data: dict) -> None:
+ self._data = data
+
+ def __str__(self) -> str:
+ return f"{self.email} as {self.role.name}"
+
+###############################################################################
+
+class Project:
+ """
+ Projects in Coscine contains resources.
+ """
+
+ client: ApiClient
+
+ _data: dict
+
+ @property
+ def id(self) -> str:
+ """
+ Unique Coscine-internal project identifier.
+ """
+ return self._data.get("id")
+
+ @property
+ def name(self) -> str:
+ """
+ The full project name as set in the project settings.
+ """
+ return self._data.get("name") or ""
+
+ @name.setter
+ def name(self, value: str) -> None:
+ self._data["name"] = value
+
+ @property
+ def display_name(self) -> str:
+ """
+ The shortened project name as displayed in
+ the Coscine web interface.
+ """
+ return self._data.get("displayName") or ""
+
+ @display_name.setter
+ def display_name(self, value: str) -> None:
+ self._data["displayName"] = value
+
+ @property
+ def description(self) -> str:
+ """
+ The project description.
+ """
+ return self._data.get("description") or ""
+
+ @description.setter
+ def description(self, value: str) -> None:
+ self._data["description"] = value
+
+ @property
+ def principal_investigators(self) -> str:
+ """
+ The project investigators.
+ """
+ return self._data.get("principleInvestigators") or ""
+
+ @principal_investigators.setter
+ def principal_investigators(self, value: str) -> None:
+ self._data["principleInvestigators"] = value
+
+ @property
+ def start_date(self) -> date:
+ """
+ Start of project lifecycle timestamp.
+ """
+ return dateutil_parse(self._data.get("startDate")).date()
+
+ @start_date.setter
+ def start_date(self, value: date) -> None:
+ self._data["startDate"] = value
+
+ @property
+ def end_date(self) -> date:
+ """
+ End of project lifecycle timestamp.
+ """
+ return dateutil_parse(self._data.get("endDate")).date()
+
+ @end_date.setter
+ def end_date(self, value: date) -> None:
+ self._data["endDate"] = value
+
+ @property
+ def keywords(self) -> list[str]:
+ """
+ Project keywords for better discoverability.
+ """
+ return self._data.get("keywords") or []
+
+ @keywords.setter
+ def keywords(self, value: list[str]) -> None:
+ self._data["keywords"] = value
+
+ @property
+ def grant_id(self) -> str:
+ """
+ Project grant id.
+ """
+ return self._data.get("grantId") or ""
+
+ @grant_id.setter
+ def grant_id(self, value: str) -> None:
+ self._data["grantId"] = value
+
+ @property
+ def slug(self) -> str:
+ """
+ Project slug - usually a combination out of original
+ project name and some arbitrary Coscine-internal
+ data appended to it.
+ """
+ return self._data.get("slug") or ""
+
+ @property
+ def pid(self) -> str:
+ """
+ Project Persistent Identifier.
+ """
+ return self._data.get("pid") or ""
+
+ @property
+ def creator(self) -> str:
+ """
+ Project creator user ID.
+ """
+ creator = self._data.get("creator")
+ if creator:
+ return creator.get("id") or ""
+ return ""
+
+ @property
+ def created(self) -> date:
+ """
+ Timestamp of when the project was created.
+ If 1998-01-01 is returned, then the created() value is erroneous
+ or missing.
+ """
+ value = self._data.get("creationDate") or "1998-01-01"
+ return dateutil_parse(value).date()
+
+ @property
+ def organizations(self) -> list[str]:
+ """
+ Organizations participating in the project.
+ """
+ values = self._data.get("organizations", [])
+ return [data["uri"] for data in values]
+
+ @property
+ def disciplines(self) -> list[Discipline]:
+ """
+ Scientific disciplines the project is involved with.
+ """
+ values = self._data.get("disciplines") or []
+ return [Discipline(data) for data in values]
+
+ @disciplines.setter
+ def disciplines(self, value: list[Discipline]) -> None:
+ self._data["disciplines"] = [
+ discipline._data for discipline in value
+ ]
+
+ @property
+ def visibility(self) -> Visibility:
+ """
+ Project visibility setting.
+ """
+ return Visibility(self._data.get("visibility"))
+
+ @visibility.setter
+ def visibility(self, value: Visibility) -> None:
+ self._data["visibility"] = value._data
+
+ @property
+ def url(self) -> str:
+ """
+ Project URL - makes the project accessible in the web browser.
+ """
+ return f"{self.client.base_url}/p/{self.slug}"
+
+ def __init__(self, client: ApiClient, data: dict) -> None:
+ self.client = client
+ self._data = data
+
+ def __str__(self) -> str:
+ return tabulate([
+ ("ID", self.id),
+ ("Name", self.name),
+ ("Display Name", self.display_name),
+ ("Description", "\n".join(wrap(self.description))),
+ ("Principle Investigators", self.principal_investigators),
+ ("Disciplines", "\n".join([str(it) for it in self.disciplines])),
+ ("Organizations", "\n".join(self.organizations)),
+ ("Start Date", self.start_date),
+ ("End Date", self.end_date),
+ ("Date created", self.created),
+ ("Creator", self.creator),
+ ("Grant ID", self.grant_id),
+ ("PID", self.pid),
+ ("Slug", self.slug),
+ ("Keywords", ",".join(self.keywords)),
+ ("Visibility", self.visibility)
+ ], disable_numparse=True)
+
+ def match(self, property: property, key: str) -> bool:
+ """
+ Attempts to match the project via the given property
+ and property value.
+
+ Returns
+ -------
+ True
+ If its a match β₯
+ False
+ Otherwise :(
+ """
+ if (
+ (property == Project.id and self.id == key) or
+ (property == Project.pid and self.pid == key) or
+ (property == Project.name and self.name == key) or
+ (property == Project.url and self.url == key) or (
+ (property == Project.display_name) and
+ (self.display_name == key)
+ )
+ ):
+ return True
+ return False
+
+ def delete(self) -> None:
+ """
+ Deletes the project on the Coscine servers.
+ Be careful when using this function in your code, as users
+ should be prevented from accidentially triggering it!
+ Best to prompt the user before calling this function on whether
+ they really wish to delete their project.
+ """
+ uri = self.client.uri("projects", self.id)
+ self.client.delete(uri)
+
+ def resources(self) -> list[Resource]:
+ """
+ Retrieves a list of all resources of the project.
+ """
+ uri = self.client.uri("projects", f"{self.id}", "resources")
+ response = self.client.get_pages(uri)
+ return [Resource(self, data) for page in response for data in page]
+
+ def resource(
+ self,
+ key: str,
+ property: property = Resource.display_name
+ ) -> Resource:
+ """
+ Returns a single resource via one of its properties.
+ The key can be specified to match any of the ResourceProperty items.
+ """
+ resources = self.resources()
+ results = list(filter(
+ lambda resource: resource.match(property, key),
+ resources
+ ))
+ if len(results) > 1:
+ raise coscine.exceptions.IndistinguishableError(
+ f"Found more than 1 resource matching the key '{key}'. "
+ "Certain properties such as the name of a resource "
+ "allow for duplicates among other resources."
+ )
+ elif len(results) == 0:
+ raise coscine.exceptions.NotFoundError(
+ f"Failed to find a resource via the key '{key}'! "
+ )
+ return results[0]
+
+ def download(self, path: str = "./") -> None:
+ """
+ Downloads the project to the local directory path.
+ """
+ path = join(path, self.display_name)
+ if not isdir(path):
+ mkdir(path)
+ for resource in self.resources():
+ resource.download(path)
+
+ def add_member(self, user: User, role: ProjectRole) -> None:
+ """
+ Adds the project member of another project to the current project.
+ The owner of the Coscine API token must be a member
+ of the other project.
+ """
+ uri = self.client.uri("projects", self.id, "members")
+ self.client.post(uri, json={
+ "roleId": role.id,
+ "userId": user.id
+ })
+
+ def remove_member(self, member: ProjectMember) -> None:
+ """
+ Removes the member from the project. Does not invalidate the member
+ object in Python - it is up to the API user to not use that variable
+ again.
+ """
+ uri = self.client.uri("projects", self.id, "members", member.id)
+ self.client.delete(uri)
+
+ def invite(self, email: str, role: ProjectRole) -> None:
+ """
+ Invites an external user via their email address to
+ the Coscine project.
+ """
+ uri = self.client.uri("projects", self.id, "invitations")
+ self.client.post(uri, json={
+ "roleId": role.id,
+ "email": email
+ })
+
+ def quotas(self) -> list[ProjectQuota]:
+ """
+ Returns the project storage quotas.
+ """
+ uri = self.client.uri("projects", self.id, "quotas")
+ response = self.client.get(uri)
+ return [ProjectQuota(data) for data in response]
+
+ def members(self) -> list[ProjectMember]:
+ """
+ Returns the list of all members of the current project.
+ """
+ uri = self.client.uri("projects", self.id, "members")
+ response = self.client.get_pages(uri)
+ return [ProjectMember(data) for page in response for data in page]
+
+ def invitations(self) -> list[ProjectInvitation]:
+ """
+ Returns the list of all outstanding project invitations.
+ """
+ uri = self.client.uri("projects", self.id, "invitations")
+ response = self.client.get_pages(uri)
+ return [ProjectInvitation(data) for page in response for data in page]
+
+ def update(self) -> None:
+ """
+ Updates a Coscine project's settings.
+ To update certain properties just access the properties
+ of the coscine.Project class directly and call Project.update()
+ when done.
+ """
+ data: dict = {
+ "name": self.name,
+ "displayName": self.display_name,
+ "description": self.description,
+ "startDate": self.start_date.isoformat(),
+ "endDate": self.end_date.isoformat(),
+ "principleInvestigators": self.principal_investigators,
+ "disciplines": [{
+ "id": discipline.id,
+ "uri": discipline.uri,
+ "displayNameEn": discipline.name
+ } for discipline in self.disciplines
+ ],
+ "organizations": [{
+ "uri": organization
+ } for organization in self.organizations
+ ],
+ "visibility": {
+ "id": self.visibility.id
+ },
+ "keywords": self.keywords,
+ "grantId": self.grant_id,
+ "slug": self.slug,
+ "pid": self.pid
+ }
+ uri = self.client.uri("projects", self.id)
+ self.client.put(uri, json=data)
+
+ def create_resource(
+ self,
+ name: str,
+ display_name: str,
+ description: str,
+ license: License,
+ visibility: Visibility,
+ disciplines: list[Discipline],
+ resource_type: ResourceType,
+ quota: int,
+ application_profile: str,
+ usage_rights: str = "",
+ keywords: list[str] = []
+ ) -> Resource:
+ """
+ Creates a new Coscine resource within the project.
+
+ Parameters
+ ----------
+ name
+ The full name of the resource.
+ display_name
+ The shortened display name of the resource.
+ description
+ The description of the resource.
+ license
+ License for the resource contents.
+ visibility
+ Resource metadata visibility (relevant for search).
+ disciplines
+ Associated/Involved scientific disciplines.
+ resource_type
+ The Cosciner resource type.
+ quota
+ Resource quota in GB (irrelevant for linked data resources).
+ application_profile
+ The metadata application profile for the resource.
+ notes
+ Data usage notes
+ keywords
+ Keywords (relevant for search).
+
+ """
+ rds_options = {
+ "quota": {
+ "value": quota,
+ "unit": "https://qudt.org/vocab/unit/GibiBYTE"
+ }
+ }
+ options = {
+ "linked": {
+ "linkedResourceTypeOptions": {}
+ }, "gitlab": {
+ "gitlabResourceTypeOptions": {}, # unsupported actually
+ }, "rds": {
+ "rdsResourceTypeOptions": rds_options
+ }, "rdss3": {
+ "rdsS3ResourceTypeOptions": rds_options
+ }, "rdss3worm": {
+ "rdsS3WormResourceTypeOptions": rds_options
+ }
+ }
+ data: dict = {
+ "name": name,
+ "displayName": display_name,
+ "description": description,
+ "keywords": keywords,
+ "license": { "id": license.id },
+ "visibility": { "id": visibility.id },
+ "disciplines": [{
+ "id": discipline.id,
+ "uri": discipline.uri,
+ "displayNameEn": discipline.name
+ } for discipline in disciplines
+ ],
+ "resourceTypeId": resource_type.id,
+ "resourceTypeOptions": options[resource_type.generalType],
+ "applicationProfile": {
+ "uri": application_profile
+ },
+ "usageRights": usage_rights
+ }
+ uri = self.client.uri("projects", self.id, "resources")
+ return Resource(self, self.client.post(uri, json=data))
diff --git a/src/coscine/resource.py b/src/coscine/resource.py
index 76805f41d8b17ff69f9a6af1074124d24d00ef58..81f2b6fef5ad1ccae967aa74985478855cee0131 100644
--- a/src/coscine/resource.py
+++ b/src/coscine/resource.py
@@ -1,752 +1,1012 @@
-###############################################################################
-# Coscine Python SDK
-# Copyright (c) 2018-2023 RWTH Aachen University
-# Licensed under the terms of the MIT License
-###############################################################################
-# Coscine, short for Collaborative Scientific Integration Environment, is
-# a platform for research data management (RDM).
-# For more information on Coscine visit https://www.coscine.de/.
-#
-# Please note that this python module is open source software primarily
-# developed and maintained by the scientific community. It is not
-# an official service that RWTH Aachen provides support for.
-###############################################################################
-
-###############################################################################
-# File description
-###############################################################################
-
-"""
-This file defines the resource object for the representation of
-Coscine resources. It provides an easy interface to interact with Coscine
-resources from python.
-"""
-
-###############################################################################
-# Dependencies
-###############################################################################
-
-from __future__ import annotations
-import json
-import os
-import posixpath
-import logging
-from typing import Callable, TYPE_CHECKING, Union, List, Optional
-import csv
-from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor
-from prettytable.prettytable import PrettyTable
-from urllib.parse import quote
-import rdflib
-try:
- import pandas
-except ImportError:
- pandas = None
-from coscine.object import FileObject, MetadataForm
-from coscine.form import InputForm
-from coscine.graph import ApplicationProfile
-from coscine.utils import ProgressBar, parallelizable, concurrent
-from coscine.s3 import S3
-if TYPE_CHECKING:
- from coscine.client import Client
- from coscine.project import Project
-
-###############################################################################
-# Module globals / Constants
-###############################################################################
-
-logger = logging.getLogger(__name__)
-
-###############################################################################
-# Classes / Functions / Scripts
-###############################################################################
-
-class ResourceForm(InputForm):
- """
- An InputForm for Coscine Resource metadata.
- """
-
-###############################################################################
-
- def __init__(self, client: Client) -> None:
- """
- Parameters
- -----------
- client : Client
- Coscine Python SDK Client instance
- """
- super().__init__(client)
- self._fields = client.vocabularies.builtin("resource")
- vocabularies = {
- "type": client.vocabularies.resource_types(True),
- "applicationProfile":
- client.vocabularies.application_profiles(True),
- "license": client.vocabularies.licenses(True),
- "visibility": client.vocabularies.visibility(True),
- "disciplines": client.vocabularies.disciplines(True)
- }
- for item in self._fields:
- if item.vocabulary:
- self._vocabularies[item.path] = vocabularies[item.path]
-
-###############################################################################
-
- def parse(self, data: dict) -> None:
- ignore = ["id", "pid", "fixedValues", "creator", "archived"]
- for path, value in data.items():
- if path not in ignore:
- try:
- key = self.name_of(path)
- self.set_value(key, value, True)
- except KeyError:
- continue
-
-###############################################################################
-
- def generate(self) -> dict:
- metadata = {}
-
- # Collect missing required fields
- missing: List[str] = []
-
- # Set metadata
- for key, value in self.items():
- if value is None:
- if self.is_required(key):
- missing.append(key)
- continue
-
- properties = self.properties(key)
- metadata[properties.path] = value.raw()
- # The resourceTypeOption requires a dict with additional
- # 'Size' field in it.
- if properties.path == "resourceTypeOption":
- metadata[properties.path] = {
- "Size": value.raw()
- }
-
- # Check for missing required fields
- if len(missing) > 0:
- if not (
- len(missing) == 1
- and missing[0] == "resourceTypeOption"
- and metadata["type"] == "linked"
- ):
- raise ValueError(missing)
-
- return metadata
-
-###############################################################################
-###############################################################################
-###############################################################################
-
-class Resource:
- """
- Python representation of a Coscine Resource type.
- """
-
- client: Client
- project: Project
- data: dict
- s3: Optional[S3]
-
-###############################################################################
-
- def __init__(self, project: Project, data: dict) -> None:
- """
- Initializes a Coscine resource object.
-
- Parameters
- ----------
- project : Project
- Coscine project handle
- data : dict
- Resource data received from Coscine.
- """
-
- self.project = project
- self.client = self.project.client
- self.data = data
- self.s3 = S3(data, self.client.settings.verbose)
-
-###############################################################################
-
- @property
- def id(self) -> str:
- """Resource ID"""
- return self.data["id"]
-
- @property
- def pid(self) -> str:
- """Resource Persistent Identifier"""
- return self.data["pid"]
-
- @property
- def access_url(self) -> str:
- """Resource Access URL via PID"""
- return f"http://hdl.handle.net/{self.pid}"
-
- @property
- def name(self) -> str:
- """Resource Name"""
- return self.data["resourceName"]
-
- @property
- def display_name(self) -> str:
- """Resource Display Name"""
- return self.data["displayName"]
-
- @property
- def description(self) -> str:
- """Resource Description"""
- return self.data["description"]
-
- @property
- def license(self) -> str:
- """Resource License Name"""
- return (
- self.data["license"]["displayName"] if self.data["license"] else ""
- )
-
- @property
- def type(self) -> str:
- """Resource Type e.g. rdss3rwth"""
- return self.data["type"]["displayName"]
-
- @property
- def disciplines(self) -> List[str]:
- """Associated Disciplines"""
- lang = {
- "en": "displayNameEn",
- "de": "displayNameDe"
- }[self.client.settings.language]
- return [k[lang] for k in self.data["disciplines"]]
-
- @property
- def profile(self) -> str:
- """Name of the Application Profile used in the Resource"""
- return self.data["applicationProfile"]
-
- @property
- def archived(self) -> bool:
- """Status indicator, indicating whether the resource is archived"""
- return bool(self.data["archived"])
-
- @property
- def creator(self) -> str:
- """Resource Creator"""
- return self.data["creator"]
-
-###############################################################################
-
- def __str__(self) -> str:
- table = PrettyTable(["Property", "Value"])
- rows = [
- ("ID", self.id),
- ("Resource Name", self.name),
- ("Display Name", self.display_name),
- ("Description", self.description),
- ("PID", self.pid),
- ("Type", self.type),
- ("Disciplines", "\n".join(self.disciplines)),
- ("License", self.license),
- ("Application Profile", self.profile),
- ("Archived", self.archived),
- ("Creator", self.creator),
- ("Project", self.project.display_name),
- ("Project ID", self.project.id)
- ]
- table.max_width["Value"] = 50
- table.add_rows(rows)
- return table.get_string(title=f"Resource {self.display_name}")
-
-###############################################################################
-
- @parallelizable
- def delete(self) -> None:
- """
- Deletes the Coscine resource and all objects contained within it on
- the Coscine servers.
- """
-
- logger.info("Deleting resource '%s' (%s)...", self.name, self.id)
- uri = self.client.uri("Resources", "Resource", self.id)
- self.client.delete(uri)
-
-###############################################################################
-
- def application_profile(self) -> ApplicationProfile:
- """
- Returns the application profile of the resource.
- """
-
- return self.client.vocabularies.application_profile(self.profile)
-
-###############################################################################
-
- def download_metadata(self, path: str = "./", format: str = "csv") -> None:
- """
- Downloads the metadata of the file objects stored within the resource
- in the specified format.
-
- Parameters
- ----------
- format : str, default: "csv"
- The file format to store the data in. Available formats are:
- "json", "csv"
-
- Raises
- -------
- ValueError
- In case of an unexpected format.
- """
-
- if format == "csv":
- with open(
- os.path.join(path, ".file-metadata.csv"),
- "w", encoding="utf-8", newline=""
- ) as file_handle:
- csvwriter = csv.writer(file_handle)
- csvwriter.writerow(["File"] + self.metadata_form().keys())
- for obj in self.contents():
- values = [obj.path] + [
- str(value) for value in obj.form().values()
- ]
- csvwriter.writerow(values)
- elif format == "json":
- metadata = []
- keys = ["File"] + self.metadata_form().keys()
- for obj in self.contents():
- values = [obj.path] + [
- str(value) for value in obj.form().values()
- ]
- entry = {
- keys[i]: values[i] for i in range(len(keys))
- }
- metadata.append(entry)
- with open(
- os.path.join(path, ".file-metadata.json"),
- "w", encoding="utf-8"
- ) as file_handle:
- json.dump(metadata, file_handle)
- else:
- raise ValueError("Unexpected metadata format!")
-
-###############################################################################
-
- def download(self, path: str = "./", metadata: bool = False) -> None:
- """
- Downloads the resource and all of its contents to the local harddrive.
-
- Parameters
- ----------
- path : str, default: "./"
- Path to the local storage location.
- metadata : bool, default: False
- If enabled, resource metadata is downloaded and put in
- the file '.resource-metadata.json'.
- """
-
- logger.info("Downloading resource '%s' (%s)...", self.name, self.id)
- path = os.path.join(path, self.name)
- if not os.path.isdir(path):
- os.mkdir(path)
- if self.client.settings.concurrent:
- with concurrent():
- for obj in self.contents():
- obj.download(path, preserve_path=True)
- else:
- for obj in self.contents():
- obj.download(path, preserve_path=True)
- if metadata:
- self.download_metadata(path)
- data = json.dumps(self.data, indent=4)
- metadata_path: str = os.path.join(path, ".resource-metadata.json")
- with open(metadata_path, "w", encoding="utf-8") as file:
- file.write(data)
-
-###############################################################################
-
- def exists(self, path: str) -> bool:
- """
- Returns whether the file referenced by key is contained
- in the resource.
-
- Parameters
- -----------
- path : str
- The key/path to the file object
-
- Returns
- -------
- bool
- True if file object exists, False if not
- """
- return self.object(path) is not None
-
-###############################################################################
-
- def contents(self) -> List[FileObject]:
- """
- Returns a list of ALL Objects stored within the resource
- by recursively traversing directories
-
- Returns
- -------
- List[FileObject]
- List of Coscine file-like objects.
- """
-
- more_folders: bool = True
- contents = []
- directories = []
- files = self.objects()
- while more_folders:
- more_folders = False
- for obj in files:
- if obj.is_folder:
- files.remove(obj)
- directories.append(obj)
- files.extend(obj.objects())
- more_folders = True
- contents.extend(directories)
- contents.extend(files)
- return contents
-
-###############################################################################
-
- def objects(self, path: Optional[str] = None) -> List[FileObject]:
- """
- Returns a list of Objects stored within the resource.
-
- Parameters
- ------------
- path : str, default: None
- Path to a directory. Irrelevant for anything other than S3.
-
- Examples
- ---------
- >>> Resource.objects(path="Folder/Subfolder")
- Returns all objects inside 'Folder/Subfolder' (s3 resources only)
-
- Returns
- -------
- List[FileObject]
- List of Coscine file-like objects.
- """
-
- objects = []
- uri = self.client.uri("Tree", "Tree", self.id)
- dirpath = posixpath.dirname(path) if path else None
- args = {"path": path} if dirpath and dirpath != "/" else None
- data = self.client.get(uri, params=args).json()
- file_storage: List[dict] = data["data"]["fileStorage"]
- metadata_storage: List[dict] = data["data"]["metadataStorage"]
- for data in file_storage:
- metadata: dict = {}
- for meta in metadata_storage:
- key: str = list(meta.keys())[0]
- if f"{self.id}/{quote(data['Path'])}" in key:
- metadata = meta
- break
- objects.append(FileObject(self, data, metadata))
- return objects
-
-###############################################################################
-
- def object(self, path: str) -> Union[FileObject, None]:
- """
- Returns an Object stored within the resource
-
- Parameters
- ------------
- path : str, default: None
- Path to a directory. Irrelevant for anything other than S3.
-
- Examples
- ---------
- >>> Resource.object(path="Folder/Subfolder/Filename.jpg")
- Returns the file inside 'Folder/Subfolder' (s3 resources only)
-
- Returns
- -------
- FileObject
- Python representation of the file-object as an Object instance
- None
- In case nothing was found.
-
- Raises
- ------
- IndexError
- In case more than 1 FileObject was found at the path.
- """
-
- dirpath = posixpath.join(posixpath.dirname(path), "")
- dirpath = dirpath if dirpath != "/" else ""
- filename = posixpath.basename(path)
- if not filename:
- filename = posixpath.dirname(dirpath)
- dirpath = ""
- filtered_list = list(filter(lambda fo: fo.name == filename,
- self.objects(path=dirpath)))
- if len(filtered_list) == 1:
- return filtered_list[0]
- if len(filtered_list) == 0:
- return None
- raise IndexError("Too many files matching the specified criteria!")
-
-###############################################################################
-
- def upload(
- self,
- key: str,
- file,
- metadata: Union[MetadataForm, dict],
- callback: Optional[Callable[[int], None]] = None
- ) -> None:
- """
- Uploads a file-like object to a resource on the Coscine server
-
- Parameters
- ----------
- key : str
- filename of the file-like object.
- file : object with read() attribute
- Either open file handle or local file location path.
- metadata : MetadataForm or dict
- File metadata matching the resource application profile.
- callback : Callable[[int], None], default: None
- Optional callback called during chunk uploads
- indicating the progress.
-
- Raises
- ------
- TypeError
- In case the file object specified cannot be used.
- """
-
- logger.info(
- "Uploading FileObject '%s' to resource '%s' (%s)...",
- key, self.name, self.id
- )
- if hasattr(file, "read"):
- self._upload_file_metadata(key, metadata)
- self._upload_file_data(key, file, callback)
- elif isinstance(file, str):
- with open(file, "rb") as file_handle:
- self._upload_file_metadata(key, metadata)
- self._upload_file_data(key, file_handle, callback)
- else:
- raise TypeError("Argument `file` has unexpected type!")
-
-###############################################################################
-
- def _upload_file_metadata(self, key: str, metadata) -> None:
- if isinstance(metadata, MetadataForm):
- metadata = metadata.generate()
- uri = self.client.uri("Tree", "Tree", self.id)
- params = {"path": key}
- self.client.put(uri, json=metadata, params=params)
-
-###############################################################################
-
- def _upload_file_data(
- self,
- key: str,
- file_handle,
- callback: Optional[Callable[[int], None]] = None
- ) -> None:
- uri = self.client.uri("Blob", "Blob", self.id)
- fields = {"files": (key, file_handle, "application/octect-stream")}
- encoder = MultipartEncoder(fields=fields)
- progress_bar = ProgressBar(
- self.client.settings.verbose, encoder.len, key, callback
- )
- monitor = MultipartEncoderMonitor(
- encoder,
- callback=lambda monitor:
- progress_bar.update(monitor.bytes_read - progress_bar.count)
- )
- headers = {"Content-Type": monitor.content_type}
- params = {"path": key}
- self.client.put(uri, data=monitor, headers=headers, params=params)
-
-###############################################################################
-
- @parallelizable
- def set_archived(self, flag: bool) -> None:
- """
- Set the archived flag of the resource to put it in read-only mode.
- Only the resource creator or project owner can do this.
-
- Parameters
- ----------
- flag : bool
- Enable with True, Disable with False.
- """
-
- uri = self.client.uri(
- "Resources", "Resource", self.id,
- f"setReadonly?status={str(flag).lower()}"
- )
- logger.info(
- "Setting resource '%s' read only flag = %d",
- self.name, flag
- )
- self.client.post(uri)
- self.data["archived"] = flag
-
-###############################################################################
-
- def form(self) -> ResourceForm:
- """
- Returns a ResourceForm filled with the metadata of
- the current resource.
-
- Returns
- -------
- ResourceForm
- """
-
- form = self.client.resource_form()
- form.parse(self.data)
- return form
-
-###############################################################################
-
- def update(self, form: Union[ResourceForm, dict]) -> Resource:
- """
- Updates the metadata of the resource using the supplied ResourceForm.
-
- Parameters
- ----------
- form : ResourceForm or dict
- ResourceForm filled with updated values
- or generated form data dict.
-
- Returns
- -------
- Resource
- A new resource object with updated resource metadata
- """
-
- if isinstance(form, ResourceForm):
- form = form.generate()
- elif not isinstance(form, dict):
- raise TypeError("Resource metadata form has unexpected type.")
-
- logger.info("Updating resource metadata of resource '%s'.", self.name)
- uri = self.client.uri("Resources", "Resource", self.id)
- response = self.client.post(uri, json=form)
- updated_resource_handle = self
- if response.ok:
- # Update resource properties:
- # The response unfortunately does not return the new resource data
- updated_resource_handle = list(
- filter(lambda r: r.id == self.id, self.project.resources())
- )[0]
- return updated_resource_handle
-
-###############################################################################
-
- def metadata_form(self, data: Optional[dict] = None) -> MetadataForm:
- """
- Creates a MetadataForm for this resource
-
- Parameters
- ----------
- data : dict, default: None
- If data is specified, the form is initialized with that data
- using the InputForm.fill() method.
- """
-
- form = MetadataForm(self.client, self.application_profile())
- if data:
- form.fill(data)
- return form
-
-###############################################################################
-# TODO: Add options to create file index, metadata index, file+metadata index
-# TODO: Speed up by not creating individual forms for each file object
- def dataframe(self):
- """
- Creates a pandas dataframe from resource metadata.
- Requires an installation of the pandas package.
-
- Returns
- -------
- pandas.DataFrame
- """
-
- if not pandas:
- raise ModuleNotFoundError("Method requires package 'pandas'!")
- form = self.metadata_form()
- headers = ["Path"] + form.keys()
- values = [
- [obj.path] + [
- str(val) for val in obj.form().values()
- ] for obj in self.contents()
- ]
- return pandas.DataFrame(values, columns=headers)
-
-###############################################################################
-
- def graph(self) -> Optional[rdflib.Graph]:
- """
- Create an rdflib graph from the resource contents.
- The rdflib graph can be queried with SPARQL, merged with the graphs
- from other resources and serialized into various formats such
- as xml or dot (for rendering it as a graph).
-
- Parameters
- ----------
- literal : bool, default: False
- If enabled, the literal values as seen in the MetadataForm
- are used for the triples in the graph (the human
- readable values instead of the URIs).
-
- Returns
- --------
- rdflib.Graph
- """
-
- #######################################################################
-
- def add_to_graph(graph: rdflib.Graph, file: FileObject):
- """Adds the FileObject to the rdflib graph"""
- metadata = file.metadata()
- if metadata:
- file_reference = rdflib.URIRef(
- file.path,
- "https://purl.org/coscine/file/"
- )
- for key, values in metadata.items():
- for value in values:
- valtype: str = value["type"]
- valuestr: str = value["value"]
- if valtype == "uri":
- rdf_object = rdflib.URIRef(valuestr)
- else:
- rdf_object = rdflib.Literal(valuestr)
- graph.add((
- file_reference,
- rdflib.URIRef(key),
- rdf_object
- ))
-
- #######################################################################
-
- file_objects = self.objects()
- logger.info("Constructing graph for resource %s...", self.display_name)
- pbar = ProgressBar(
- self.client.settings.verbose,
- len(file_objects),
- f"Creating rdf graph for '{self.display_name}'"
- )
- graph = rdflib.Graph(bind_namespaces="rdflib")
- for file in file_objects:
- pbar.update(1)
- if file.has_metadata:
- add_to_graph(graph, file)
- return graph
-
-###############################################################################
+###############################################################################
+# Coscine Python SDK
+# Copyright (c) 2020-2023 RWTH Aachen University
+# Licensed under the terms of the MIT License
+# For more information on Coscine visit https://www.coscine.de/.
+###############################################################################
+
+"""
+"""
+
+from __future__ import annotations
+from typing import TYPE_CHECKING, Callable
+if TYPE_CHECKING:
+ from coscine.client import ApiClient
+ from coscine.project import Project
+from datetime import date
+from dateutil.parser import parse as dateutil_parse
+from os import mkdir
+from os.path import isdir, join
+from io import BytesIO
+from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor
+from tabulate import tabulate
+from textwrap import wrap
+from tqdm import tqdm
+from coscine.common import Discipline, License, Visibility
+from coscine.metadata import ApplicationProfile, MetadataForm, FileMetadata
+import coscine.exceptions
+import boto3
+import rdflib
+
+###############################################################################
+
+class ResourceQuota:
+ """
+ Models the Coscine resource quota data.
+ """
+
+ _data: dict
+
+ @property
+ def resource_id(self) -> str:
+ """
+ The associated Coscine resource id.
+ """
+ return self._data.get("resource")["id"] or ""
+
+ @property
+ def used_percentage(self) -> float:
+ """
+ The ratio of used up quota in relation to the available quota.
+ """
+ value = self._data.get("usedPercentage") or 0.00
+ return float(value)
+
+ @property
+ def used(self) -> int:
+ """
+ The used quota in bytes.
+ """
+ return int(self._data.get("used")["value"])
+
+ @property
+ def reserved(self) -> int:
+ """
+ The reserved quota for the resource.
+ """
+ return int(self._data.get("reserved")["value"] * 1024**3)
+
+ def __init__(self, data: dict) -> None:
+ self._data = data
+
+###############################################################################
+
+def progress_callback(
+ bar: tqdm,
+ bytes_read: int,
+ fn: Callable[[int], None] = None
+) -> None:
+ """
+ Updates the progress bar and calls a callback if one has been specified.
+ """
+ bar.update(bytes_read - bar.n)
+ if fn:
+ fn(bytes_read)
+
+###############################################################################
+
+class ResourceType:
+ """
+ Models the resource types available in Coscine.
+ """
+
+ _data: dict
+
+ @property
+ def id(self) -> str:
+ """
+ Coscine-internal resource type identifier.
+ """
+ return self._data.get("id")
+
+ @property
+ def generalType(self) -> str:
+ """
+ General resource type, e.g. rdss3
+ """
+ return self._data.get("generalType")
+
+ @property
+ def specificType(self) -> str:
+ """
+ Specific resource type, e.g. rdss3rwth
+ """
+ return self._data.get("specificType")
+
+ @property
+ def active(self) -> str:
+ """
+ Whether the resource type is enabled on the Coscine instance.
+ """
+ return self._data.get("status")
+
+ def __init__(self, data: dict) -> None:
+ self._data = data
+
+ def __str__(self) -> str:
+ return self.specificType
+
+###############################################################################
+
+class ResourceTypeOptions:
+ """
+ """
+
+ _data: dict
+
+ @property
+ def access_key_read(self) -> str:
+ """
+ """
+ return self._data.get("accessKeyRead") or ""
+
+ @property
+ def secret_key_read(self) -> str:
+ """
+ """
+ return self._data.get("secretKeyRead") or ""
+
+ @property
+ def access_key_write(self) -> str:
+ """
+ """
+ return self._data.get("accessKeyWrite") or ""
+
+ @property
+ def secret_key_write(self) -> str:
+ """
+ """
+ return self._data.get("secretKeyWrite") or ""
+
+ @property
+ def endpoint(self) -> str:
+ """
+ """
+ return self._data.get("endpoint") or ""
+
+ @property
+ def size(self) -> int:
+ """
+ The size setting of the resource type in GibiByte.
+ """
+ return int(self._data.get("size").get("value"))
+
+ @size.setter
+ def size(self, value: int) -> None:
+ self._data["size"] = {
+ "value": value,
+ "unit": "https://qudt.org/vocab/unit/GibiBYTE"
+ }
+
+ def __init__(self, data: dict = {}) -> None:
+ self._data = data
+
+###############################################################################
+
+class Resource:
+ """
+ """
+
+ client: ApiClient
+ project: Project
+ _data: dict
+
+ @property
+ def id(self) -> str:
+ """
+ Unique Coscine-internal resource identifier.
+ """
+ return self._data.get("id")
+
+ @property
+ def name(self) -> str:
+ """
+ Full resource name as displayed in the resource settings.
+ """
+ return self._data.get("name")
+
+ @name.setter
+ def name(self, value: str) -> None:
+ self._data["name"] = value
+
+ @property
+ def display_name(self) -> str:
+ """
+ Shortened resource name as displayed in the Coscine web interface.
+ """
+ return self._data.get("displayName")
+
+ @display_name.setter
+ def display_name(self, value: str) -> None:
+ self._data["displayName"] = value
+
+ @property
+ def description(self) -> str:
+ """
+ The resource description.
+ """
+ return self._data.get("description")
+
+ @description.setter
+ def description(self, value: str) -> None:
+ self._data["description"] = value
+
+ @property
+ def type(self) -> ResourceType:
+ """
+ The resource's resource type.
+ """
+ return ResourceType(self._data.get("type"))
+
+ @property
+ def options(self) -> ResourceTypeOptions | None:
+ """
+ The resource's resource type specific options.
+ """
+ options = self._data.get("type")["options"]
+ if self.type.generalType == "rdss3":
+ options = options.get("rdsS3")
+ elif self.type.generalType == "rds":
+ options = options.get("rds")
+ elif self.type.generalType == "gitlab":
+ options = options.get("gitLab")
+ elif self.type.generalType == "linked":
+ options = options.get("linkedData")
+ return ResourceTypeOptions(options)
+
+ @property
+ def pid(self) -> str:
+ """
+ The persistent identifier assigned to the resource.
+ """
+ return self._data.get("pid") or ""
+
+ @property
+ def url(self) -> str:
+ """
+ Project URL - makes the resource accessible in the web browser.
+ """
+ return f"{self.client.base_url}/p/{self.project.slug}/r/{self.id}/-/"
+
+ @property
+ def keywords(self) -> list[str]:
+ """
+ List of keywords for better discoverability.
+ """
+ return self._data.get("keywords") or []
+
+ @keywords.setter
+ def keywords(self, value: list[str]) -> None:
+ self._data["keywords"] = value
+
+ @property
+ def license(self) -> License | None:
+ """
+ The license used for the resource data.
+ """
+ value = self._data.get("license")
+ if value:
+ return License(value)
+ return None
+
+ @license.setter
+ def license(self, value: License) -> None:
+ self._data["license"] = value._data
+
+ @property
+ def usage_rights(self) -> str:
+ """
+ The usage rights specified for the data inside the resource.
+ """
+ return self._data.get("usageRights") or ""
+
+ @usage_rights.setter
+ def usage_rights(self, value: str) -> None:
+ self._data["usageRights"] = value
+
+ @property
+ def application_profile(self) -> ApplicationProfile:
+ """
+ """
+ uri = self._data.get("applicationProfile")["uri"]
+ return self.client.application_profile(uri)
+
+ @property
+ def disciplines(self) -> list[Discipline]:
+ """
+ The scientific disciplines set for the resource.
+ """
+ values = self._data.get("disciplines", [])
+ return [Discipline(data) for data in values]
+
+ @disciplines.setter
+ def disciplines(self, value: list[Discipline]) -> None:
+ self._data["disciplines"] = [
+ discipline._data for discipline in value
+ ]
+
+ @property
+ def visibility(self) -> Visibility:
+ """
+ The Coscine visibility setting for the resource.
+ """
+ return Visibility(self._data.get("visibility"))
+
+ @visibility.setter
+ def visibility(self, value: Visibility) -> None:
+ self._data["visibility"] = value._data
+
+ @property
+ def created(self) -> date:
+ """
+ Timestamp of when the resource was created.
+ """
+ value = self._data.get("dateCreated") or "1998-01-01"
+ return dateutil_parse(value).date()
+
+ @property
+ def creator(self) -> str:
+ """
+ The Coscine user id of the resource creator.
+ """
+ return self._data.get("creator") or ""
+
+ @property
+ def archived(self) -> bool:
+ """
+ Evaluates to True when the resource is set to archived.
+ """
+ return bool(self._data.get("archived"))
+
+ @archived.setter
+ def archived(self, value: bool) -> None:
+ self._data["archived"] = value
+
+ @property
+ def quota(self) -> ResourceQuota:
+ """
+ The resources storage quota.
+ """
+ uri = self.client.uri(
+ "projects", self.project.id,
+ "resources", self.id,
+ "quota"
+ )
+ response = self.client.get(uri)
+ return ResourceQuota(response)
+
+ @property
+ def fixed_values(self) -> dict:
+ """
+ The resources default metadata values.
+ """
+ return self._data.get("fixedValues") or {}
+
+ def __init__(self, project: Project, data: dict) -> None:
+ self.client = project.client
+ self.project = project
+ self._data = data
+
+ def __str__(self) -> str:
+ return tabulate([
+ ("ID", self.id),
+ ("Name", self.name),
+ ("Display Name", self.display_name),
+ ("Description", "\n".join(wrap(self.description))),
+ ("Disciplines", "\n".join([str(i) for i in self.disciplines])),
+ ("Date created", self.created),
+ ("Creator", self.creator),
+ ("PID", self.pid),
+ ("Keywords", self.keywords),
+ ("Visibility", self.visibility),
+ ("Application Profile", self.application_profile.name),
+ ("Usage rights", self.usage_rights),
+ ("License", self.license),
+ ("Archived", self.archived)
+ ], disable_numparse=True)
+
+ def match(self, property: property, key: str) -> bool:
+ """
+ Attempts to match the resource via the given property
+ and property value.
+
+ Returns
+ -------
+ True
+ If its a match β₯
+ False
+ Otherwise :(
+ """
+ if (
+ (property == Resource.id and self.id == key) or
+ (property == Resource.pid and self.pid == key) or
+ (property == Resource.name and self.name == key) or
+ (property == Resource.url and self.url == key) or (
+ (property == Resource.display_name) and
+ (self.display_name == key)
+ )
+ ):
+ return True
+ return False
+
+ def metadata_form(self) -> MetadataForm:
+ """
+ Returns the resource metadata form.
+ """
+ return MetadataForm(self)
+
+ def update(self) -> None:
+ """
+ Change the values locally via setter properties
+ """
+ data: dict = {
+ "name": self.name,
+ "displayName": self.display_name,
+ "description": self.description,
+ "keywords": self.keywords,
+ "license": { "id": self.license.id },
+ "visibility": { "id": self.visibility.id },
+ "disciplines": [{
+ "id": discipline.id,
+ "uri": discipline.uri,
+ "displayNameEn": discipline.name
+ } for discipline in self.disciplines
+ ],
+ "resourceTypeId": self.type.id,
+ "applicationProfile": {
+ "uri": self.application_profile.uri
+ },
+ "usageRights": self.usage_rights,
+ "fixedValues": {
+ # TODO - But I won't do it!
+ },
+ "archived": self.archived
+ }
+ uri = self.client.uri("projects", self.id, "resources")
+ self.client.post(uri, json=data)
+
+ def delete(self) -> None:
+ """
+ Deletes the Coscine resource and along with it all files
+ and metadata contained within it on the Coscine servers.
+ Special care should be taken when using that method in code
+ as to not accidentially trigger a delete on a whole resource.
+ Therefore this method is best combined with additional input
+ from the user e.g. by prompting them with the message
+ "Do you really want to delete the resource? (Y/N)".
+ """
+ uri = self.client.uri(
+ "projects", self.project.id,
+ "resources", self.id
+ )
+ self.client.delete(uri)
+
+ def download(self, path: str = "./") -> None:
+ """
+ Downloads the resource to the local directory given by path.
+ """
+ path = join(path, self.display_name)
+ if not isdir(path):
+ mkdir(path)
+ for f in self.files():
+ f.download(path)
+
+ def upload(
+ self,
+ path: str,
+ handle: BytesIO | bytes | str,
+ metadata: MetadataForm | dict,
+ progress: Callable[[int], None] = None
+ ) -> None:
+ """
+ Uploads a file-like object to a resource in Coscine.
+
+ Parameters
+ ----------
+ path : str
+ The path the file shall assume inside of the Coscine resource.
+ Not the path on your local harddrive!
+ The terms path, key and filename can be used interchangeably.
+ handle : BytesIO | bytes | str
+ A binary file handle that supports reading or
+ a set of bytes or a string that can be utf-8 encoded.
+ metadata : MetadataForm or dict
+ Metadata for the file that matches the resource
+ application profile.
+ progress : Callable "def function(int)"
+ Optional callback function that gets occasionally called
+ during the upload with the progress in bytes.
+ """
+ if isinstance(metadata, MetadataForm):
+ metadata = metadata.generate(path)
+ else:
+ raise NotImplementedError("Fill MetadataForm")
+ if isinstance(handle, str):
+ handle = handle.encode("utf-8")
+ if isinstance(handle, bytes):
+ handle = BytesIO(handle)
+ self.update_metadata(metadata)
+ if self.type.generalType == "rdss3":
+ self._upload_blob_s3(path, handle)
+ else:
+ self._upload_blob(path, handle, progress)
+
+ def set_metadata(self, metadata: dict) -> None:
+ """
+ """
+ uri = self.client.uri(
+ "projects", self.resource.project.id,
+ "resources", self.resource.id,
+ "trees", "metadata"
+ )
+ self.client.post(uri, json=metadata)
+
+ def update_metadata(self, metadata: dict) -> None:
+ """
+ """
+ uri = self.client.uri(
+ "projects", self.project.id,
+ "resources", self.id,
+ "trees", "metadata"
+ )
+ try:
+ self.client.post(uri, json=metadata)
+ except coscine.exceptions.RequestRejected:
+ self.client.put(uri, json=metadata)
+
+ def _upload_blob(
+ self,
+ path: str,
+ handle: BytesIO,
+ progress: Callable[[int], None] = None
+ ):
+ """
+ Uploads a file-like object to a resource in Coscine.
+
+ Parameters
+ ----------
+ path : str
+ The path the file shall assume inside of the Coscine resource.
+ Not the path on your local harddrive!
+ The terms path, key and filename can be used interchangeably.
+ handle : BytesIO
+ A binary file handle that supports reading.
+ metadata : MetadataForm or dict
+ Metadata for the file that matches the resource
+ application profile.
+ progress : Callable "def function(int)"
+ Optional callback function that gets occasionally called
+ during the upload with the progress in bytes.
+ """
+ uri = self.client.uri(
+ "projects", self.project.id,
+ "resources", self.id,
+ "blobs", path
+ )
+ files = {
+ "file": (path, handle, "application/octect-stream")
+ }
+ encoder = MultipartEncoder(fields=files)
+ progress_bar = tqdm(
+ desc=path, total=encoder.len,
+ unit="B", unit_scale=True, ascii=True,
+ disable=not self.client.verbose
+ )
+ monitor = MultipartEncoderMonitor(
+ encoder,
+ lambda mon:
+ progress_callback(progress_bar, mon.bytes_read, progress)
+ )
+ headers = {"Content-Type": monitor.content_type}
+ self.client.post(uri, data=monitor, headers=headers)
+
+ def _upload_blob_s3(self, path: str, handle: BytesIO) -> None:
+ """
+ Works only on rdss3 resources and should not be called
+ on other resource types! Bypasses Coscine and uploads
+ directly to the underlying s3 storage.
+ """
+ progress_bar = tqdm(
+ desc=path, unit="B", unit_scale=True, ascii=True,
+ disable=not self.client.verbose
+ )
+ s3 = boto3.client(
+ "s3",
+ aws_access_key_id=self.options.access_key_write,
+ aws_secret_access_key=self.options.secret_key_write,
+ endpoint_url=self.options.endpoint
+ )
+ s3.upload_file(
+ handle,
+ self.options.bucket_name,
+ path,
+ Callback=progress_bar.update
+ )
+
+ def _fetch_files_recursively(self, path: str = "") -> list[FileObject]:
+ more_folders: bool = True
+ contents: list[FileObject] = []
+ directories: list[FileObject] = []
+ files = self._fetch_files(path)
+ while more_folders:
+ more_folders = False
+ for obj in files:
+ if obj.is_folder:
+ files.remove(obj)
+ directories.append(obj)
+ if obj.path != "/":
+ files.extend(self._fetch_files(obj.path))
+ more_folders = True
+ contents.extend(directories)
+ contents.extend(files)
+ return contents
+
+ def _fetch_files(self, path: str = "") -> list[FileObject]:
+ uri = self.client.uri(
+ "projects", self.project.id,
+ "resources", self.id,
+ "trees", "files"
+ )
+ params = {"Path": path} if path else {}
+ response = self.client.get_pages(uri, params=params)
+ return [
+ FileObject(self, data)
+ for page in response for data in page
+ ]
+
+ def files(
+ self,
+ path: str = "",
+ recursive: bool = False,
+ with_metadata: bool = False
+ ) -> list[FileObject]:
+ """
+ Retrieves the list of files that are contained in the resource.
+ Via an additional single API call the metadata for those files
+ can be fetched and made available in the returned files.
+
+ Parameters
+ ----------
+ path
+ You can limit the set of returned files to a path.
+ The path may be the path to a single file in which
+ case a list containing that single file will be returned.
+ Or it may point to a "directory" in which case all
+ the files contained in that "directory" are returned.
+ recursive
+ S3 resources may have folders inside them. Set the recursive
+ parameter to True to also fetch all files contained
+ in these folders.
+ with_metadata
+ If set to True the set of files are returned alongside
+ with their metadata. This internally requires another
+ API request which is considerably slower (1 to 2 seconds).
+ However if you plan on manipulating each files metadata
+ this is the way to go. Otherwise you would have to make
+ an API call to fetch the metadata for each file which
+ in case of large resources will prove to be very painful... :)
+ """
+ if recursive:
+ files = self._fetch_files_recursively(path)
+ else:
+ files = self._fetch_files(path)
+ if with_metadata:
+ metadata = self.metadata(path)
+ for item in metadata:
+ for file in files:
+ if file.path == item.path:
+ file._metadata = item
+ break
+ return files
+
+ def file(self, path: str) -> FileObject:
+ """
+ Returns a single file of the resource via its unique path.
+ """
+ uri = self.client.uri(
+ "projects", self.project.id,
+ "resources", self.id,
+ "trees", "files"
+ )
+ files = [
+ FileObject(self, data)
+ for data in self.client.get(uri, params={"Path": path})
+ ]
+ if len(files) > 1:
+ raise coscine.exceptions.IndistinguishableError
+ elif len(files) == 0:
+ raise coscine.exceptions.NotFoundError
+ return files[0]
+
+ def metadata(self, path: str = "") -> list[FileMetadata]:
+ """
+ Returns the full set of metadata for each file in the resource.
+ """
+ uri = self.client.uri(
+ "projects", self.project.id,
+ "resources", self.id,
+ "trees", "metadata"
+ )
+ params = { "Path": path } if path else {}
+ response = self.client.get_pages(uri, params=params)
+ return [
+ FileMetadata(data)
+ for page in response for data in page
+ ]
+
+ def graph(self) -> rdflib.Graph:
+ """
+ """
+ graph = self.application_profile.graph
+ for metadata in self.metadata():
+ if metadata.is_latest:
+ subgraph = metadata.fixed_graph(self)
+ graph += subgraph
+ return graph
+
+ def query(self, sparql: str) -> list[FileObject]:
+ """
+ Runs a SPARQL query on the underlying resource knowledge graph and
+ returns the file objects whose metadata matches the query.
+ IMPORTANT: The query must (!) include ?path as a variable/column.
+ Otherwise it will get rejected and a ValueError is raised.
+
+ Examples
+ --------
+ >>> resource.query("SELECT ?path ?p ?o { ?path ?p ?o . }")
+
+ >>> project = client.project("Solaris")
+ >>> resource = project.resource("Chest X-Ray CNN")
+ >>> files = resource.query(
+ >>> "SELECT ?path WHERE { "
+ >>> " ?path dcterms:creator ?creator . "
+ >>> " FILTER(?creator != 'Dr. Akula') "
+ >>> "}"
+ >>> )
+ >>> for file in files:
+ >>> print(file.path)
+ """
+ results = self.graph().query(sparql)
+ columns: list[str] = [x.toPython() for x in results.vars]
+ if "?path" not in columns:
+ raise ValueError("?path not present in sparql query string!")
+ files: list[FileObject] = []
+ filepaths: list[str] = [
+ row.path.split("/")[6]
+ for row in results
+ ]
+ for file in self.files(recursive=True, with_metadata=True):
+ if file.path in filepaths:
+ files.append(file)
+ return files
+
+###############################################################################
+
+class FileObject:
+ """
+ Models files or file-like objects in Coscine resources.
+ """
+
+ _data: dict
+ _metadata: FileMetadata | None
+ resource: Resource
+
+ @property
+ def path(self) -> str:
+ """
+ """
+ return self._data.get("path") or ""
+
+ @property
+ def type(self) -> str:
+ """
+ """
+ return self._data.get("type") or ""
+
+ @property
+ def directory(self) -> str:
+ """
+ """
+ return self._data.get("directory") or ""
+
+ @property
+ def name(self) -> str:
+ """
+ """
+ return self._data.get("name") or ""
+
+ @property
+ def extension(self) -> str:
+ """
+ """
+ return self._data.get("extension") or ""
+
+ @property
+ def size(self) -> int:
+ """
+ """
+ value = self._data.get("size") or 0
+ return int(value)
+
+ @property
+ def created(self) -> date:
+ """
+ """
+ value = self._data.get("creationDate") or "1998-01-01"
+ return dateutil_parse(value).date()
+
+ @property
+ def modified(self) -> date:
+ """
+ """
+ value = self._data.get("changeDate") or "1998-01-01"
+ return dateutil_parse(value).date()
+
+ @property
+ def is_folder(self) -> bool:
+ """
+ Evaluates to True when the FileObject represents a folder and
+ not an actual file.
+ """
+ return self.type == "Tree"
+
+ @property
+ def client(self) -> ApiClient:
+ return self.resource.client
+
+ def __init__(
+ self,
+ resource: Resource,
+ data: dict,
+ metadata: dict | None = None
+ ) -> None:
+ self.resource = resource
+ self._data = data
+ self._metadata = metadata
+
+ def __str__(self) -> str:
+ return self.path
+
+ def delete(self) -> None:
+ """
+ Deletes the FileObject remote on the Coscine server.
+ """
+ uri = self.client.uri(
+ "projects", self.resource.project.id,
+ "resources", self.resource.id,
+ "blobs", self.path
+ )
+ self.client.delete(uri)
+
+ def metadata(self) -> FileMetadata | None:
+ """
+ """
+ if not self._metadata:
+ data = self.resource.metadata(path=self.path)
+ if len(data) > 0:
+ self._metadata = data[0]
+ return self._metadata
+
+ def metadata_form(self) -> MetadataForm:
+ """
+ Returns the metadata of the file or an empty metadata form if
+ no metadata has been attached to the file.
+ """
+ form = MetadataForm(self.resource)
+ metadata = self.metadata()
+ if metadata:
+ form.parse(metadata)
+ return form
+
+ def download(self, path: str = "./") -> None:
+ """
+ Downloads the file to the computer.
+ """
+ path = join(path, self.path)
+ with open(path, "wb") as fp:
+ if self.resource.type.generalType == "rdss3":
+ self._stream_s3(fp)
+ else:
+ self.stream(fp)
+
+ def stream(self, fp: BytesIO) -> None:
+ """
+ Streams file contents.
+ """
+ uri = self.client.uri(
+ "projects", self.resource.project.id,
+ "resources", self.resource.id,
+ "blobs", self.path
+ )
+ progress_bar = tqdm(
+ desc=self.path, total=self.size,
+ unit="B", unit_scale=True, ascii=True,
+ disable=not self.client.verbose
+ )
+ response = self.client.request("GET", uri, stream=True)
+ for chunk in response.iter_content(chunk_size=4096):
+ progress_bar.update(len(chunk))
+ fp.write(chunk)
+
+ def _stream_s3(self, handle: BytesIO) -> None:
+ """
+ Works only on rdss3 resources and should not be called
+ on other resource types! Bypasses Coscine and uploads
+ directly to the underlying s3 storage.
+ """
+ progress_bar = tqdm(
+ desc=self.path, total=self.size,
+ unit="B", unit_scale=True, ascii=True,
+ disable=not self.client.verbose
+ )
+ s3 = boto3.client(
+ "s3",
+ aws_access_key_id=self.options.access_key_write,
+ aws_secret_access_key=self.options.secret_key_write,
+ endpoint_url=self.options.endpoint
+ )
+ s3.download_fileobj(
+ self.options.bucket_name,
+ self.path,
+ handle,
+ Callback=progress_bar.update
+ )
+
+ def update(
+ self,
+ handle: BytesIO | bytes | str,
+ progress: Callable[[int], None] = None
+ ) -> None:
+ """
+ Uploads a file-like object to a resource in Coscine.
+
+ Parameters
+ ----------
+ handle : BytesIO | bytes | str
+ A binary file handle that supports reading or
+ bytes or str.
+ progress : Callable "def function(int)"
+ Optional callback function that gets occasionally called
+ during the upload with the progress in bytes.
+ """
+ if isinstance(handle, str):
+ handle = handle.encode("utf-8")
+ if isinstance(handle, bytes):
+ handle = BytesIO(handle)
+ uri = self.client.uri(
+ "projects", self.resource.project.id,
+ "resources", self.resource.id,
+ "blobs", self.path
+ )
+ files = {
+ "file": (self.path, handle, "application/octect-stream")
+ }
+ encoder = MultipartEncoder(fields=files)
+ progress_bar = tqdm(
+ desc=self.path, total=encoder.len,
+ unit="B", unit_scale=True, ascii=True,
+ disable=not self.client.verbose
+ )
+ monitor = MultipartEncoderMonitor(
+ encoder,
+ lambda mon:
+ progress_callback(progress_bar, mon.bytes_read, progress)
+ )
+ headers = {"Content-Type": monitor.content_type}
+ self.client.put(uri, data=monitor, headers=headers)
+
+ def set_metadata(self, metadata: MetadataForm) -> None:
+ """
+ """
+ self._metadata = None
+ if isinstance(metadata, MetadataForm):
+ metadata = metadata.generate(self.path)
+ self.resource.set_metadata(metadata)
+
+ def update_metadata(self, metadata: MetadataForm) -> None:
+ """
+ """
+ self._metadata = None
+ if isinstance(metadata, MetadataForm):
+ metadata = metadata.generate(self.path)
+ self.resource.update_metadata(metadata)
diff --git a/src/coscine/s3.py b/src/coscine/s3.py
deleted file mode 100644
index 6ab8436c6643a6597e992996deb3370468dec82d..0000000000000000000000000000000000000000
--- a/src/coscine/s3.py
+++ /dev/null
@@ -1,207 +0,0 @@
-###############################################################################
-# Coscine Python SDK
-# Copyright (c) 2018-2023 RWTH Aachen University
-# Licensed under the terms of the MIT License
-###############################################################################
-# Coscine, short for Collaborative Scientific Integration Environment, is
-# a platform for research data management (RDM).
-# For more information on Coscine visit https://www.coscine.de/.
-#
-# Please note that this python module is open source software primarily
-# developed and maintained by the scientific community. It is not
-# an official service that RWTH Aachen provides support for.
-###############################################################################
-
-###############################################################################
-# File description
-###############################################################################
-
-"""
-This file defines the S3 object for interacting with Coscine S3
-resources (natively).
-"""
-
-###############################################################################
-# Dependencies
-###############################################################################
-
-from __future__ import annotations
-import logging
-import os
-from typing import Optional, Callable, Any
-from prettytable import PrettyTable
-from coscine.utils import ProgressBar
-try:
- import boto3
-except ImportError:
- boto3 = None
-
-###############################################################################
-# Module globals / Constants
-###############################################################################
-
-logger = logging.getLogger(__name__)
-
-###############################################################################
-# Classes / Functions / Scripts
-###############################################################################
-
-class S3:
- """
- Provides a simple pythonic interface to the underlying S3 storage
- of a resource. The attributes may all be null in case the resource
- is not of type S3.
- """
-
- _read_access_key: Optional[str]
- _read_secret_key: Optional[str]
- _write_access_key: Optional[str]
- _write_secret_key: Optional[str]
- _endpoint: Optional[str]
- _bucket: Optional[str]
- _data: dict
- _s3_client: Optional[Any]
- verbose: bool
-
- @property
- def read_access_key(self) -> Optional[str]:
- """The S3 resource read_access_key for read only access"""
- return self._read_access_key
-
- @property
- def read_secret_key(self) -> Optional[str]:
- """The S3 resource read_secret_key for read only access"""
- return self._read_secret_key
-
- @property
- def write_access_key(self) -> Optional[str]:
- """The S3 resource write_access_key for read/write access"""
- return self._write_access_key
-
- @property
- def write_secret_key(self) -> Optional[str]:
- """The S3 resource write_secret_key for read/write access"""
- return self._write_secret_key
-
- @property
- def bucket(self) -> Optional[str]:
- """The S3 bucket name of the resource"""
- return self._bucket
-
- @property
- def endpoint(self) -> Optional[str]:
- """The S3 endpoint"""
- return self._endpoint
-
-###############################################################################
-
- def __init__(self, data: dict, verbose: bool = False) -> None:
- self._data = data
- self.verbose = verbose
- if "resourceTypeOption" in data and data["resourceTypeOption"]:
- opt: dict = data["resourceTypeOption"]
- self._read_access_key = opt.get("ReadAccessKey")
- self._read_secret_key = opt.get("ReadSecretKey")
- self._write_access_key = opt.get("WriteAccessKey")
- self._write_secret_key = opt.get("WriteSecretKey")
- self._endpoint = opt.get("Endpoint")
- self._bucket = opt.get("BucketName")
- if boto3 and self.write_secret_key:
- self._s3_client = boto3.client(
- "s3",
- aws_access_key_id=self.write_access_key,
- aws_secret_access_key=self.write_secret_key,
- endpoint_url=self.endpoint
- )
- else:
- self._s3_client = None
-
-###############################################################################
-
- def __str__(self) -> str:
- table = PrettyTable(["Property", "Value"])
- rows = [
- ("read_access_key", self.read_access_key),
- ("read_secret_key", self.read_secret_key),
- ("write_access_key", self.write_access_key),
- ("write_secret_key", self.write_secret_key),
- ("endpoint", self.endpoint),
- ("bucket", self.bucket)
- ]
- table.max_width["Value"] = 50
- table.add_rows(rows)
- return table.get_string(title="S3")
-
-###############################################################################
-
- def mkdir(self, path: str) -> None:
- """
- Creates a directory
- """
- if not self._s3_client:
- raise ModuleNotFoundError("S3.mkdir() requires package 'boto3'!")
- logger.info(
- "Creating directory '%s' in S3 resource '%s'...",
- path, self.bucket
- )
- self._s3_client.put_object(Bucket=self.bucket, Key=path)
-
-###############################################################################
-
- def upload(
- self,
- key: str,
- file,
- callback: Optional[Callable[[int], None]] = None
- ) -> None:
- """
- Uploads a file-like object to an S3 resource on the Coscine server
-
- Parameters
- ----------
- key : str
- filename of the file-like object.
- file : object with read() attribute
- Either open file handle or local file location path.
- callback : Callable[[int], None], default: None
- Optional callback called during chunk uploads
- indicating the progress.
-
- Raises
- ------
- TypeError
- In case the file object specified cannot be used.
- """
- if not self._s3_client:
- raise ModuleNotFoundError("S3.upload() requires package 'boto3'!")
- logger.info(
- "Uploading FileObject '%s' to S3 resource '%s'...",
- key, self.bucket
- )
-
- #######################################################################
- def s3upload(
- s3_client,
- key: str,
- file,
- filesize: int,
- callback: Optional[Callable[[int], None]] = None
- ) -> None:
- progress_bar = ProgressBar(self.verbose, filesize, key, callback)
- s3_client.upload_file(
- file, self.bucket, key,
- Callback=progress_bar.update
- )
- #######################################################################
-
- if hasattr(file, "read"): # FIXME: How to get the size?
- s3upload(self._s3_client, key, file, 10**6, callback)
- elif isinstance(file, str):
- s3upload(
- self._s3_client, key, file,
- os.stat(file).st_size, callback
- )
- else:
- raise TypeError("Argument `file` has unexpected type!")
-
-###############################################################################
diff --git a/src/coscine/utils.py b/src/coscine/utils.py
deleted file mode 100644
index 801fa17ce5e0f2b1d2c47f31a9498e9077154a09..0000000000000000000000000000000000000000
--- a/src/coscine/utils.py
+++ /dev/null
@@ -1,336 +0,0 @@
-###############################################################################
-# Coscine Python SDK
-# Copyright (c) 2018-2023 RWTH Aachen University
-# Licensed under the terms of the MIT License
-###############################################################################
-# Coscine, short for Collaborative Scientific Integration Environment, is
-# a platform for research data management (RDM).
-# For more information on Coscine visit https://www.coscine.de/.
-#
-# Please note that this python module is open source software primarily
-# developed and maintained by the scientific community. It is not
-# an official service that RWTH Aachen provides support for.
-###############################################################################
-
-###############################################################################
-# File description
-###############################################################################
-
-"""
-This file contains utility classes and functions, sometimes taken from another
-source like StackOverflow. Credit is given where it is due.
-"""
-
-###############################################################################
-# Dependencies
-###############################################################################
-
-from typing import Any, Union, List, Callable, Optional
-import contextlib
-from concurrent.futures import Future, ThreadPoolExecutor, wait
-import html
-import sys
-from tqdm.auto import tqdm
-import rdflib
-
-###############################################################################
-# Module globals / Constants
-###############################################################################
-
-futures: List[Future] = []
-executor: Union[ThreadPoolExecutor, None] = None
-concurrent_mode: bool = False
-
-###############################################################################
-# Classes / Functions / Scripts
-###############################################################################
-
-@contextlib.contextmanager
-def concurrent():
- """
- A context manager for threadpool-based concurrency. Functions with
- the decorator parallelizable are executed in parallel within the
- scope of this context manager.
-
- Examples
- --------
- >>> with coscine.concurrent():
- >>> client.upload(key, file, metadata)
- """
- global futures
- global executor
- global concurrent_mode
- executor = ThreadPoolExecutor(8)
- concurrent_mode = True
- yield
- wait(futures)
- for future in futures:
- future.result()
- executor.shutdown()
- futures = []
- executor = None
- concurrent_mode = False
-
-###############################################################################
-
-def parallelizable(func):
- """
- Function decorator providing basic concurrency to coscine functions.
- When a function with the parallelizable decorator is called within
- the coscine.concurrent context manager, it is executed in parallel.
- Examples
- --------
- >>> with coscine.concurrent():
- >>> client.upload(key, file, metadata)
- """
-
- def inner(*args, **kwargs) -> Union[Future, Callable]:
- if concurrent_mode and executor is not None:
- future = executor.submit(func, *args, **kwargs)
- futures.append(future)
- return future
- else:
- return func(*args, **kwargs)
- return inner
-
-###############################################################################
-
-def in_cli_mode() -> bool:
- """
- Returns true if we are running in a command line interface.
- """
- return sys.stdout is not None and sys.stderr.isatty()
-
-###############################################################################
-
-def is_subset(x: dict, y: dict) -> bool:
- """
- Check whether dict x is a subset of dict y
- """
- return x.items() <= y.items()
-
-###############################################################################
-# rdf_to_dot function
-###############################################################################
-# Heavily inspired by the rdf2dot function provided with rdflib.tools
-###############################################################################
-
-def rdf_to_dot(graph: rdflib.Graph, stream):
- """
- Convert the RDF graph to DOT
- writes the dot output to the stream
- """
-
- nodes: dict = {}
-
- def node(x: Any):
- if x not in nodes:
- nodes[x] = "node%d" % len(nodes)
- return nodes[x]
-
- def label(x, g):
- LABEL_PROPERTIES = [
- rdflib.RDFS.label,
- rdflib.URIRef("http://purl.org/dc/elements/1.1/title"),
- rdflib.URIRef("http://xmlns.com/foaf/0.1/name"),
- rdflib.URIRef("http://www.w3.org/2006/vcard/ns#fn"),
- rdflib.URIRef("http://www.w3.org/2006/vcard/ns#org"),
- ]
- for labelProp in LABEL_PROPERTIES:
- l_ = g.value(x, labelProp)
- if l_:
- return l_
- try:
- return g.namespace_manager.compute_qname(x)[2]
- except (KeyError, ValueError):
- return x
-
- def qname(x, g):
- try:
- q = g.compute_qname(x)
- return q[0] + ":" + q[2]
- except (KeyError, ValueError):
- return x
-
- def color(p):
- return "BLACK"
-
- stream.write('digraph { \n node [ fontname="DejaVu Sans" ] ; \n')
-
- for s, p, o in graph:
- sn = node(s)
- if p == rdflib.RDFS.label:
- continue
- if isinstance(o, (rdflib.URIRef, rdflib.BNode, rdflib.Literal)):
- on = node(o)
- opstr = (
- "\t%s -> %s [ color=%s, label=< <font point-size='10' "
- + "color='#336633'>%s</font> > ] ;\n"
- )
- stream.write(opstr % (sn, on, color(p), qname(p, graph)))
-
- for u, n in nodes.items():
- opstr = (
- f"{n} [ shape=none, color=black label=< <table color='#666666'"
- " cellborder='0' cellspacing='0' border='1'><tr>"
- "<td colspan='2' bgcolor='#eeeeee'>"
- f"<B>{html.escape(label(u, graph))}</B></td>"
- "</tr></table> > ] \n"
- )
- stream.write(opstr)
-
- stream.write("}\n")
-
-###############################################################################
-# HumanBytes class
-###############################################################################
-# Source: https://stackoverflow.com/questions/12523586/
-# python-format-size-application-converting-b-to-kb-mb-gb-tb
-# by Mitch McMabers (https://stackoverflow.com/users/8874388/mitch-mcmabers)
-# Licensed under the Public Domain
-###############################################################################
-
-class HumanBytes:
- METRIC_LABELS: List[str] = ["B", "kB", "MB", "GB", "TB",
- "PB", "EB", "ZB", "YB"]
- BINARY_LABELS: List[str] = ["B", "KiB", "MiB", "GiB",
- "TiB", "PiB", "EiB", "ZiB", "YiB"]
- # PREDEFINED FOR SPEED:
- PRECISION_OFFSETS: List[float] = [0.5, 0.05, 0.005, 0.0005]
- PRECISION_FORMATS: List[str] = ["{}{:.0f} {}", "{}{:.1f} {}",
- "{}{:.2f} {}", "{}{:.3f} {}"]
-
- @staticmethod
- def format(
- num: Union[int, float],
- metric: bool = False,
- precision: int = 1
- ) -> str:
- """
- Human-readable formatting of bytes, using binary (powers of 1024)
- or metric (powers of 1000) representation.
- """
-
- if not isinstance(num, (int, float)):
- raise TypeError("num must be an int or float")
- if not isinstance(metric, bool):
- raise TypeError("metric must be a bool")
- if not (
- isinstance(precision, int) and precision >= 0 and precision <= 3
- ):
- raise ValueError("precision must be an int (range 0-3)")
-
- unit_labels = (HumanBytes.METRIC_LABELS if metric
- else HumanBytes.BINARY_LABELS)
- last_label = unit_labels[-1]
- unit_step = 1000 if metric else 1024
- unit_step_thresh = unit_step - HumanBytes.PRECISION_OFFSETS[precision]
-
- is_negative = num < 0
- # Faster than ternary assignment
- # or always running abs():
- if is_negative:
- num = abs(num)
-
- for unit in unit_labels:
- # VERY IMPORTANT:
- # Only accepts the CURRENT unit if we're BELOW the threshold where
- # float rounding behavior would place us into the NEXT unit: F.ex.
- # when rounding a float to 1 decimal, any number ">= 1023.95" will
- # be rounded to "1024.0". Obviously we don't want ugly output such
- # as "1024.0 KiB", since the proper term for that is "1.0 MiB".
- if num < unit_step_thresh:
- break
- # We only shrink the number if we HAVEN'T reached the last unit.
- # NOTE: These looped divisions accumulate floating point rounding
- # errors, but each new division pushes the rounding errors further
- # and further down in the decimals, so it doesn't matter at all.
- if unit != last_label:
- num /= unit_step
-
- return HumanBytes.PRECISION_FORMATS[precision].format(
- "-" if is_negative else "", num, unit
- )
-
-###############################################################################
-
-class ProgressBar:
- """
- The ProgressBar class is a simple wrapper around tqdm
- progress bars. It is used in download/upload methods and provides the
- benefit of remembering state information and printing only when in
- verbose mode.
-
- Attributes
- ----------
- enabled : bool
- Enable/Disable Progress Bar (if disabled it doesn't show up)
- bar : tqdm.tqdm
- tqdm Progress Bar instance.
- filesize : int
- Filesize in bytes.
- count : int
- Number of bytes read.
- trip : int
- Force refresh of the progress bar if 0 reached.
- callback : Callable[[int], None] or None
- Callback function to call on update.
- """
-
- enabled: bool
- progress_bar: tqdm
- filesize: int
- count: int
- trip: int
- callback: Optional[Callable[[int], None]]
-
-###############################################################################
-
- def __init__(self, enabled: bool, filesize: int, key: str,
- callback: Optional[Callable[[int], None]] = None) -> None:
- """
- Initializes a state-aware tqdm ProgressBar.
-
- Parameters
- -----------
- enabled : bool
- Enable / Disable progress bar
- filesize : int
- Size of the file object in bytes
- key : str
- key/filename of the file object
- callback : function(chunksize: int)
- Callback function to call after each update
- """
-
- self.enabled = enabled
- self.callback = callback
- self.filesize = filesize
- self.count = 0
- self.trip = 3
- if self.enabled:
- self.progress_bar = tqdm(total=filesize, unit="B", unit_scale=True,
- desc=key, ascii=True, maxinterval=1)
-
-###############################################################################
-
- def update(self, chunksize: int) -> None:
- """
- Updates the progress bar with respect to the consumed chunksize.
- If a callafter function has been provided to the Constructor, it is
- called during the update.
- """
-
- self.count += chunksize
- if self.enabled:
- self.progress_bar.update(chunksize)
- self.trip -= 1
- if self.trip == 0:
- self.progress_bar.refresh()
- self.trip = 3
- if self.count >= self.filesize:
- self.progress_bar.refresh()
- if self.callback:
- self.callback(chunksize)
-
-###############################################################################
diff --git a/src/coscine/vocabulary.py b/src/coscine/vocabulary.py
deleted file mode 100644
index 6602dd5ea8fc534f8decce069d0c11fdd0ed4e8b..0000000000000000000000000000000000000000
--- a/src/coscine/vocabulary.py
+++ /dev/null
@@ -1,429 +0,0 @@
-###############################################################################
-# Coscine Python SDK
-# Copyright (c) 2018-2023 RWTH Aachen University
-# Licensed under the terms of the MIT License
-###############################################################################
-# Coscine, short for Collaborative Scientific Integration Environment, is
-# a platform for research data management (RDM).
-# For more information on Coscine visit https://www.coscine.de/.
-#
-# Please note that this python module is open source software primarily
-# developed and maintained by the scientific community. It is not
-# an official service that RWTH Aachen provides support for.
-###############################################################################
-
-###############################################################################
-# File description
-###############################################################################
-
-"""
-This file implements various classes for querying, parsing and interacting
-with data inside Coscine vocabularies.
-"""
-
-###############################################################################
-# Dependencies
-###############################################################################
-
-from __future__ import annotations
-from typing import TYPE_CHECKING, Tuple, Union, List
-import os
-import json
-import difflib
-import pkgutil
-import urllib.parse
-from coscine.defaults import LANGUAGES
-from coscine.form import FormField
-from coscine.graph import ApplicationProfile
-from coscine.utils import is_subset
-if TYPE_CHECKING:
- from coscine.client import Client
-
-###############################################################################
-# Classes / Functions / Scripts
-###############################################################################
-
-class Vocabulary:
- """
- A wrapper around Coscine vocabularies. Vocabularies from Coscine
- do not necessarily all follow the same format, so this class takes
- care of abstracting the Vocabulary interface.
- The internal language of the Vocabulary changes automatically based
- on the Client setting `language`.
-
- Attributes
- -----------
- raw : dict
- Contains the raw dictionary data that is served by Coscine. Parsing
- for language is not taken into account.
- content : dict
- Contains the raw dictionary data that is served by Coscine
- for the language preset set in coscine.Client .
- """
-
- client: Client
- _data: dict
-
- @property
- def raw(self) -> dict:
- """Refers to the raw Coscine vocabulary"""
- return self._data
-
- @property
- def content(self) -> dict:
- """Returns the vocabulary in the client language setting"""
- data = self.raw
- languages = self.languages()
- # Select appropriate language branch
- if self.client.settings.language in languages:
- data = data[self.client.settings.language]
- elif len(languages) > 0:
- data = data[languages[0]]
- return data
-
-###############################################################################
-
- def __init__(self, client: Client, data: dict) -> None:
- self.client = client
- self._data = data
-
-###############################################################################
-
- def __str__(self) -> str:
- return json.dumps(self.raw, indent=4)
-
-###############################################################################
-
- def keys(self) -> List[str]:
- return [entry["name"] for entry in self.content]
-
-###############################################################################
-
- def values(self) -> List[object]:
- return [entry["value"] for entry in self.content]
-
-###############################################################################
-
- def items(self) -> List[Tuple[str, object]]:
- return list(zip(self.keys(), self.values()))
-
-###############################################################################
-
- def languages(self) -> List[str]:
- """
- Returns the languages supported by the dictionary.
- Some dictionaries may not contain a language setting and thus
- are valid for all languages. In that case an empty list is returned.
- """
-
- languages = []
- for lang in LANGUAGES:
- if lang in self.raw:
- languages.append(lang)
- return languages
-
-###############################################################################
-
- def contains(self, key: str) -> bool:
- """
- Determines whether the vocabulary contains the given key.
-
- Parameters
- ----------
- key : object (str, list or dict)
- The key to look for in the vocabulary.
- reverse : bool
- If set to true, key is assumed to be a value inside of the
- vocabulary and thus all values are searched for key.
- """
-
- return self.lookup_key(key) is not None
-
-###############################################################################
-
- def suggest(self, key: str) -> str:
- """
- Returns the closest matches for a given key based on
- the Levenshtein distance
- """
- keys: List[str] = [entry["name"] for entry in self.content]
- close_matches = difflib.get_close_matches(key, keys, n=3, cutoff=0.6)
- suggestions = [f"'{match}'" for match in close_matches]
- return " or ".join(suggestions)
-
-###############################################################################
-
- def lookup_key(self, key: str) -> object:
- """
- Looks up the counterpart of $key.
-
- Parameters
- ----------
- key : object (str, list or dict)
- The key to look for in the vocabulary.
- reverse : bool
- If set to true, key is assumed to be a value inside of the
- vocabulary and thus all values are searched for key.
- """
-
- for entry in self.content:
- if entry["name"] == key:
- return entry["value"]
- return None
-
-###############################################################################
-
- def lookup_value(self, value: object) -> Union[str, None]:
- """
- Returns the key referenced by value
- """
-
- if isinstance(value, list) and len(value) == 1:
- value = value[0]
- for entry in self.content:
- if isinstance(value, str):
- if entry["value"] == value:
- return entry["name"]
- elif isinstance(value, dict):
- if is_subset(value, entry["value"]):
- return entry["name"]
- return None
-
-###############################################################################
-# Class
-###############################################################################
-
-class VocabularyManager:
- """
- The VocabularyManager takes care of loading local InputForms and
- querying vocabularies from the Coscine servers. Local InputForms
- are stored in src/data in json format.
- It makes use of the Cache class thus making it more efficient at
- querying vocabularies.
- """
-
- client: Client
- _builtins: dict
-
-###############################################################################
-
- def __init__(self, client: Client) -> None:
- """
- Initializes the VocabularyManager with a client instance for
- talking to the Coscine servers.
- """
- self.client = client
- project_template = pkgutil.get_data(__name__, "data/project.json")
- resource_template = pkgutil.get_data(__name__, "data/resource.json")
- if project_template is None or resource_template is None:
- raise FileNotFoundError()
- self._builtins = {
- "project": json.loads(project_template.decode("utf-8")),
- "resource": json.loads(resource_template.decode("utf-8"))
- }
- for graph in self._builtins:
- container = []
- for element in self._builtins[graph]:
- container.append(FormField(client, element))
- self._builtins[graph] = container
-
-###############################################################################
-
- def builtin(self, name: str) -> List[FormField]:
- """
- Returns a builtin InputForm as a list of FormFields.
-
- Parameters
- -----------
- name : str
- Name of the builtin vocabulary (filename in src/data
- without filetype).
- """
- if name not in self._builtins:
- raise ValueError("Invalid builtin InputForm name!")
- return self._builtins[name]
-
-###############################################################################
-
- def licenses(self, normalize: bool = True) -> Vocabulary:
- """
- Returns a dictionary containing a list of licenses
- available in Coscine.
- """
-
- uri = self.client.uri("Project", "License")
- data = self.client.static_request(uri)
- if normalize:
- normalized: dict = {"en": []}
- for entry in data:
- normalized["en"].append({
- "name": entry["displayName"],
- "value": entry
- })
- data = normalized
- return Vocabulary(self.client, data)
-
-###############################################################################
-
- def resource_types(self, normalize: bool = True) -> Vocabulary:
- """
- Retrieves a list of the available resource types in Coscine.
- Only enabled (currently usable) resource types are returned.
- """
-
- uri = self.client.uri("Resources", "ResourceType", "types")
- data = self.client.static_request(uri)
- if normalize:
- normalized: dict = {"en": []}
- for entry in data:
- if entry["isEnabled"]:
- normalized["en"].append({
- "name": entry["displayName"],
- "value": entry
- })
- data = normalized
- return Vocabulary(self.client, data)
-
-###############################################################################
-
- def application_profiles(self, normalize: bool = True) -> Vocabulary:
- """
- Returns a list of all available Coscine application profiles.
- """
-
- uri = self.client.uri("Metadata", "Metadata", "profiles")
- data = self.client.static_request(uri)
- if normalize:
- normalized: dict = {"en": []}
- for entry in data:
- name = urllib.parse.urlparse(entry)[2]
- name = os.path.relpath(name, "/coscine/ap/")
- normalized["en"].append({
- "name": name,
- "value": entry
- })
- data = normalized
- return Vocabulary(self.client, data)
-
-###############################################################################
-
- def application_profile(self, path: str) -> ApplicationProfile:
- """
- Retrieves a specific Coscine application profile.
-
- Parameters
- -----------
- path : str
- Path/Url to the application profile.
- (e.g. Resource.data["applicationProfile"]))
- id : str, default: None
- Coscine resource ID.
-
- Returns
- -------
- ApplicationProfile
- A parsed application profile.
- """
-
- uri = self.client.uri("Metadata", "Metadata", "profiles", path)
- data = self.client.static_request(uri)
- graph = json.dumps(data[0]["@graph"])
- return ApplicationProfile(self.client, graph)
-
-###############################################################################
-
- def instance(self, link: str) -> Vocabulary:
- """
- Retrieves a data instance. Instances are always normalized.
-
- Parameters
- -----------
- link : str
- link to the instance
- """
-
- uri = self.client.uri("Metadata", "Metadata", "instances", link)
- instance = self.client.static_request(uri)
- return Vocabulary(self.client, instance)
-
-###############################################################################
-
- def organizations(
- self,
- normalize: bool = True,
- filter: str = ""
- ) -> Vocabulary:
- """
- Queries the organizations (e.g. 'RWTH Aachen University') available
- for selection in Coscine.
- """
-
- if filter:
- raise NotImplementedError(
- "Handling of filter argument not implemented!"
- )
- uri = self.client.uri("Organization", "Organization", "-", "ror")
- data = self.client.static_request(uri)
- if normalize:
- normalized: dict = {"en": []}
- for entry in data["data"]:
- normalized["en"].append({
- "name": entry["displayName"],
- "value": entry
- })
- data = normalized
- return Vocabulary(self.client, data)
-
-###############################################################################
-
- def visibility(self, normalize: bool = True) -> Vocabulary:
- """
- Returns the key-value mapping of the Coscine project visibility field.
-
- Returns
- -------
- dict
- Key-value mapped Coscine project visibility field.
- """
-
- uri = self.client.uri("Project", "Visibility")
- data = self.client.static_request(uri)
- if normalize:
- normalized: dict = {"en": []}
- for entry in data:
- normalized["en"].append({
- "name": entry["displayName"],
- "value": entry
- })
- data = normalized
- return Vocabulary(self.client, data)
-
-###############################################################################
-
- def disciplines(self, normalize: bool = True) -> Vocabulary:
- """
- Queries the scientific disciplines available for selection in Coscine.
-
- Returns
- -------
- dict
- Dictionary containing the disciplines.
- """
-
- uri = self.client.uri("Project", "Discipline")
- data = self.client.static_request(uri)
- if normalize:
- normalized: dict = {"en": [], "de": []}
- for entry in data:
- normalized["en"].append({
- "name": entry["displayNameEn"],
- "value": entry
- })
- normalized["de"].append({
- "name": entry["displayNameDe"],
- "value": entry
- })
- data = normalized
- return Vocabulary(self.client, data)
-
-###############################################################################
diff --git a/src/docs/examples.md b/src/docs/examples.md
deleted file mode 100644
index 902148b594b258a850f77d3aa5e2d5d78027cd8b..0000000000000000000000000000000000000000
--- a/src/docs/examples.md
+++ /dev/null
@@ -1,252 +0,0 @@
-# Examples
-## Inviting a list of project members
-Sometimes you need to invite a lot of people to a project. If you already have
-a list of their E-Mail addresses, you can easily iterate over the list and
-invite every single one of them.
-```python
-from typing import List
-import coscine
-
-TOKEN: str = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
-PROJECT_NAME: str = "My Project"
-EMAILS: List[str] = [
- "adam@example.com",
- "eva@example.com"
-]
-
-client = coscine.Client(TOKEN)
-project = client.project(PROJECT_NAME)
-for email in EMAILS:
- project.invite(email)
-```
-## Uploading a file to a resource
-```python
-# Import the package
-import coscine
-from datetime import datetime
-
-# We read our token from a file called 'token.txt'.
-# Note: We could store our token directly inside the sourcecode
-# as a string, but this is not recommended!
-with open("token.txt", "rt", encoding="utf-8") as file:
- token = file.read()
-
-# Create an instance of the Coscine client
-client = coscine.Client(token)
-project = client.project("My Project")
-resource = project.resource("My Resource")
-
-# We could create metadata in json-ld format by hand and use it
-# to upload a file or assign metadata to a file. However this
-# proves to be cumbersome and prone to ambiguous errors.
-# Therefore we'd like to use some form or template, which the
-# coscine python package is able to create for us!
-# The template is created by requesting the application profile
-# and the controlled vocabulary used within project and resource
-# and examining their fields as well as the constraints put
-# upon those fields.
-# After that a custom dictionary-like datatype is created and filled
-# with default and fixed values (as specified during resource creation).
-# We can interact with this dictionary just like with any other dictionary
-# with the difference being, that we can only set fields specified in
-# the application profile and - in case of fields controlled
-# by a vocabulary - can only use values of that vocabulary.
-# Furthermore the printing functionality has been altered, to convey
-# more information about the fields stored within the template.
-# Just try print(form) and figure it out for yourself.
-form = resource.metadata_form()
-
-# We can now modify the template.
-# Let's assume our resource is using the ENGMETA metadata scheme, and set
-# the required values for our metadata:
-form["Title"] = "Hello World!"
-form["Creator"] = "John Doe"
-form["Contact"] = "John Doe"
-form["Creation Date"] = datetime.now()
-form["Publication Date"] = datetime.now()
-form["Embargo End Date"] = datetime(1900, 1, 1)
-form["Version"] = "1.0"
-form["Mode"] = "Experiment"
-form["Step"] = 42
-form["Type"] = "Image"
-form["Subject Area"] = "Medicine"
-
-# Do note that the fields "Mode", "Type" and "Subject Area" are
-# controlled by a vocabulary. You can just set the value you'd
-# select in the web interface of Coscine and let the custom dictionary
-# automatically resolve the actual value.
-# "Actual value?" you are asking? Well - when we set "Type" to "Image"
-# we are not actually assigning the string "Image" to it, but rather
-# a uniform resource identifier indicating the image type. This is
-# intransparent to the user, but allows us to work with "Images" and
-# "Waveforms" rather than "http://long-url.xd/id=Images", etc.
-# Note that this also applies to controlled fields containing a fixed
-# or default value - they will not contain "Images" but rather a
-# uri resolving to "Images".
-
-# Let's print our template and inspect how it looks.
-# Make sure we filled in all required fields, so that no errors
-# are thrown around when we try to upload.
-print(form)
-
-# We can now directly use this template as our metadata or generate
-# a json-ld representation of our metadata using:
-metadata = form.generate()
-
-# Note that the generate() function will validate that each required
-# field is present and that all fields are correctly formatted before
-# generating the metadata. Thus be prepared to catch some exceptions.
-# We do not need to generate the metadata before using functions like
-# upload() though, as this is done internally.
-
-# The upload_file() function requires 3 arguments:
-# filename: Which filename/path should the file assume within the resource?
-# path: The local path to the file
-# metadata: Either a filled in metadata form or json-ld formatted
-# metadata dictionary. If you specify a template it is validated before
-# uploading. Thus, if you have not used template.generate() before
-# you should now be prepared to catch some exceptions in case the
-# metadata is containing bad values.
-filename = "My Research Data.csv"
-resource.upload(filename, filename, metadata)
-```
-
-## Modifying metadata of files inside an (S3-)resource
-Files stored in an s3-resource do not require you to specify metadata on upload. However it is considered good practice to tag each file with metadata anyway. Here is an easy way of doing just that. The example assumes you've got a few files present inside an s3-resource, all of which have little to no metadata yet.
-
-```python
-import coscine
-
-TOKEN: str = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
-PROJECT_NAME: str = "My Project"
-RESOURCE_NAME: str = "My Resource"
-
-client = coscine.Client(TOKEN)
-project = client.project(PROJECT_NAME)
-resource = project.resource(RESOURCE_NAME)
-# This loop would set the same metadata for each file.
-# Obviously this doesn't make much sense in the real world.
-for file in resource.contents():
- if not file.is_folder:
- form = file.form()
- form["Title"] = "My Title"
- form["Author"] = "Mrs. X"
- file.update(form)
-```
-
-## Establishing an S3-connection
-Using the S3 library of your choice it's quite easy to connect to an S3 resource. The following short snippet shows how to use amazons boto3 SDK to connect to a Coscine S3 resource.
-
-```python
-import coscine
-import boto3
-
-TOKEN: str = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
-PROJECT_NAME: str = "My Project"
-RESOURCE_NAME: str = "My S3 Resource"
-FILE_NAME: str = "MyFile.ext"
-
-client = coscine.Client(TOKEN)
-project = client.project(PROJECT_NAME)
-resource = project.resource(RESOURCE_NAME)
-s3 = boto3.resource("s3", aws_access_key_id=resource.s3.write_access_key,
- aws_secret_access_key=resource.s3.write_secret_key,
- endpoint_url=resource.s3.endpoint)
-bucket = s3.Bucket(resource.s3.bucket)
-filesize = bucket.Object(FILE_NAME).content_length
-bucket.download_file(FILE_NAME, "./") # path "./" (current directory)
-```
-
-## Asynchronous requests
-The Coscine Python SDK uses the requests module, which does only
-offer synchronous http requests. However with the help of some
-additional modules, we can easily make any function of the Coscine Python
-SDK asynchronous and thus might be able to speed certain things up.
-With the *concurrent.futures* module we can take advantage of a
-*ThreadPool*, offloading tasks to multiple threads:
-
-```python
-import coscine
-from concurrent.futures import ThreadPoolExecutor, wait
-
-PROJECT_NAME: str = "My Project"
-RESOURCE_NAME: str = "My S3 Resource"
-
-client = coscine.Client()
-project = client.project(PROJECT_NAME)
-resource = project.resource(RESOURCE_NAME)
-
-# Asynchronously download files in a resource
-# (Note: This has already been incorporated into the SDK, no need
-# to do that by hand. However other functions may benefit from the
-# same strategy too...)
-with ThreadPoolExecutor(max_workers=4) as executor:
- files = resource.objects()
- futures = [executor.submit(file.download, path) for file in files]
- wait(futures)
- for index, fut in enumerate(futures):
- fut.result()
-```
-
-## GUI integration
-The Coscine Python SDK has been implicitly written with GUIs in mind -
-building a GUI around it is very easy.
-The *Qt Framework* is well established in commercial, industrial and scientific
-use. We'll be using its python binding *PyQt5* in this short example:
-
-```python
-import coscine
-from PyQt5.QtCore import *
-from PyQt5.QtGui import *
-from PyQt5.QtWidgets import *
-
-TOKEN: str = "XXXXXXXXXXXXXXXXXXX"
-PROJECT_NAME: str = "My Project"
-RESOURCE_NAME: str = "My Resource"
-FILE_NAME: str = "My file.jpeg"
-
-client = coscine.Client(TOKEN)
-project = client.project(PROJECT_NAME)
-resource = project.resource(RESOURCE_NAME)
-file = resource.object(FILE_NAME)
-metadata = file.form()
-
-# GUI visualization
-# (Obviously you should put that in a class)
-app = QApplication([])
-dialog = QDialog()
-dialog.setWindowTitle("Metadata Editor")
-buttonBox = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
-#buttonBox.accepted.connect(self.accept)
-#buttonBox.rejected.connect(self.reject)
-
-# The magic happens here
-group = QGroupBox("Metadata")
-layout = QFormLayout()
-for key in metadata.keys():
- if metadata.is_controlled(key):
- widget = QComboBox()
- widget.setEditable(True)
- widget.setInsertPolicy(QComboBox.NoInsert)
- widget.completer().setCompletionMode(QCompleter.PopupCompletion)
- for entry in metadata.get_vocabulary(key):
- widget.addItem(entry)
- else:
- widget = QLineEdit(metadata[key])
- layout.addRow(QLabel(key), widget)
-group.setLayout(layout)
-
-scrollArea = QScrollArea()
-scrollArea.setWidget(group)
-scrollArea.setHorizontalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAlwaysOff)
-
-mainLayout = QVBoxLayout()
-mainLayout.addWidget(scrollArea)
-mainLayout.addWidget(buttonBox)
-dialog.setLayout(mainLayout)
-dialog.exec_()
-```
-
-## Need more?
-For more insights have a look at the unittests in `src/tests` or directly
-take a look at the source code of the package. It is not that complicated!
diff --git a/src/docs/getting_started.md b/src/docs/getting_started.md
deleted file mode 100644
index dde1848ff8ab4e2ffd603556b05fca96bf329179..0000000000000000000000000000000000000000
--- a/src/docs/getting_started.md
+++ /dev/null
@@ -1,130 +0,0 @@
-# Getting Started
-## Installation
-### Installing python
-This is platform dependent, so you need to figure out how to install python
-by yourself, if it is not already installed on your system.
-This will get you started:
-
-- [https://wiki.python.org/moin/BeginnersGuide](https://wiki.python.org/moin/BeginnersGuide)
-- [https://www.python.org/downloads/](https://www.python.org/downloads/)
-
-In the following snippets, depending on your installation
-and platform, you may have to substitute `python` with
-`py` or `python3`.
-
-### Installing the Coscine Python SDK package
-#### Using pip
-The Coscine Python SDK package is hosted on the
-Python Package Index (PyPi). You can install and update it
-and all of its dependencies via the Python Package
-Manager (pip):
-```sh
-python -m pip install --upgrade coscine
-```
-The `--upgrade` option updates the package if it is already
-installed on your system.
-
-#### Using conda
-This module's pypi version is mirrored in conda forge. You can
-install it and all of its dependencies via Conda:
-```sh
-conda install -c conda-forge coscine
-```
-and likewise for updating it:
-```sh
-conda update -c conda-forge coscine
-```
-
-#### Using git
-You can install the python module with a local copy of the
-source code, which you can grab directly via git:
-```sh
-git clone https://git.rwth-aachen.de/coscine/community-features/coscine-python-sdk.git
-cd ./coscine-python-sdk
-py setup.py
-```
-Please note that the git version may contain features not
-yet released on any of the other hosting platforms. There
-features might not be stable yet or provisional. In general
-the git repository contains the most up to date version but
-might be a little rough around the edges in certain areas.
-If you would like a stable release, refer to one of the
-git tags.
-
-## Creating an API token
-You need an API token to use the Coscine API. If you have not already,
-[create a new API token](https://docs.coscine.de/en/user/token/).
-Once you have an API token you are ready to use the API.
-
-### A word of advice
-The token represents sensible data and grants anyone in
-possesion of it full access to your data in coscine.
-Do not leak it publicly on online platforms such as github! Do
-not include it within your sourcecode if you intend on
-uploading that to the internet or sharing it among peers.
-Take precautions and follow best practices to avoid
-corruption, theft or loss of data!
-
-### Using the API token
-There are two simple and safe methods of using the API token
-without exposing it to unintended audiences.
-
-#### Storing the token in a file and loading it as needed
-Simply put your API token in a file on your harddrive and read
-the file when initializing the Coscine client. This has the
-advantage of keeping the token out of the sourcecode and
-offering the user an easy way to switch between tokens by
-changing the filename in the sourcecode.
-```python
-import os
-
-fd = open("token.txt", "rt")
-token = fd.read()
-fd.close()
-```
-However it comes at the disadvantage of potentially exposing
-the token by accidentially leaking the file together with
-the sourcecode e. g. on online platforms such as GitHub.
-Therefore precautions must be taken i. e. when using git as a
-versioning system. A `.gitignore` file including any possible
-token name or file extension should be mandatory. You could
-for example exclude the filename *token.txt*. A better way
-would be to agree upon a common token file extension
-such as `.token` and exclude that file extension. Then you can
-safely push your code to online platforms such as GitLab
-or GitHub, since the `.gitignore` makes sure your sensitive
-files are not included in any uploads.
-- [gitignore guide by W3schools.com](https://www.w3schools.com/git/git_ignore.asp)
-- [gitignore docs on git-scm.com](https://git-scm.com/docs/gitignore)
-
-#### Storing the token in an environment variable
-This method does not rely on any files but instead on
-[environment variables](https://en.wikipedia.org/wiki/Environment_variable).
-Simply set an environment variable containing your token and
-use that variable from your python program.
-***Set environment variable:***
-```python
-import os
-os.environ["COSCINE_API_TOKEN"] = "My Token Value"
-```
-You would do this only once, and obviously not in your
-final program/script. There are alternative
-platform-specific methods for creating an environment
-variable (see links given below), but this pythonic
-approach should work on most # platforms out of the box.
-Just don't do this in your actual program - do it with an
-extra script or in an interactive python session (shell).
-***Get the environment variable***
-```python
-import os
-token = os.getenv("COSCINE_API_TOKEN")
-```
-This is certainly a little more complex for some users
-who may want to use your program. They can easily share
-tokens by sending a file to colleagues but sharing
-environment variables requires each user to additionally
-create the environment variable on their local system.
-Find out how to temporarily or permanently set environment
-variables on certain Operating Systems:
-- [Windows](http://www.dowdandassociates.com/blog/content/howto-set-an-environment-variable-in-windows-command-line-and-registry/)
-- [Linux](https://phoenixnap.com/kb/linux-set-environment-variable)
diff --git a/src/docs/template/module.html.jinja2 b/src/docs/template/module.html.jinja2
deleted file mode 100644
index 4a60d0cc3ec8368105fee119438baf8c1ea69ebd..0000000000000000000000000000000000000000
--- a/src/docs/template/module.html.jinja2
+++ /dev/null
@@ -1,63 +0,0 @@
-{% extends "default/module.html.jinja2" %}
-
-{% block title %}Coscine Python SDK{% endblock %}
-{% block nav %}
- {% block module_list_link %}
- {% set parentmodule = ".".join(module.modulename.split(".")[:-1]) %}
- {% if parentmodule and parentmodule in all_modules %}
- <a class="pdoc-button module-list-button" href="../{{ parentmodule.split(".")[-1] }}.html">
- {% include "resources/box-arrow-in-left.svg" %}
-
- {{- parentmodule -}}
- </a>
- {% elif not root_module_name %}
- <a class="pdoc-button module-list-button" href="{{ "../" * module.modulename.count(".") }}index.html">
- {% include "resources/box-arrow-in-left.svg" %}
-
- Module Index
- </a>
- {% endif %}
- {% endblock %}
-
- {% block nav_title %}
- <a href="https://git.rwth-aachen.de/coscine/community-features/coscine-python-sdk">
- <img src="https://git.rwth-aachen.de/coscine/community-features/coscine-python-sdk/-/raw/master/data/coscine_logo_rgb.png" class="logo" alt="project logo"/>
- </a>
- {% endblock %}
-
- {% block search_box %}
- {% if search and all_modules|length > 1 %}
- {# we set a pattern here so that we can use the :valid CSS selector #}
- <input type="search" placeholder="Search..." role="searchbox" aria-label="search"
- pattern=".+" required>
- {% endif %}
- {% endblock %}
-
- {% set index = module.docstring | to_markdown | to_html | attr("toc_html") %}
- {% if index %}
- <h2>Contents</h2>
- {{ index | safe }}
- {% endif %}
-
- {% if module.submodules %}
- <h2>API Reference</h2>
- <ul>
- {% for submodule in module.submodules if is_public(submodule) | trim %}
- <li>{{ submodule.taken_from | link(text=submodule.name) }}</li>
- {% endfor %}
- </ul>
- {% endif %}
-
- {% if module.members %}
- <h2>API Reference</h2>
- {{ nav_members(module.members.values()) }}
- {% endif %}
-
- {% block nav_footer %}
- <footer>Coscine Python SDK Version 0.9.0</footer>
- {% endblock %}
-{% endblock nav %}
-
-{% block favicon %}
- <link rel="icon" type="image/png" href="https://git.rwth-aachen.de/coscine/community-features/coscine-python-sdk/-/raw/master/data/coscine_python_sdk_icon_rgb.png"/>
-{% endblock %}
diff --git a/src/docs/template/syntax-highlighting.css b/src/docs/template/syntax-highlighting.css
deleted file mode 100644
index b0a7fe3746cdf607a1b31a6fb97a691eab9a2286..0000000000000000000000000000000000000000
--- a/src/docs/template/syntax-highlighting.css
+++ /dev/null
@@ -1,80 +0,0 @@
-/* monokai color scheme, see pdoc/template/README.md */
-pre { line-height: 125%; }
-span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 20px; }
-.pdoc-code .hll { background-color: #49483e }
-.pdoc-code { background: #272822; color: #f8f8f2 }
-.pdoc-code .c { color: #75715e } /* Comment */
-.pdoc-code .err { color: #960050; background-color: #1e0010 } /* Error */
-.pdoc-code .esc { color: #f8f8f2 } /* Escape */
-.pdoc-code .g { color: #f8f8f2 } /* Generic */
-.pdoc-code .k { color: #66d9ef } /* Keyword */
-.pdoc-code .l { color: #ae81ff } /* Literal */
-.pdoc-code .n { color: #f8f8f2 } /* Name */
-.pdoc-code .o { color: #f92672 } /* Operator */
-.pdoc-code .x { color: #f8f8f2 } /* Other */
-.pdoc-code .p { color: #f8f8f2 } /* Punctuation */
-.pdoc-code .ch { color: #75715e } /* Comment.Hashbang */
-.pdoc-code .cm { color: #75715e } /* Comment.Multiline */
-.pdoc-code .cp { color: #75715e } /* Comment.Preproc */
-.pdoc-code .cpf { color: #75715e } /* Comment.PreprocFile */
-.pdoc-code .c1 { color: #75715e } /* Comment.Single */
-.pdoc-code .cs { color: #75715e } /* Comment.Special */
-.pdoc-code .gd { color: #f92672 } /* Generic.Deleted */
-.pdoc-code .ge { color: #f8f8f2; font-style: italic } /* Generic.Emph */
-.pdoc-code .gr { color: #f8f8f2 } /* Generic.Error */
-.pdoc-code .gh { color: #f8f8f2 } /* Generic.Heading */
-.pdoc-code .gi { color: #a6e22e } /* Generic.Inserted */
-.pdoc-code .go { color: #66d9ef } /* Generic.Output */
-.pdoc-code .gp { color: #f92672; font-weight: bold } /* Generic.Prompt */
-.pdoc-code .gs { color: #f8f8f2; font-weight: bold } /* Generic.Strong */
-.pdoc-code .gu { color: #75715e } /* Generic.Subheading */
-.pdoc-code .gt { color: #f8f8f2 } /* Generic.Traceback */
-.pdoc-code .kc { color: #66d9ef } /* Keyword.Constant */
-.pdoc-code .kd { color: #66d9ef } /* Keyword.Declaration */
-.pdoc-code .kn { color: #f92672 } /* Keyword.Namespace */
-.pdoc-code .kp { color: #66d9ef } /* Keyword.Pseudo */
-.pdoc-code .kr { color: #66d9ef } /* Keyword.Reserved */
-.pdoc-code .kt { color: #66d9ef } /* Keyword.Type */
-.pdoc-code .ld { color: #e6db74 } /* Literal.Date */
-.pdoc-code .m { color: #ae81ff } /* Literal.Number */
-.pdoc-code .s { color: #e6db74 } /* Literal.String */
-.pdoc-code .na { color: #a6e22e } /* Name.Attribute */
-.pdoc-code .nb { color: #f8f8f2 } /* Name.Builtin */
-.pdoc-code .nc { color: #a6e22e } /* Name.Class */
-.pdoc-code .no { color: #66d9ef } /* Name.Constant */
-.pdoc-code .nd { color: #a6e22e } /* Name.Decorator */
-.pdoc-code .ni { color: #f8f8f2 } /* Name.Entity */
-.pdoc-code .ne { color: #a6e22e } /* Name.Exception */
-.pdoc-code .nf { color: #a6e22e } /* Name.Function */
-.pdoc-code .nl { color: #f8f8f2 } /* Name.Label */
-.pdoc-code .nn { color: #f8f8f2 } /* Name.Namespace */
-.pdoc-code .nx { color: #a6e22e } /* Name.Other */
-.pdoc-code .py { color: #f8f8f2 } /* Name.Property */
-.pdoc-code .nt { color: #f92672 } /* Name.Tag */
-.pdoc-code .nv { color: #f8f8f2 } /* Name.Variable */
-.pdoc-code .ow { color: #f92672 } /* Operator.Word */
-.pdoc-code .w { color: #f8f8f2 } /* Text.Whitespace */
-.pdoc-code .mb { color: #ae81ff } /* Literal.Number.Bin */
-.pdoc-code .mf { color: #ae81ff } /* Literal.Number.Float */
-.pdoc-code .mh { color: #ae81ff } /* Literal.Number.Hex */
-.pdoc-code .mi { color: #ae81ff } /* Literal.Number.Integer */
-.pdoc-code .mo { color: #ae81ff } /* Literal.Number.Oct */
-.pdoc-code .sa { color: #e6db74 } /* Literal.String.Affix */
-.pdoc-code .sb { color: #e6db74 } /* Literal.String.Backtick */
-.pdoc-code .sc { color: #e6db74 } /* Literal.String.Char */
-.pdoc-code .dl { color: #e6db74 } /* Literal.String.Delimiter */
-.pdoc-code .sd { color: #e6db74 } /* Literal.String.Doc */
-.pdoc-code .s2 { color: #e6db74 } /* Literal.String.Double */
-.pdoc-code .se { color: #ae81ff } /* Literal.String.Escape */
-.pdoc-code .sh { color: #e6db74 } /* Literal.String.Heredoc */
-.pdoc-code .si { color: #e6db74 } /* Literal.String.Interpol */
-.pdoc-code .sx { color: #e6db74 } /* Literal.String.Other */
-.pdoc-code .sr { color: #e6db74 } /* Literal.String.Regex */
-.pdoc-code .s1 { color: #e6db74 } /* Literal.String.Single */
-.pdoc-code .ss { color: #e6db74 } /* Literal.String.Symbol */
-.pdoc-code .bp { color: #f8f8f2 } /* Name.Builtin.Pseudo */
-.pdoc-code .fm { color: #a6e22e } /* Name.Function.Magic */
-.pdoc-code .vc { color: #f8f8f2 } /* Name.Variable.Class */
-.pdoc-code .vg { color: #f8f8f2 } /* Name.Variable.Global */
-.pdoc-code .vi { color: #f8f8f2 } /* Name.Variable.Instance */
-.pdoc-code .vm { color: #f8f8f2 } /* Name.Variable.Magic */
diff --git a/src/docs/template/theme.css b/src/docs/template/theme.css
deleted file mode 100644
index 1dfe7c4a1c300ecced5a18c5030c7efcd18779e4..0000000000000000000000000000000000000000
--- a/src/docs/template/theme.css
+++ /dev/null
@@ -1,20 +0,0 @@
-:root {
- --pdoc-background: #212529;
-}
-
-.pdoc {
- --text: #f7f7f7;
- --muted: #9d9d9d;
- --link: #58a6ff;
- --link-hover: #3989ff;
- --code: #333;
- --active: #555;
-
- --accent: #343434;
- --accent2: #555;
-
- --nav-hover: rgba(0, 0, 0, 0.1);
- --name: #77C1FF;
- --def: #0cdd0c;
- --annotation: #00c037;
-}
diff --git a/src/docs/usage.md b/src/docs/usage.md
deleted file mode 100644
index 93c92aeb2b0d4e7d78d1e80e077c7523ffe5f533..0000000000000000000000000000000000000000
--- a/src/docs/usage.md
+++ /dev/null
@@ -1,657 +0,0 @@
-# Using the Coscine Python SDK
-## General Information
-### Input Forms
-In many places of the Python SDK we use a custom datatype called
-InputForm, which is basically a dictionary with a lot of extra
-functionality added on top (such as controlled vocabularies). They provide
-an easier way of interacting with metadata by presenting and expecting
-data in human-readable form and only internally generating the Coscine
-expected format, which consists of a lot of ambiguous URLs and identifiers.
-
-#### Motivation
-> Why does the Coscine Python SDK use InputForms over methods with
-named arguments like the following one?
-```python
-def create_project("test", "test", datetime.now(), ...)
-```
-Multiple benefits arise from the use of a dictionary:
-- The dictionary keys are familiar from the Coscine web interface and
- change based on the python client language preset
-- The fields inside of the dictionary can be iterated over and their
- properties like controlled vocabularies can be queried
-
-The 2nd benefit ultimately enables easy use in GUI applications.
-You will encounter InputForms when dealing with
-Project, Resource or FileObject metadata.
-
-#### Printing an InputForm
-InputForms present metadata in human-readable form and expect data in
-human-readable format - just like in the Coscine web interface. When
-printing an InputForm, required fields are marked with an asterisk (*)
-next to their property name. Fields controlled by a vocabulary are marked with
-a `V` in the C (=controlled) column. Fields controlled by a selection are
-marked with an `S`. The expected datatype of a field is printed in the
-datatype column. In case the type is enclosed by angular brackets, more
-than one value can be specified (e.g. with a list).
-An example InputForm for metadata of a FileObject is printed below.
-```
-+---+------------------+--------------------------------+
-| C | Property | Value |
-+---+------------------+--------------------------------+
-| | Creator* | Dr. Akula |
-| | Title* | Pneumonia positive |
-| | Production Date* | 2021-05-20 |
-| V | Subject Area | Radiology and Nuclear Medicine |
-| V | Resource | Image |
-| | Rights | |
-| | Rightsholder | |
-+---+------------------+--------------------------------+
-```
-
-#### Gaining access to the controlled vocabulary
-Some fields inside an InputForm are controlld by a vocabulary. These fields
-only accept values that are contained inside of their vocabulary. To find
-out which values that includes, we can directly access the underlying
-vocabulary with
-```python
-import coscine
-
-client = coscine.Client("My Coscine API Token")
-form = client.project_form() # or any other form
-# get the vocabulary of the Disciplines key
-vocabulary = form.vocabulary("Discipline")
-
-# Find out whether the vocabulary contains some stuff
-if vocabulary.contains("Alienology"):
- send_greetings_to_mars("They know! Prepare the Invasion!")
-
-# Look up the Coscine internal representation of a key
-# We can also do the other direction and lookup the key given an internal
-# value with vocabulary.lookup_value()
-value = vocabulary.lookup_key("Computer Science 409")
-print(value) # yields:
-# {'id': 'cfd6b656-f4ba-48c6-b5d8-2f98a7b60d1f', 'url': 'http://www.dfg.de/dfg_profil/gremien/fachkollegien/liste/index.jsp?id=409', 'displayNameDe': 'Informatik 409', 'displayNameEn': 'Computer Science 409'}
-
-# Print all keys of the vocabulary which includes all accepted values
-for key in vocabulary.keys():
- print(key) # yields a lot of keys | Useful for GUI dropdown lists
-```
-
-### Parallelization
-Certain tasks run faster if executed concurrently. To facilitate faster
-execution, the Coscine Python SDK makes use of the `ThreadPool` class
-provided by the `concurrent.futures` module. It does not automatically
-parallelize parallelizable tasks though. Concurrency is achieved explicitly
-via the dedicated parallelization context manager provided by
-the Coscine Client class:
-```python
-import coscine
-client = coscine.Client("My Coscine API Token")
-project = client.project("My Coscine Project")
-emails = ["caesar@tu.brutus", ...]
-# Send all invites at once
-with coscine.concurrent():
- for email in emails:
- project.invite(email)
-# The next line only executes when all invites have been sent
-print("Done inviting!")
-```
-
-## ... With Projects
-Coscine is a project-based data management platform. A project represents
-the central hub for all of the data resources and people involved
-in a scientific undertaking.
-
-### The Project object
-The Coscine Python SDK abstract Coscine projects with a
-custom Project datatype. An instance of type `coscine.Project`
-has the following properties:
-
-|Property | Type | Description |
-|-----------------------|--------|-------------|
-|id|`str` |Unqiue Project identifier|
-|name|`str` |Full Project Name|
-|display_name|`str` |Project Display Name|
-|description|`str` |Project Description|
-|principle_investigators|`str` |Project Principle Investigators|
-|start_date|`datetime` |Project Inception date|
-|end_date|`datetime` |End of project lifespan|
-|disciplines|`list[str]` |List of involved scientific disciplines|
-|organizations|`list[str]` |List of involved institutions|
-|visibility|`str` |Project visiblity setting|
-
-Properties can be accessed via dot notation
-i. e. `print(project.name)`.
-
-### Get a list of projects
-The following snippet demonstrates how to get a list of
-Coscine Projects and prints all returned projects by their
-full name. The call `client.projects()` will only return
-toplevel projects (i.e. only the projects at the root level).
-To include subprojects in the results, the `toplevel`
-parameter must be set to `False` as demonstrated in
-another snippet further down below.
-```python
-import coscine
-
-client = coscine.Client("My Coscine API Token")
-projects = client.projects()
-for project in projects:
- print(project.name)
-```
-
-#### Filtering the Project result list
-The resulting list of projects can be filtered according
-to the project properties:
-```python
-from datetime import datetime
-import coscine
-
-client = coscine.Client("My Coscine API Token")
-
-# Get a list of all projects that have already ended.
-# This time including subprojects (toplevel=False).
-projects = client.projects(toplevel=False)
-results = filter(lambda project: project.end_date < datetime.now(), projects)
-for project in list(results):
- print(f"Project {project.name} has already ended.")
-```
-
-#### Getting an individual project
-Individual projects can directly be accessed through
-the `client.project()` method. This is just a convenience
-function, that internally does the same filtering as
-described above. It does however not return a list - it
-guarantees to return a single Project or fail.
-```python
-try:
- project = client.project("My Project Display Name")
- print(project.name)
-except IndexError:
- print("More than 1 project with the same name!")
-```
-
-#### Dealing with Subprojects
-The project object itself contains methods to
-query subprojects. Those are equivalent to the methods
-described above, but only return subprojects of a specific
-project.
-```python
-project.subprojects()
-project.subproject("Subproject Display Name")
-```
-
-### Printing a project
-When printing a project object, its metadata is printed
-formatted inside of a table.
-
-```python
-print(project)
-```
-*yields:*
-```
-+----------------------------------------------------------------+
-| Project [Cool] |
-+-------------------------+--------------------------------------+
-| Property | Value |
-+-------------------------+--------------------------------------+
-| ID | 501cfbe5-39ca-414e-9b42-2b033d740eef |
-| Name | Cool |
-| Display Name | Cool |
-| Principle Investigators | DFG Vampir |
-| Disciplines | Computer Science 409 |
-| Organizations | https://ror.org/04xfq0f34 |
-| Start Date | 2022-10-29 22:10:05 |
-| End Date | 2022-10-29 22:10:05 |
-| Visibility | Project Members |
-+-------------------------+--------------------------------------+
-```
-
-The keys and values are human readable and easy to understand.
-Under the hood it does not look that simple but sometimes we might
-need to access the Coscine internal identifiers for a certain field.
-In that case we can use the *data* dictionary, which grants us
-access to the metadata as seen and expected by the Coscine servers.
-Printing it yields a JSON representation of our projects metadata:
-
-```python
-print(project.data)
-```
-*yields:*
-
-```
-{
- "projectName": "Cool",
- "displayName": "Cool",
- "description": "Blablabla",
- "principleInvestigators": "DFG Vampir",
- "startDate": "2022-10-29T22:10:05",
- "endDate": "2022-10-29T22:10:05",
- "disciplines": [
- {
- "id": "cfd6b656-f4ba-48c6-b5d8-2f98a7b60d1f",
- "url": "http://www.dfg.de/dfg_profil/gremien/fachkollegien/liste/index.jsp?id=409",
- "displayNameDe": "Informatik 409",
- "displayNameEn": "Computer Science 409"
- }
- ],
- "organizations": [
- {
- "displayName": "RWTH Aachen University",
- "url": "https://ror.org/04xfq0f34"
- }
- ],
- "visibility": {
- "id": "8ab9c883-eb0d-4402-aaad-2e4007badce6",
- "displayName": "Project Members"
- }
-}
-```
-The data looks different and is (subjectively) a lot harder to interact
-with than before. Coscine does not care about human readable values
-such as `RWTH Aachen University` - those are actually silently ignored.
-What really matters are the unique identifiers and urls. Working with
-those may lead to one or another headache though... Thankfully the
-*ProjectForm* abstracts all of that away by assigning these identifiers
-automatically with its `generate()` method.
-The *ProjectForm* is described in detail in the example code for creating
-a project.
-
-### Create a project
-The python module provides a simple means of creating Coscine Projects:
-The *ProjectForm*.
-
-#### The Project Form
-
-#### Preset initialization
-```python
-import coscine
-
-client = coscine.Client("My Coscine API Token")
-project_metadata: dict = {
- "Project Name": "My Name",
- "Display Name": "My Display Name",
- ...
- "Visibility": "Project Members"
-}
-form = client.project_form(data)
-client.create_project(form)
-## Alternatively:
-# form = client.project_form()
-# form.fill(data)
-```
-#### Manual initialization/editing
-```python
-import coscine
-
-client = coscine.Client("My Coscine API Token")
-form = client.project_form()
-form["Projektname"] = "test"
-form["Anzeigename"] = "test"
-form["Projektbeschreibung"] = "test"
-form["Principal Investigators"] = "Mr. Test"
-form["Projektstart"] = datetime.now()
-form["Projektende"] = datetime.now()
-form["Disziplin"] = ["Informatik 409"]
-form["Teilnehmende Organisation"] = ["https://ror.org/04xfq0f34"]
-form["Sichtbarkeit"] = "Project Members"
-client.create_project(form)
-```
->**Special consideration** must be taken for the Organizations field!
-You can't just enter "RWTH Aachen" or similar organizations here, you have to
-first look them up at "[https://ror.org/search](https://ror.org/search)". If
-you do just put in an invalid string, you break something in your project.
-The creation will not fail, but you may get an error when visiting the
-project settings. Do not worry - you can simply fix the issue later on via
-the web interface or by updating the project metadata with the python sdk.
-The reason for all these shenanigans it, that the organizations database
-exceeds a couple of gigabytes. We cannot use a controlled vocabulary for this.
-You can query ror urls yourself and search for organizations via the ror API:
-[https://ror.readme.io/docs/rest-api](https://ror.readme.io/docs/rest-api).
-
-Just like any other `coscine.InputForm` you can `print()` a form to
-get a human-readable overview over the data.
-
-### Delete a project
-If we create projects with the Coscine Python SDK we may also need to
-delete some projects.
-```python
-import coscine
-
-client = coscine.Client("My Coscine API Token")
-project = client.project("Some Project I don't like")
-try:
- project.delete()
-except PermissionError:
- print("We are not authorized to delete that project. :(")
-```
-Be aware though, that this function may fail due to unsufficient privileges.
-Only project owners can delete a project - members can not!
-The call may yield an error, which we should catch and handle.
-
-### Download a project
-Downloading a project and all of the resources inside of that project
-is a piece of cake with the `download()` method. It accepts an optional
-path where it should save the project to. If no path is specified, the
-current local directory is chosen as path.
-```python
-import coscine
-
-client = coscine.Client("My Coscine API Token")
-project = client.project("Fancy project - please don't steal")
-project.download(path="./") # This is my project now huehehe
-```
-
-### Member management
-Coscine Projects contain Project Members. We can get a list of the members
-of a project by calling the `members()` method.
-```python
-import coscine
-
-client = coscine.Client("My Coscine API Token")
-project = client.project("My Project with Members")
-members = project.members()
-for member in members:
- print(member.name)
- print(member.email)
-
-# Set project role for a member
-member.set_role("Owner")
-
-# Remove a member from the project
-member.remove()
-```
-Inviting new members:
-```python
-EMAIL_ADDRESS: str = "john@example.com"
-ROLE: str = "Member" # or "Owner"
-project.invite(EMAIL_ADDRESS, ROLE)
-```
-
-## ... With Resources
-In Coscine, Resources store all of your data and metadata. As such they
-represent a key data structure, which you most certainly will
-interact a lot with.
-
-### The Resource object
-The Coscine Python SDK abstract Coscine resources with a
-custom Resource datatype. An instance of type `coscine.Resource`
-has the following properties:
-
-|Property | Type | Description |
-|-----------------------|--------|-------------|
-|id|`str` |Unqiue Resource identifier|
-|pid|`str` |Persistent identifier of the Resource|
-|name|`str` |Full Resource Name|
-|display_name|`str` |Resource Display Name|
-|license|`str` |Name of the license for the data contained in the resource|
-|type|`str` |Resource Type, e. g. "rdss3rwth"|
-|disciplines|`list[str]` |List of involved scientific disciplines|
-|profile|`str` |Name of the Application Profile used in the Resource|
-|archived|`bool` |Indicating whether the resource is archived (read-only)|
-|creator|`str` |Resource Creator GUID|
-
-Properties can be accessed via dot notation
-i. e. `print(resource.name)`.
-
-### Getting a list of resources
-Analoguous to getting a list of projects, you can get a list
-of resoures. The only difference is, that the `resources()`
-method is part of the project object and does only query
-resources contained within that project. Just like with
-projects, you can filter by certain resource properties.
-
-```python
-project = client.project("Some project")
-resources = project.resources()
-for resource in resources:
- print(resource.name)
-```
-
-If we know which resource we require beforehand, we can just
-query it by its name. We now get a single object of type
-Resource, instead of a list:
-```python
-resource = project.resource("ResourceNameS3")
-print(resource)
-```
-
-### Creating a resource
-Once again we are using InputForms to set metadata of a Coscine object.
-```python
-form = client.resource_form()
-form["Resource Type"] = "rdsrwth"
-form["Resource Size"] = 31
-form["Resource Name"] = "My Cool Resource"
-form["Display Name"] = "CoolResource"
-form["Resource Description"] = "Testing Coscine Client Resources"
-form["Discipline"] = ["Computer Science 409"]
-form["Application Profile"] = "radar"
-form["Visibility"] = "Project Members"
-project.create_resource(form)
-```
-
-### Deleting a resource
-```python
-try:
- resource.delete()
-except PermissionError:
- print("Not authorized (not a project owner).")
-```
-
-### Downloading a resource
-Downloading a resource and all of the data contained within the resource is just as simple as downloading a project. In fact internally `project.download()` just calls `Resource.download()` for all resources contained within the project.
-
-```python
-resource.download(path="./")
-```
-
-### Resource Quota
-You can fetch the used up quota of a resource as an integer indicating the size used in Bytes.
-
-```python
-quota = resource.quota()
-print(quota)
-```
-
-### Resource Application profile
-An application profile specifies a template for metadata. There may be times where you need to interact with that profile. To get the application profile of a resource you simple call the `application_profile()` method.
-```python
-# Print the application profile used by the resource
-profile = resource.application_profile()
-print(profile)
-```
-
-### Accessing S3 credentials
-RDS-S3 resources can be directly accessed via an S3-client. Direct connections require S3 credentials, which s3 resource instances happily provide to us. Only works for s3 resources.
-
-```python
-# Keys with write privileges, substitute for key_read for read-only privileges
-access_key: str = resource.s3.write_access_key
-secret_key: str = resource.s3.write_secret_key
-endpoint: str = resource.s3.endpoint
-bucket: str = resource.s3.bucket
-```
-
-### Interfacing with S3 natively
-The s3 instance of a resource provides some additional functionality when
-the `boto3` package is installed, to enable users to directly
-interface with S3.
-
-You can create folders (though S3 only mimicks folders):
-```python
-resource.s3.mkdir("directory/")
-```
-
-And upload files without metadata and size restrictions:
-```python
-resource.s3.upload("root/Testfile.txt", "./myfile.txt")
-```
-
-### Resource to RDF graph
-Coscine Resources contain FileObjects and Metadata. We can generate
-an RDF graph from the metadata of files in a resource. With such
-a graph a whole lot of new possibilities open up for interacting
-with the data. We can run SPARQL queries on the triples stored
-in the graph, serialize it into many different formats such as
-XML, CSV, Turtle, [...] and in general use our data in line
-with the concept of Semantic Web.
-
-#### Generating an RDF graph from a resource
-The Coscine Python SDK internally uses rdflib for working with RDF
-graphs. The `Resource.graph()` method returns an instance of type
-`class rdflib.Graph`. We can then interact with that graph just
-like with any other rdflib graph.
-Creating a graph for a lot of metadata can be a time consuming task.
-If the verbose setting of the client is enabled, a progress bar
-is shown in the command line to indicate the progress.
-Anything over 1000 files will take a noticable amount
-of time (seconds). Luckily, once generated, RDF graphs can be
-stored on the harddisk and reloaded later on with minimal overhead.
-Only files with metadata are reflected in the RDF graph, other
-files or folders are ignored, as they do not contribute relevant
-information to the graph.
-```python
-resource = client.project("My Project").resource("My Resource")
-graph = resource.graph()
-```
-
-#### Combining graphs of different resources
-The RDF graph returned created from a resource can be merged with
-RDF graphs of other resources, making it possible to perform
-multi-resource SPARQL queries on your data and to visualize the
-data relationship between resources. Intuitively it makes sense
-that all resources should roughly have equivalent application
-profiles and ontologies. It is not relevant whether the resources
-lay within the same project.
-```python
-import rdflib
-g1 = client.project("My Project").resource("My Resource 1").graph()
-g2 = client.project("My Project").resource("My Resource 2").graph()
-g3 = client.project("Different Project").resource("Resource").graph()
-master_graph = g1 + g2 + g3
-```
-
-#### Serializing an RDF graph
-RDF graphs can be serialized into various formats. For example
-to XML:
-```python
-graph = client.project("P").resource("R").graph()
-# Write the graph in XML format to 'file.xml'
-graph.serialize("file.xml", format="xml")
-# Our metadata is now accessible as XML in 'file.xml'!
-```
-
-#### Printing an RDF graph
-The Coscine Python SDK ships with a small utility function to
-serialize an RDF graph into dot format. Dot format can be used by
-tools like GraphViz to generate nice looking figures, indicating
-the relations between your data.
-The following snippet demonstrates how to create such a dotfile...
-```python
-resource = client.project("My Project").resource("My Resource")
-graph = resource.graph()
-with open("dotfile.dot", "wt", encoding="utf-8") as fd:
- coscine.utils.rdf_to_dot(graph, fd)
-```
-...which can subsequently be converted to an image with graphviz:
-```
-Graphviz/bin/dot -Tsvg dotfile.dot > image.svg
-Graphviz/bin/fdp -Tsvg dotfile.dot > image2.svg
-```
-Graphviz offers multiple rendering engines (e.g. 'dot', 'fdp', ...),
-each producing slightly different output.
-It is worth checking them all out to figure out which suits
-your data best.
-
-## ... With File Objects
-Files are stored inside of resources. However Resources do not necessarily contain files. It depends on the resource type. RDS and RDS-S3 contain files, but Linked Data resources contain references to files. Therefore we cannot just talk about files, but have to use a more abstract term such as 'object'. Objects represent files and file-like instances in Coscine.
-Nonetheless the methods of interacting with files and file-like objects is always the same - just don't expect file contents for objects of linked data resources, as those merely contain links or whatever has been specified as their content.
-
-### The FileObject Object
-The Coscine Python SDK abstract Coscine files with a
-custom FileObject datatype. An instance of type `coscine.FileObject`
-has the following properties:
-
-|Property | Type | Description |
-|-----------------------|--------|-------------|
-|name|`str` |Full Resource Name|
-|size|`int` |Filesize in bytes|
-|type|`str` |Type of FileObject, i. e. "file" or "folder"|
-|modified|`datetime` |Last time the FileObject was modified|
-|created|`datetime` |Time the FileObject was created|
-|provider|`str` |Resource type|
-|filetype|`str` |FileObject file type, e. g. '.png'|
-|path|`str` |Full path including folders (in case of S3 resources)|
-|is_folder|`bool` |Indicates whether the FileObject is a folder|
-|has_metadata|`bool` |Indicates whether the FileObject has metadata|
-
-Properties can be accessed via dot notation
-i. e. `print(file.name)`.
-
-### Getting a list of files and folders
-```python
-files = resource.objects()
-for file in files:
- print(file.name)
-
-# Files inside Folder/
-files2 = resource.objects("Folder/")
-```
-
-```python
-file = resource.object("Messung (1).bin")
-print(file.name)
-data = file.content()
-```
-
-### Getting all objects contained in a resource
-The methods for files and folders only work on one level
-of a directory hierarchy. To get all files and folders
-regardless of hierarchy, we can use the `content()s` method:
-```python
-# No directory hierarchy anymore, all files on a flat level
-all_files = resource.contents()
-```
-
-### Downloading a file
-```python
-file = resource.object("Messung (1).bin")
-path = "./"
-file.download(path)
-```
-
-### Uploading a file
-```python
-metadata = resource.metadata_form()
-
-metadata["Title"] = "..."
-# fill in fields
-
-filename = "messung.bin" # filename as it should appear in Coscine
-path = "./data/messung.bin" # path to file on harddrive
-resource.upload(filename, path, metadata)
-```
-
-### Deleting a file
-```python
-file = resource.object("Messung (1).bin")
-file.delete()
-# The file object is still valid until garbage collected.
-# The file on the Coscine server has already been removed though.
-```
-
-### Updating file metadata
-We can interact with metadata using a MetadataForm.
-```python
-form = file.form()
-print(form)
-```
-
-The form fields change depending on the selected application profile for the resource.
-```python
-form["Title"] = "My Title"
-
-# Update metadata
-file.update(form)
-```
diff --git a/src/tests/data/testfile.txt b/src/tests/data/testfile.txt
deleted file mode 100644
index c57eff55ebc0c54973903af5f72bac72762cf4f4..0000000000000000000000000000000000000000
--- a/src/tests/data/testfile.txt
+++ /dev/null
@@ -1 +0,0 @@
-Hello World!
\ No newline at end of file
diff --git a/src/tests/test.py b/src/tests/test.py
deleted file mode 100644
index 9c3feaa18eb9653911aa709b55ca9c826c375af3..0000000000000000000000000000000000000000
--- a/src/tests/test.py
+++ /dev/null
@@ -1,220 +0,0 @@
-###############################################################################
-# Coscine Python SDK
-# Copyright (c) 2018-2023 RWTH Aachen University
-# Licensed under the terms of the MIT License
-###############################################################################
-# Coscine, short for Collaborative Scientific Integration Environment, is
-# a platform for research data management (RDM).
-# For more information on Coscine visit https://www.coscine.de/.
-#
-# Please note that this python module is open source software primarily
-# developed and maintained by the scientific community. It is not
-# an official service that RWTH Aachen provides support for.
-###############################################################################
-
-###############################################################################
-# File description
-###############################################################################
-
-"""
-This file contains some tests to make sure that everything in `src/coscine`
-works as intended.
-For testing we are using the python unittest test suite, however due to
-many of our functions leading to (expected) side-effects, we "abuse"
-the unittest package a little bit by not sticking to general unit testing
-guidelines. That means we are not mocking requests or anything like that -
-everything is performed on the live system.
-"""
-
-###############################################################################
-# Dependencies
-###############################################################################
-
-from datetime import datetime
-import unittest
-import os
-import os.path
-import sys
-import logging
-sys.path.append("src")
-import coscine
-
-###############################################################################
-# Configuration
-###############################################################################
-
-# Set via environment variables
-# -> Makes it easy to set token in gitlab ci pipeline
-TOKEN: str = os.getenv("COSCINE_API_TOKEN")
-
-# Master project display name
-# -> This project must already be present and have sufficient quota
-# for all sorts of resource types
-MASTER_PROJECT_NAME: str = "Solaris"
-
-# Path to the tests directory
-BASEPATH: str = os.path.dirname(os.path.realpath(__file__))
-
-# Set up logger
-logging.basicConfig(level=logging.INFO)
-
-###############################################################################
-# Class definitions / Functions / Scripts
-###############################################################################
-
-class CoscineUnittest(unittest.TestCase):
- client: coscine.Client
- master_project: coscine.Project
-
-###############################################################################
-
- @classmethod
- def setUpClass(cls) -> None:
- cls.client = coscine.Client(TOKEN)
- cls.master_project = cls.client.project(MASTER_PROJECT_NAME)
- return super().setUpClass()
-
-###############################################################################
-
- def test_switch_language(self) -> None:
- self.client.settings.language = "de"
- self.client.settings.language = "en"
- with self.assertRaises(ValueError):
- self.client.settings.language = "es"
-
-###############################################################################
-
- def test_project_form_fill(self) -> None:
- data: dict = {
- "Project Name": "Test Name",
- "Display Name": "Test Display Name",
- "Project Description": "Test Description",
- "Principal Investigators": "Test Principal Investigator",
- "Project Start": datetime.now(),
- "Project End": datetime.now(),
- "Discipline": ["Computer Science 409"],
- "Participating Organizations": ["https://ror.org/04xfq0f34"],
- "Project Keywords": "Test Keywords",
- "Metadata Visibility": "Project Members",
- "Grant ID": "Test Grant ID"
- }
- form = self.client.project_form(data)
- self.assertIsInstance(form, coscine.ProjectForm)
- self.assertIsInstance(form.generate(), dict)
-
-###############################################################################
-
- def test_resource_form_fill(self) -> None:
- data: dict = {
- "Resource Type": "rdss3rwth",
- "Resource Size": 16,
- "Resource Name": "Test Name",
- "Display Name": "Test Display Name",
- "Resource Description": "Test Description",
- "Discipline": ["Computer Science 409", "Mathematics 312"],
- "Resource Keywords": "Test Keywords",
- "Metadata Visibility": "Project Members",
- "License": "MIT License",
- "Internal Rules for Reuse": "Test Rules",
- "Application Profile": "engmeta"
- }
- form = self.client.resource_form(data)
- self.assertIsInstance(form, coscine.ResourceForm)
- self.assertIsInstance(form.generate(), dict)
-
-###############################################################################
-
- def test_project_form_parse(self) -> None:
- form = self.master_project.form()
- self.assertIsInstance(form, coscine.ProjectForm)
-
-###############################################################################
-
- def test_list_project_members(self) -> None:
- # There is always at least one project member per project
- for member in self.master_project.members():
- self.assertIsInstance(member, coscine.ProjectMember)
-
-###############################################################################
-
-###############################################################################
-
- def test_subproject(self) -> None:
- projectForm = self.client.project_form()
- self.assertIsInstance(projectForm, coscine.ProjectForm)
- projectForm["Project Name"] = "coscine-sdk-test-project"
- projectForm["Display Name"] = "sdk-test-project"
- projectForm["Project Description"] = (
- "A test project for testing the Coscine Python SDK"
- )
- projectForm["Principal Investigators"] = "Lt. Cmndr. Research Data"
- projectForm["Project Start"] = datetime.now()
- projectForm["Project End"] = datetime.now()
- projectForm["Discipline"] = "Computer Science 409"
- projectForm["Participating Organizations"] = "https://ror.org/04xfq0f34"
- projectForm["Metadata Visibility"] = "Project Members"
- subproject = self.master_project.create_subproject(projectForm)
- self.assertIsInstance(subproject, coscine.Project)
-
- # Edit project metadata
- project = self.client.project(MASTER_PROJECT_NAME)\
- .subproject("sdk-test-project")
- projectForm = project.form()
- projectForm["Display Name"] = "edited-test-project"
- project.update(projectForm)
-
- # Clean up
- subproject.delete()
-
-###############################################################################
-
- def test_s3_resource(self) -> None:
- # Create an S3-resource
- resourceForm = self.client.resource_form()
- self.assertIsInstance(resourceForm, coscine.ResourceForm)
- resourceForm["Resource Type"] = "rdss3rwth"
- resourceForm["Resource Size"] = 1
- resourceForm["Resource Name"] = "test-resource-s3"
- resourceForm["Display Name"] = "test-resource-s3"
- resourceForm["Resource Description"] = "test-resource-s3"
- resourceForm["Discipline"] = ["Computer Science 409", "Mathematics 312"]
- resourceForm["Metadata Visibility"] = "Project Members"
- resourceForm["Application Profile"] = "radar"
- s3Resource = self.master_project.create_resource(resourceForm)
- self.assertIsInstance(s3Resource, coscine.Resource)
-
- # Create folders and files inside of the S3 resource
- metadataForm = s3Resource.metadata_form()
- metadataForm["Title"] = "Hello World!"
- metadataForm["Creator"] = "John Doe"
- metadataForm["Production Date"] = datetime.now()
- metadataForm["Resource"] = "Dataset"
- metadataForm["Subject Area"] = "History"
- s3Resource.upload("root/child/leaf/testfile.txt",
- os.path.join(BASEPATH, "data/testfile.txt"), metadataForm)
- metadataForm["Title"] = "Flohzirkus"
- s3Resource.upload("Lumpi.JPG", os.path.join(BASEPATH,
- "data/testfile.txt"), metadataForm.generate())
-
- # Edit resource metadata
- resource = self.master_project.resource("test-resource-s3")
- print(resource)
- resourceForm = resource.form()
- resourceForm["Display Name"] = "edited-resource-name"
- print(resource.update(resourceForm))
-
- # Edit file metadata
- object = resource.object("root/child/leaf/testfile.txt")
- metadataForm = object.form()
- metadataForm["Title"] = "John Doe β₯ Jane"
- object.update(metadataForm)
-
- # Clean up
- s3Resource.delete()
-
-###############################################################################
-
-if __name__ == "__main__":
- unittest.main()
-
-###############################################################################
diff --git a/tests/test.py b/tests/test.py
new file mode 100644
index 0000000000000000000000000000000000000000..a3d6a4c24b9f6ba2730e2eb9f4e269d326d7e38d
--- /dev/null
+++ b/tests/test.py
@@ -0,0 +1,282 @@
+###############################################################################
+# Coscine Python SDK
+# Copyright (c) 2020-2023 RWTH Aachen University
+# Licensed under the terms of the MIT License
+# For more information on Coscine visit https://www.coscine.de/.
+###############################################################################
+
+"""
+This executable module provides tests that verify the Coscine Python SDK
+for correctness. It is mathematically impossible to test with absolute
+certainty, but the results should be "good enough".
+The tests can be run on the live system or on a development instance.
+One nice side-effect is that the tester can also use these tests to verify
+that the Coscine instance and API is working correctly.
+"""
+
+###############################################################################
+
+import sys
+sys.path.append("../src")
+from typing import Tuple
+from os import getenv
+from datetime import datetime
+from concurrent.futures import ThreadPoolExecutor, wait
+from time import sleep
+import coscine
+import logging
+import traceback
+
+###############################################################################
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger("coscine-python-sdk-test")
+
+###############################################################################
+
+exceptions: list[str] = []
+profiles: list[str] = []
+good = 0
+bad = 0
+errid = 1
+
+MARKUP: str = """
+# Coscine Python SDK Test Report
+This report was automatically generated by the Coscine Python SDK
+integration test.
+
+## Application Profiles
+Showcases whether the Coscine Python SDK is able to interface with
+the listed application profile. All application profiles marked with
+β have been tested and considered working, while all profiles marked
+with β may bring along issues. That does not mean that those will
+not work with the Python SDK, just that functionality may be limited
+or that the application profile itself is broken. The integration tests
+are very strict on whether a profile is considered working.
+
+### Statistics
+
+| Property | Value |
+|----------|---------|
+| Total | {total} |
+| β | {good} |
+| β | {bad} |
+
+### Test Results
+
+|Name | Link | Supported| EID |
+|-----|------|----------|-----|
+{profiles}
+
+## Exceptions
+Lists the exceptions that were raised during testing.
+
+{exceptions}
+
+"""
+
+###############################################################################
+
+def report_result(
+ profile: coscine.ApplicationProfile,
+ works: bool,
+ err: Exception | None = None
+) -> None:
+ """
+ """
+ global good
+ global bad
+ global errid
+ if works:
+ good += 1
+ eid = " "
+ else:
+ bad += 1
+ message = "".join(traceback.format_exception(err))
+ logger.error(message)
+ eid = f"{errid}"
+ errid += 1
+ exceptions.append(
+ f"### Exception EID:{eid}: \n"
+ f"{message}"
+ )
+ profiles.append(
+ f"{profile.name} | "
+ f"[{profile.uri}]({profile.uri}) | "
+ f"{'β' if works else 'β'} | "
+ f"{eid} | "
+ )
+
+###############################################################################
+
+def print_test_report(path: str) -> None:
+ """
+ Prints the test report as html.
+ """
+ data = MARKUP.format(
+ total=len(profiles),
+ good=good,
+ bad=bad,
+ profiles="\n".join(profiles),
+ exceptions="\n".join(exceptions)
+ )
+ with open(path, "w", encoding="utf-8") as fp:
+ fp.write(data)
+
+###############################################################################
+
+def setup_test_environment() -> Tuple[coscine.ApiClient, coscine.Project]:
+ """
+ This function creates the test environment.
+ """
+ logger.info("Initializing Api Client...")
+ token = getenv("COSCINE_API_TOKEN")
+ if not token:
+ raise ValueError(
+ "No Coscine API Token specified! "
+ "Create a new environment variable with "
+ "the name COSCINE_API_TOKEN!"
+ )
+ client = coscine.ApiClient(token, verbose=False)
+ logger.info("Creating root test project...")
+ try:
+ project = client.project("coscine-python-sdk-test")
+ project.delete()
+ except coscine.CoscineException:
+ pass
+ project = client.create_project(
+ "coscine-python-sdk-test",
+ "coscine-python-sdk-test",
+ "This project was automatically generated by the "
+ "Coscine Python SDKs test script. If you are still able "
+ "to see this project you may go ahead and delete it.",
+ datetime.now().date(),
+ datetime.now().date(),
+ "Test User",
+ [ client.discipline("Computer Science 409") ],
+ [ client.organization("https://ror.org/04xfq0f34") ],
+ # -> RWTH Aachen University
+ client.visibility("Project Members")
+ )
+ return client, project
+
+###############################################################################
+
+def tear_down_test_environment(project: coscine.Project) -> None:
+ """
+ This function clears the test environment that has been created
+ with setup_test_environment().
+ """
+ project.delete()
+
+###############################################################################
+
+def test_member_management(
+ client: coscine.ApiClient,
+ project: coscine.Project
+) -> None:
+ """
+ This function tests member management by adding a bunch of test
+ users to the given project.
+ """
+ logger.info("Testing member management...")
+ for user in client.users("coscine@protonmail.com")[:3]:
+ if user.id != client.self().id:
+ project.add_member(user, client.role("Member"))
+ logger.info(f"-> Successfully added {user.display_name} to {project.name}!")
+ for member in project.members():
+ if member.user.id != client.self().id:
+ project.remove_member(member)
+
+###############################################################################
+
+def test_application_profiles(
+ client: coscine.ApiClient,
+ project: coscine.Project
+) -> None:
+ """
+ Iterates through the list of available application profiles in Coscine
+ and creates a resource for each. Within this resource, metadata upload
+ is tested.
+ """
+ logger.info("Testing all application profiles...")
+ application_profiles = client.application_profiles()
+ length = len(application_profiles)
+ futures = []
+ with ThreadPoolExecutor(10) as executor:
+ for index, profile in enumerate(application_profiles):
+ futures.append(executor.submit(
+ test_application_profile,
+ client,
+ project,
+ profile,
+ index,
+ length
+ ))
+ wait(futures)
+ for future in futures:
+ future.result()
+
+###############################################################################
+
+def test_application_profile(
+ client: coscine.ApiClient,
+ project: coscine.Project,
+ profile: coscine.ApplicationProfile,
+ index: int,
+ length: int
+) -> None:
+ """
+ Tests an application profile.
+ """
+ logger.info(f"Testing profile [{index}/{length}] -> {profile.name}")
+ resource = project.create_resource(
+ profile.name,
+ profile.name[:24],
+ profile.description or "No description set.",
+ client.license("MIT License"),
+ client.visibility("Project Members"),
+ [ client.discipline("Computer Science 409") ],
+ client.resource_type("linked"),
+ 0,
+ profile.uri
+ )
+ is_working: bool = True
+ repeats: int = 0
+ while True:
+ try:
+ form = resource.metadata_form()
+ form.test()
+ str(form)
+ form.generate("testfile.txt")
+ form.validate()
+ resource.upload("testfile.txt", "http://example.org", form)
+ report_result(profile, is_working)
+ break
+ except coscine.ConnectionError as err:
+ if repeats > 15:
+ break
+ logger.info("Timeout - retrying...")
+ repeats += 1
+ sleep(10.0)
+ except Exception as err:
+ is_working = False
+ report_result(profile, is_working, err)
+ break
+ resource.delete()
+
+###############################################################################
+
+def main():
+ client, project = setup_test_environment()
+ test_member_management(client, project)
+ test_application_profiles(client, project)
+ tear_down_test_environment(project)
+ print_test_report("report.md")
+
+###############################################################################
+
+if __name__ == "__main__":
+ main()
+
+###############################################################################