From 159f4b2cafc3f9816fb5742ed9f8b7c63675a3a6 Mon Sep 17 00:00:00 2001 From: Lennard Strohmeyer <lennard.strohmeyer@digitallearning.gmbh> Date: Thu, 27 Mar 2025 16:47:43 +0100 Subject: [PATCH 01/12] greatly improved test coverage (1) --- README.md | 2 + src/__init__.py | 0 .../tests/tests_provider_endpoints.py | 190 ++++++++++ src/consents/tests/tests_third_party.py | 269 ++++++++++++++ src/consents/tests/tests_user_endpoints.py | 327 ++++++++++++++++++ src/consents/urls.py | 4 +- src/consents/views.py | 14 +- src/data_disclosure/tests.py | 3 - src/data_disclosure/tests/__init__.py | 0 .../tests/tests_data_disclosure.py | 251 ++++++++++++++ src/data_removal/tests.py | 3 - src/data_removal/tests/__init__.py | 0 src/data_removal/tests/tests_data_removal.py | 130 +++++++ src/data_removal/views.py | 2 +- src/frontend/src/app/data-removal.service.ts | 2 +- src/providers/models.py | 2 + src/providers/tests/__init__.py | 0 src/providers/tests/tests_analytics_tokens.py | 241 +++++++++++++ .../tests_provider_management.py} | 2 +- src/providers/views.py | 6 +- src/settings/tests/__init__.py | 0 src/settings/tests/tests_privacy_policy.py | 136 ++++++++ 22 files changed, 1564 insertions(+), 20 deletions(-) create mode 100644 src/__init__.py create mode 100644 src/consents/tests/tests_provider_endpoints.py create mode 100644 src/consents/tests/tests_user_endpoints.py delete mode 100644 src/data_disclosure/tests.py create mode 100644 src/data_disclosure/tests/__init__.py create mode 100644 src/data_disclosure/tests/tests_data_disclosure.py delete mode 100644 src/data_removal/tests.py create mode 100644 src/data_removal/tests/__init__.py create mode 100644 src/data_removal/tests/tests_data_removal.py create mode 100644 src/providers/tests/__init__.py create mode 100644 src/providers/tests/tests_analytics_tokens.py rename src/providers/{tests.py => tests/tests_provider_management.py} (98%) create mode 100644 src/settings/tests/__init__.py create mode 100644 src/settings/tests/tests_privacy_policy.py diff --git a/README.md b/README.md index 47b62fd..7cadda7 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ + + # POLARIS - Rights Engine / Consent Engine This is the repository contains the source code of the rights engine of the Polaris project. The Consent Engine is a software solution designed to provide a secure and efficient way for obtaining and managing user consent. The project is divided into two main components: frontend and backend. The frontend component is responsible for providing a user-friendly interface for obtaining consent, while the backend component is responsible for storing and managing user consent data. diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/consents/tests/tests_provider_endpoints.py b/src/consents/tests/tests_provider_endpoints.py new file mode 100644 index 0000000..adf6a9f --- /dev/null +++ b/src/consents/tests/tests_provider_endpoints.py @@ -0,0 +1,190 @@ +import json +import os +from io import StringIO + +from rest_framework.test import APIClient +from rolepermissions.roles import assign_role +from django.core.management import call_command + +from consents.tests.tests_consent_operations import BaseTestCase +from providers.models import ProviderAuthorization, Provider, ProviderVerbGroup, ProviderSchema +from users.models import CustomUser + + +class TestVerbGroups(BaseTestCase): + verb_groups = [ + { + "provider_id": 1, + "id": "default_group", + "label": "Default group", + "description": "default", + "showVerbDetails": True, + "purposeOfCollection": "Lorem Ipsum", + "requiresConsent": True, + "verbs": [ + {"id": "http://h5p.example.com/expapi/verbs/experienced"}, + {"id": "http://h5p.example.com/expapi/verbs/attempted"}, + {"id": "http://h5p.example.com/expapi/verbs/interacted"}, + {"id": "http://h5p.example.com/expapi/verbs/answered"} + ] + }, + ] + + def setUp(self): + call_command('check_and_apply_migrations') + normal_user = CustomUser.objects.create_user( + self.test_user_email, self.test_user_password + ) + provider_user = CustomUser.objects.create_user( + self.test_provider_email, self.test_provider_password + ) + + assign_role(normal_user, "user") + assign_role(provider_user, "provider_manager") + + response = self.client.post( + "/api/v1/auth/token", + {"email": self.test_user_email, "password": self.test_user_password}, + ) # obtain token + self.assertEqual(response.status_code, 200) + self.assertTrue("access" in response.data) + user_token = response.data["access"] + + response = self.client.post( + "/api/v1/auth/token", + { + "email": self.test_provider_email, + "password": self.test_provider_password, + }, + ) # obtain token + self.assertEqual(response.status_code, 200) + self.assertTrue("access" in response.data) + provider_token = response.data["access"] + + self.user_client = APIClient() + self.user_client.credentials(HTTP_AUTHORIZATION="Bearer " + user_token) + + self.provider_client = APIClient() + self.provider_client.credentials(HTTP_AUTHORIZATION="Bearer " + provider_token) + + # Create provider schema + with StringIO(json.dumps(self.h5p_provider_schema)) as fp: + response = self.provider_client.put( + "/api/v1/consents/provider/create", + {"provider-schema": fp}, + format="multipart", + ) + self.assertEqual(response.status_code, 201) + + # Get application token for created provider + self.application_token = ProviderAuthorization.objects.get( + provider__name="H5P" + ).key + self.third_party_client = APIClient() + self.third_party_client.credentials( + HTTP_AUTHORIZATION="Basic " + self.application_token + ) + + # create some verb groups + for group in self.verb_groups: + response = self.provider_client.post( + "/api/v1/consents/provider/" + str(Provider.objects.latest('id').id) + "/create-verb-group", + group, + format="json", + ) + self.assertEqual(response.status_code, 200) + + + def test_get_verb_groups(self): + """ + Ensure correct verb groups are returned for a given provider. + """ + provider_id = Provider.objects.latest('id').id + # create a verb group + group = {"id": "default_group", "label": "Group 2", + "description": "Lorem ipsum", "showVerbDetails": True, "purposeOfCollection": "Lorem Ipsum", + "requiresConsent": True, "verbs": [ + {"id": "http://h5p.example.com/expapi/verbs/experienced"}, + {"id": "http://h5p.example.com/expapi/verbs/attempted"}, + {"id": "http://h5p.example.com/expapi/verbs/interacted"}, + {"id": "http://h5p.example.com/expapi/verbs/answered"}, + ]} + response = self.provider_client.post( + f"/api/v1/consents/provider/{provider_id}/create-verb-group", + group, + format="json", + ) + self.assertEqual(response.status_code, 200) + + # test permissions + response = self.user_client.get( + f"/api/v1/consents/provider/{provider_id}/verb-groups" + ) + self.assertEqual(response.status_code, 403) + + # check if group exists + response = self.provider_client.get( + f"/api/v1/consents/provider/{provider_id}/verb-groups" + ) + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertTrue(isinstance(data, list)) + self.assertEqual(len(data), 2) # another group is created during setup + self.assertEqual(data[-1]["purposeOfCollection"], "Lorem Ipsum") + self.assertTrue(isinstance(data[-1]["verbs"], list)) + self.assertEqual(data[-1]["verbs"][0]["id"], "http://h5p.example.com/expapi/verbs/experienced") + + def test_create_invalid_verb_group(self): + provider_id = Provider.objects.latest('id').id + group = {"id": "default_group", "label": "Group 2", + "description": "Lorem ipsum", "showVerbDetails": True, "purposeOfCollection": "Lorem Ipsum", + "requiresConsent": True, "verbs": [ + {"id": "http://h5p.example.com/expapi/verbs/experienced"}, + {"id": "http://h5p.example.com/expapi/verbs/atztempted"}, # mistyped on purpose + {"id": "http://h5p.example.com/expapi/verbs/interacted"}, + {"id": "http://h5p.example.com/expapi/verbs/answered"}, + ]} + response = self.provider_client.post( + f"/api/v1/consents/provider/{provider_id}/create-verb-group", + group, + format="json", + ) + self.assertEqual(response.status_code, 400) + + + def test_update_existing_verb_group(self): + provider_id = Provider.objects.latest('id').id + group = {"id": "default_group", "label": "Group 2", + "description": "Lorem ipsum", "showVerbDetails": True, "purposeOfCollection": "Lorem Ipsum", + "requiresConsent": True, "verbs": [ + {"id": "http://h5p.example.com/expapi/verbs/experienced"}, + {"id": "http://h5p.example.com/expapi/verbs/interacted"}, + {"id": "http://h5p.example.com/expapi/verbs/answered"}, + ]} + response = self.provider_client.post( + f"/api/v1/consents/provider/{provider_id}/create-verb-group", + group, + format="json", + ) + self.assertEqual(response.status_code, 200) + + group_id = ProviderVerbGroup.objects.latest('group_id').group_id # generated on server, has to be supplied for update + + group = {"id": group_id, "label": "Group 2", # id has to be the same and is generated when creating the group + "description": "Lorem ipsum!", "showVerbDetails": True, "purposeOfCollection": "Lorem Ipsum!", # changed + "requiresConsent": True, "verbs": [ + {"id": "http://h5p.example.com/expapi/verbs/experienced"}, + {"id": "http://h5p.example.com/expapi/verbs/attempted"}, # added + {"id": "http://h5p.example.com/expapi/verbs/interacted"}, + {"id": "http://h5p.example.com/expapi/verbs/answered"}, + ]} + response = self.provider_client.post( + f"/api/v1/consents/provider/{provider_id}/create-verb-group", + group, + format="json", + ) + self.assertEqual(response.status_code, 200) + + groups = ProviderVerbGroup.objects.filter(provider_id=provider_id).all() + + self.assertEqual(len(groups), 2) \ No newline at end of file diff --git a/src/consents/tests/tests_third_party.py b/src/consents/tests/tests_third_party.py index a1aae6a..547841e 100644 --- a/src/consents/tests/tests_third_party.py +++ b/src/consents/tests/tests_third_party.py @@ -303,3 +303,272 @@ class TestThirdPartyUserConsentUpdate(BaseTestCase): "/api/v1/consents/user/save/third-party", data=payload, format="json" ) self.assertEqual(response.status_code, 400) + + + + +class TestThirdPartyGetProviderStatus(BaseTestCase): + verb_groups = [ + { + "provider_id": 1, + "id": "default_group", + "label": "Default group", + "description": "default", + "showVerbDetails": True, + "purposeOfCollection": "Lorem Ipsum", + "requiresConsent": True, + "verbs": [ + {"id": "http://h5p.example.com/expapi/verbs/experienced"}, + {"id": "http://h5p.example.com/expapi/verbs/attempted"}, + {"id": "http://h5p.example.com/expapi/verbs/interacted"}, + {"id": "http://h5p.example.com/expapi/verbs/answered"} + ] + }, + ] + def setUp(self): + call_command('check_and_apply_migrations') + normal_user = CustomUser.objects.create_user( + self.test_user_email, self.test_user_password + ) + provider_user = CustomUser.objects.create_user( + self.test_provider_email, self.test_provider_password + ) + + assign_role(normal_user, "user") + assign_role(provider_user, "provider_manager") + + response = self.client.post( + "/api/v1/auth/token", + {"email": self.test_user_email, "password": self.test_user_password}, + ) # obtain token + self.assertEqual(response.status_code, 200) + self.assertTrue("access" in response.data) + user_token = response.data["access"] + + response = self.client.post( + "/api/v1/auth/token", + { + "email": self.test_provider_email, + "password": self.test_provider_password, + }, + ) # obtain token + self.assertEqual(response.status_code, 200) + self.assertTrue("access" in response.data) + provider_token = response.data["access"] + + self.user_client = APIClient() + self.user_client.credentials(HTTP_AUTHORIZATION="Bearer " + user_token) + + self.provider_client = APIClient() + self.provider_client.credentials(HTTP_AUTHORIZATION="Bearer " + provider_token) + + # Create provider schema + with StringIO(json.dumps(self.h5p_provider_schema)) as fp: + response = self.provider_client.put( + "/api/v1/consents/provider/create", + {"provider-schema": fp}, + format="multipart", + ) + self.assertEqual(response.status_code, 201) + + # Get application token for created provider + self.application_token = ProviderAuthorization.objects.get( + provider__name="H5P" + ).key + self.third_party_client = APIClient() + self.third_party_client.credentials( + HTTP_AUTHORIZATION="Basic " + self.application_token + ) + + # create some verb groups + for group in self.verb_groups: + response = self.provider_client.post( + "/api/v1/consents/provider/" + str(Provider.objects.latest('id').id) + "/create-verb-group", + group, + format="json", + ) + self.assertEqual(response.status_code, 200) + + def test_provider_status_401(self): + """ + Ensure requests including an invalid application token are rejected. + """ + response = self.user_client.get( + "/api/v1/consents/provider-status/third-party" + ) + self.assertEqual(response.status_code, 401) + + def test_provider_status(self): + response = self.third_party_client.get( + "/api/v1/consents/provider-status/third-party" + ) + self.assertEqual(response.status_code, 200) + + data = json.loads(response.content.decode()) + self.assertTrue(all(key in data.keys() for key in ["id", "name", "groups", "versions"])) + self.assertEqual(data["name"], "H5P") + +class TestThirdPartyUserManagement(BaseTestCase): + verb_groups = [ + { + "provider_id": 1, + "id": "default_group", + "label": "Default group", + "description": "default", + "showVerbDetails": True, + "purposeOfCollection": "Lorem Ipsum", + "requiresConsent": True, + "verbs": [ + {"id": "http://h5p.example.com/expapi/verbs/experienced"}, + {"id": "http://h5p.example.com/expapi/verbs/attempted"}, + {"id": "http://h5p.example.com/expapi/verbs/interacted"}, + {"id": "http://h5p.example.com/expapi/verbs/answered"} + ] + }, + ] + + def setUp(self): + call_command('check_and_apply_migrations') + normal_user = CustomUser.objects.create_user( + self.test_user_email, self.test_user_password + ) + provider_user = CustomUser.objects.create_user( + self.test_provider_email, self.test_provider_password + ) + + assign_role(normal_user, "user") + assign_role(provider_user, "provider_manager") + + response = self.client.post( + "/api/v1/auth/token", + {"email": self.test_user_email, "password": self.test_user_password}, + ) # obtain token + self.assertEqual(response.status_code, 200) + self.assertTrue("access" in response.data) + user_token = response.data["access"] + + response = self.client.post( + "/api/v1/auth/token", + { + "email": self.test_provider_email, + "password": self.test_provider_password, + }, + ) # obtain token + self.assertEqual(response.status_code, 200) + self.assertTrue("access" in response.data) + provider_token = response.data["access"] + + self.user_client = APIClient() + self.user_client.credentials(HTTP_AUTHORIZATION="Bearer " + user_token) + + self.provider_client = APIClient() + self.provider_client.credentials(HTTP_AUTHORIZATION="Bearer " + provider_token) + + # Create provider schema + with StringIO(json.dumps(self.h5p_provider_schema)) as fp: + response = self.provider_client.put( + "/api/v1/consents/provider/create", + {"provider-schema": fp}, + format="multipart", + ) + self.assertEqual(response.status_code, 201) + + # Get application token for created provider + self.application_token = ProviderAuthorization.objects.get( + provider__name="H5P" + ).key + self.third_party_client = APIClient() + self.third_party_client.credentials( + HTTP_AUTHORIZATION="Basic " + self.application_token + ) + + # create some verb groups + for group in self.verb_groups: + response = self.provider_client.post( + "/api/v1/consents/provider/" + str(Provider.objects.latest('id').id) + "/create-verb-group", + group, + format="json", + ) + self.assertEqual(response.status_code, 200) + + + def test_create_user(self): + # test permission check + response = self.user_client.post( + "/api/v1/consents/user/create", + {} + ) + self.assertEqual(response.status_code, 401) + + # check validation + response = self.third_party_client.post( + "/api/v1/consents/user/create", + {} + ) + self.assertEqual(response.status_code, 400) + + response = self.third_party_client.post( + "/api/v1/consents/user/create", + { + "email": "test@polaris.com", + "first_name": "Max", + "last_name": "Mustermann" + } + ) + self.assertEqual(response.status_code, 200) + + count_before = CustomUser.objects.count() + 0 + + # try to create with same email again + response = self.third_party_client.post( + "/api/v1/consents/user/create", + { + "email": "test@polaris.com", + "first_name": "Max", + "last_name": "Mustermann" + } + ) + self.assertEqual(response.status_code, 200) + # no new user should have been created + self.assertEqual(CustomUser.objects.count(), count_before) + + def test_create_user_via_connect_service(self): + # test permission check + response = self.user_client.post( + "/api/v1/consents/user/create-via-connect-service", + {} + ) + self.assertEqual(response.status_code, 401) + + # check validation + response = self.third_party_client.post( + "/api/v1/consents/user/create-via-connect-service", + {} + ) + self.assertEqual(response.status_code, 400) + + # create user + response = self.third_party_client.post( + "/api/v1/consents/user/create-via-connect-service", + { + "email": "test@polaris.com", + } + ) + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertEqual(data["message"], "user has been created") + + count_before = CustomUser.objects.count() + 0 + + # try to create with same email again + response = self.third_party_client.post( + "/api/v1/consents/user/create-via-connect-service", + { + "email": "test@polaris.com", + } + ) + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertEqual(data["message"], "user exists") + # no new user should have been created + self.assertEqual(CustomUser.objects.count(), count_before) \ No newline at end of file diff --git a/src/consents/tests/tests_user_endpoints.py b/src/consents/tests/tests_user_endpoints.py new file mode 100644 index 0000000..6c9f175 --- /dev/null +++ b/src/consents/tests/tests_user_endpoints.py @@ -0,0 +1,327 @@ +import json +import os +from io import StringIO + +from rest_framework.test import APIClient +from rolepermissions.roles import assign_role +from django.core.management import call_command + +from consents.tests.tests_consent_operations import BaseTestCase +from providers.models import ProviderAuthorization, Provider, ProviderVerbGroup, ProviderSchema +from users.models import CustomUser + +from consents.models import UserConsents + + +class TestUserEndpoints(BaseTestCase): + verb_groups = [ + { + "provider_id": 1, + "id": "default_group", + "label": "Default group", + "description": "default", + "showVerbDetails": True, + "purposeOfCollection": "Lorem Ipsum", + "requiresConsent": True, + "verbs": [ + {"id": "http://h5p.example.com/expapi/verbs/experienced"}, + {"id": "http://h5p.example.com/expapi/verbs/attempted"}, + {"id": "http://h5p.example.com/expapi/verbs/interacted"}, + {"id": "http://h5p.example.com/expapi/verbs/answered"} + ] + }, + ] + + def setUp(self): + call_command('check_and_apply_migrations') + normal_user = CustomUser.objects.create_user( + self.test_user_email, self.test_user_password + ) + provider_user = CustomUser.objects.create_user( + self.test_provider_email, self.test_provider_password + ) + + assign_role(normal_user, "user") + assign_role(provider_user, "provider_manager") + assign_role(provider_user, 'analyst') # needed to create analytics tokens later on + + response = self.client.post( + "/api/v1/auth/token", + {"email": self.test_user_email, "password": self.test_user_password}, + ) # obtain token + self.assertEqual(response.status_code, 200) + self.assertTrue("access" in response.data) + user_token = response.data["access"] + + response = self.client.post( + "/api/v1/auth/token", + { + "email": self.test_provider_email, + "password": self.test_provider_password, + }, + ) # obtain token + self.assertEqual(response.status_code, 200) + self.assertTrue("access" in response.data) + provider_token = response.data["access"] + + self.user_client = APIClient() + self.user_client.credentials(HTTP_AUTHORIZATION="Bearer " + user_token) + + self.provider_client = APIClient() + self.provider_client.credentials(HTTP_AUTHORIZATION="Bearer " + provider_token) + + # Create provider schema + with StringIO(json.dumps(self.h5p_provider_schema)) as fp: + response = self.provider_client.put( + "/api/v1/consents/provider/create", + {"provider-schema": fp}, + format="multipart", + ) + self.assertEqual(response.status_code, 201) + + # Get application token for created provider + self.application_token = ProviderAuthorization.objects.get( + provider__name="H5P" + ).key + self.third_party_client = APIClient() + self.third_party_client.credentials( + HTTP_AUTHORIZATION="Basic " + self.application_token + ) + + # create some verb groups + for group in self.verb_groups: + response = self.provider_client.post( + "/api/v1/consents/provider/" + str(Provider.objects.latest('id').id) + "/create-verb-group", + group, + format="json", + ) + self.assertEqual(response.status_code, 200) + + def test_get_consent_for_provider(self): + # Note: this route is currently not being used in the frontend since the consent history replaces most of the functionality + provider_id = Provider.objects.latest('id').id + + response = self.user_client.get( + f"/api/v1/consents/user/{provider_id}", + ) + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + + # right now we haven't declared consent for any groups + self.assertEqual(data["consent"], None) + + # create consent + group_id = ProviderVerbGroup.objects.latest('id').id # maybe todo: rework frontend to use the "group_id" attribute instead of primary keys + response = self.user_client.post( + "/api/v1/consents/user/save", {"groups": [group_id]}, format="json" + ) + self.assertEqual(response.status_code, 200) + + # now we would expect the consent to be present in the response + response = self.user_client.get( + f"/api/v1/consents/user/{provider_id}", + ) + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + + self.assertTrue(all(key in data.keys() for key in ["consent", "provider_schema", "paused_data_recording"])) + self.assertFalse(data["consent"] is None) + + def test_get_consent_history(self): + provider_id = Provider.objects.latest('id').id + + # consent history should be empty + response = self.user_client.get( + "/api/v1/consents/user/history", + ) + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertTrue("groups" in data.keys()) + self.assertEqual(len(data["groups"]), 0) + + # create consent + group_id = ProviderVerbGroup.objects.latest('id').id + response = self.user_client.post( + "/api/v1/consents/user/save", {"groups": [group_id]}, format="json" + ) + self.assertEqual(response.status_code, 200) + + response = self.user_client.get( + "/api/v1/consents/user/history", + ) + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertTrue("groups" in data.keys()) + self.assertEqual(len(data["groups"]), 1) + + # create another group + group = {"id": "default_group", "label": "Group 2", + "description": "Lorem ipsum", "showVerbDetails": True, "purposeOfCollection": "Lorem Ipsum", + "requiresConsent": True, "verbs": [ + {"id": "http://h5p.example.com/expapi/verbs/experienced"}, + {"id": "http://h5p.example.com/expapi/verbs/interacted"}, + {"id": "http://h5p.example.com/expapi/verbs/answered"}, + ]} + response = self.provider_client.post( + f"/api/v1/consents/provider/{provider_id}/create-verb-group", + group, + format="json", + ) + self.assertEqual(response.status_code, 200) + + # create consent for second group + group_id = ProviderVerbGroup.objects.latest('id').id + response = self.user_client.post( + "/api/v1/consents/user/save", {"groups": [group_id]}, format="json" + ) + self.assertEqual(response.status_code, 200) + + # check that both consents are present in history + response = self.user_client.get( + "/api/v1/consents/user/history", + ) + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertTrue("groups" in data.keys()) + self.assertEqual(len(data["groups"]), 2) + + def test_update_consent_group_active(self): + provider_id = Provider.objects.latest('id').id + + response = self.user_client.get( + f"/api/v1/consents/user/{provider_id}", + ) + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + + # right now we haven't declared consent for any groups + self.assertEqual(data["consent"], None) + + # create consent + group_id = ProviderVerbGroup.objects.latest('id').id + response = self.user_client.post( + "/api/v1/consents/user/save", {"groups": [group_id]}, format="json" + ) + self.assertEqual(response.status_code, 200) + + # check if incorrect request is rejected + response = self.user_client.post( + "/api/v1/consents/user/update-consent-group-active", {}, format="json" + ) + self.assertEqual(response.status_code, 400) + + # update group consent + response = self.user_client.post( + "/api/v1/consents/user/update-consent-group-active", {"id": group_id, "active": False}, format="json" + ) + self.assertEqual(response.status_code, 200) + # check if status correctly stored in DB + self.assertFalse(UserConsents.objects.last().active) + + def test_revoke_consent_group(self): + provider_id = Provider.objects.latest('id').id + + response = self.user_client.get( + f"/api/v1/consents/user/{provider_id}", + ) + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + + # right now we haven't declared consent for any groups + self.assertEqual(data["consent"], None) + + # create consent + group_id = ProviderVerbGroup.objects.latest('id').id + response = self.user_client.post( + "/api/v1/consents/user/save", {"groups": [group_id]}, format="json" + ) + self.assertEqual(response.status_code, 200) + + # check if incorrect request is rejected + response = self.user_client.post( + "/api/v1/consents/user/revoke-consent-group", {}, format="json" + ) + self.assertEqual(response.status_code, 400) + + # update group consent + response = self.user_client.post( + "/api/v1/consents/user/revoke-consent-group", {"id": group_id}, format="json" + ) + self.assertEqual(response.status_code, 200) + # check if consent correctly updated in DB + self.assertFalse(UserConsents.objects.last().consented) + + def test_get_analytics_tokens(self): + response = self.provider_client.get( + f"/api/v1/consents/user/analytics-tokens", + ) + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertEqual(len(data), 0) + + response = self.provider_client.post( + f"/api/v1/provider/analytics-tokens/create", + { + "name": "Test analytics token", + "description": "Test analytics token", + "expires": "12/31/2099", + "can_access": [] + } + ) + self.assertEqual(response.status_code, 201) + + response = self.provider_client.get( + f"/api/v1/consents/user/analytics-tokens", + ) + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertEqual(len(data), 1) + + def test_get_analytics_token_consent(self): + response = self.provider_client.get( + f"/api/v1/consents/user/analytics-tokens", + ) + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertEqual(len(data), 0) + + response = self.provider_client.post( + f"/api/v1/provider/analytics-tokens/create", + { + "name": "Test analytics token", + "description": "Test analytics token", + "expires": "12/31/2099", + "can_access": [] + } + ) + self.assertEqual(response.status_code, 201) + data = json.loads(response.content.decode()) + analyticTokenId = data["id"] + + response = self.user_client.post( + f"/api/v1/consents/user/analytics-tokens/consent", + { + "analyticTokenId": analyticTokenId + } + ) + self.assertEqual(response.status_code, 200) + # maybe todo: check if consents created correctly (?) + + def test_get_providers(self): + response = self.user_client.get( + f"/api/v1/consents/user/providers", + ) + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertEqual(len(data), 1) + + + def test_get_status(self): + response = self.user_client.get( + f"/api/v1/consents/user/status", + ) + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertTrue("result" in data.keys()) + self.assertTrue("1" in data["result"].keys()) + self.assertEqual(data["result"]["1"]["status"], "none") # since consent is coupled with groups, consent for a whole provider schema can not exist \ No newline at end of file diff --git a/src/consents/urls.py b/src/consents/urls.py index a1e4f71..fc0ea1c 100644 --- a/src/consents/urls.py +++ b/src/consents/urls.py @@ -6,8 +6,8 @@ urlpatterns = [ path('provider', views.GetProviderSchemasView.as_view()), path('provider-status/third-party', views.GetProviderStatusThirdPartyView.as_view()), path('provider/create', views.CreateProviderConsentView.as_view()), - path('provider/<provider_id>/create-verb-group', views.CreateProviderVerbGroupView.as_view()), # TODO - path('provider/<provider_id>/verb-groups', views.GetProviderVerbGroupsView.as_view()), # TODO + path('provider/<provider_id>/create-verb-group', views.CreateProviderVerbGroupView.as_view()), + path('provider/<provider_id>/verb-groups', views.GetProviderVerbGroupsView.as_view()), path('user/save', views.SaveUserConsentView.as_view()), path('user/create', views.CreateUserConsentView.as_view()), path('user/create-via-connect-service', views.CreateUserConsentViaConnectServiceView.as_view()), diff --git a/src/consents/views.py b/src/consents/views.py index 6e140bc..c4731ba 100644 --- a/src/consents/views.py +++ b/src/consents/views.py @@ -199,14 +199,16 @@ class GetProviderVerbGroupsView(generics.ListAPIView): """ Lists all verb groups for the current provider. """ - def get(self, request): - application_token = request.headers.get("Authorization", "").split("Basic ")[-1] - provider = ProviderAuthorization.objects.filter(key=application_token).first() + permission_classes = [IsAuthenticated, IsProvider] + + def get(self, request, provider_id): + + provider = ProviderAuthorization.objects.filter(pk=provider_id).first() if provider is None: return JsonResponse( - {"message": "invalid access token"}, + {"message": "invalid provider supplied"}, safe=False, - status=status.HTTP_401_UNAUTHORIZED, + status=status.HTTP_404_NOT_FOUND, ) verb_groups = ProviderVerbGroup.objects.filter(provider=provider.provider).all() serializer = ProviderVerbGroupSerializer(verb_groups, many=True) @@ -487,7 +489,7 @@ class GetUserConsentHistoryView(APIView): if not user_consents.first(): return JsonResponse( { - "consents": [], + "groups": [], }, safe=False, status=status.HTTP_200_OK, diff --git a/src/data_disclosure/tests.py b/src/data_disclosure/tests.py deleted file mode 100644 index 7ce503c..0000000 --- a/src/data_disclosure/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/src/data_disclosure/tests/__init__.py b/src/data_disclosure/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/data_disclosure/tests/tests_data_disclosure.py b/src/data_disclosure/tests/tests_data_disclosure.py new file mode 100644 index 0000000..d338e4f --- /dev/null +++ b/src/data_disclosure/tests/tests_data_disclosure.py @@ -0,0 +1,251 @@ +import json +import os +from io import StringIO + +from django.conf import settings +from django.core.files.base import ContentFile +from django.core.files.storage import default_storage +from rest_framework.test import APIClient +from rolepermissions.roles import assign_role +from django.core.management import call_command + +from consents.tests.tests_consent_operations import BaseTestCase +from providers.models import ProviderAuthorization, Provider, ProviderVerbGroup, ProviderSchema +from users.models import CustomUser + +from data_disclosure.models import DataDisclosure + + +class TestsDataDisclosure(BaseTestCase): + verb_groups = [ + { + "provider_id": 1, + "id": "default_group", + "label": "Default group", + "description": "default", + "showVerbDetails": True, + "purposeOfCollection": "Lorem Ipsum", + "requiresConsent": True, + "verbs": [ + {"id": "http://h5p.example.com/expapi/verbs/experienced"}, + {"id": "http://h5p.example.com/expapi/verbs/attempted"}, + {"id": "http://h5p.example.com/expapi/verbs/interacted"}, + {"id": "http://h5p.example.com/expapi/verbs/answered"} + ] + }, + ] + + def setUp(self): + call_command('check_and_apply_migrations') + normal_user = CustomUser.objects.create_user( + self.test_user_email, self.test_user_password + ) + provider_user = CustomUser.objects.create_user( + self.test_provider_email, self.test_provider_password + ) + + assign_role(normal_user, "user") + assign_role(provider_user, "provider_manager") + + response = self.client.post( + "/api/v1/auth/token", + {"email": self.test_user_email, "password": self.test_user_password}, + ) # obtain token + self.assertEqual(response.status_code, 200) + self.assertTrue("access" in response.data) + user_token = response.data["access"] + + response = self.client.post( + "/api/v1/auth/token", + { + "email": self.test_provider_email, + "password": self.test_provider_password, + }, + ) # obtain token + self.assertEqual(response.status_code, 200) + self.assertTrue("access" in response.data) + provider_token = response.data["access"] + + self.user_client = APIClient() + self.user_client.credentials(HTTP_AUTHORIZATION="Bearer " + user_token) + + self.provider_client = APIClient() + self.provider_client.credentials(HTTP_AUTHORIZATION="Bearer " + provider_token) + + # Create provider schema + with StringIO(json.dumps(self.h5p_provider_schema)) as fp: + response = self.provider_client.put( + "/api/v1/consents/provider/create", + {"provider-schema": fp}, + format="multipart", + ) + self.assertEqual(response.status_code, 201) + + # Get application token for created provider + self.application_token = ProviderAuthorization.objects.get( + provider__name="H5P" + ).key + self.third_party_client = APIClient() + self.third_party_client.credentials( + HTTP_AUTHORIZATION="Basic " + self.application_token + ) + + # create some verb groups + for group in self.verb_groups: + response = self.provider_client.post( + "/api/v1/consents/provider/" + str(Provider.objects.latest('id').id) + "/create-verb-group", + group, + format="json", + ) + self.assertEqual(response.status_code, 200) + + + def test_create_job(self): + self.assertEqual(DataDisclosure.objects.count(), 0) + + response = self.user_client.post("/api/v1/data-disclosure/create") + self.assertEqual(response.status_code, 200) + + self.assertEqual(DataDisclosure.objects.count(), 1) + + def test_list_jobs(self): + response = self.user_client.get("/api/v1/data-disclosure/list") + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertEqual(len(data), 0) + + response = self.user_client.post("/api/v1/data-disclosure/create") + self.assertEqual(response.status_code, 200) + + response = self.user_client.get("/api/v1/data-disclosure/list") + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertEqual(len(data), 1) + + def test_get_secret(self): + response = self.user_client.get("/api/v1/data-disclosure/list") + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertEqual(len(data), 0) + + response = self.user_client.post("/api/v1/data-disclosure/create") + self.assertEqual(response.status_code, 200) + + # create second job from another account + response = self.provider_client.post("/api/v1/data-disclosure/create") + self.assertEqual(response.status_code, 200) + + response = self.user_client.get("/api/v1/data-disclosure/list") + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertEqual(len(data), 1) + + # check if we can get the key + response = self.user_client.get(f"/api/v1/data-disclosure/file_secret/{data[0]['id']}") + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertTrue("secret" in data.keys()) + + # get ID for other account to test permission check + response = self.provider_client.get("/api/v1/data-disclosure/list") + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertEqual(len(data), 1) + + response = self.user_client.get(f"/api/v1/data-disclosure/file_secret/{data[0]['id']}") # note the "user_client" + self.assertEqual(response.status_code, 403) + + + def test_get_file(self): + response = self.user_client.get("/api/v1/data-disclosure/list") + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertEqual(len(data), 0) + + response = self.user_client.post("/api/v1/data-disclosure/create") + self.assertEqual(response.status_code, 200) + + + response = self.user_client.get("/api/v1/data-disclosure/list") + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertEqual(len(data), 1) + + file_id = data[0]["id"] + + # get the key + response = self.user_client.get(f"/api/v1/data-disclosure/file_secret/{file_id}") + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertTrue("secret" in data.keys()) + + secret = data["secret"] + + # create zip file to fulfil requirements + empty_zip_data = ContentFile(b'PK\x05\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00') + file_name = default_storage.save( + f"{settings.DATA_DISCLOSURE_LOCATION}/{file_id}.zip", empty_zip_data + ) + + disc = DataDisclosure.objects.get(id=file_id) + disc.filename = file_name + disc.save() + + # try with wrong secret + response = self.user_client.get(f"/api/v1/data-disclosure/files/{file_id}/somerandomstring") + self.assertEqual(response.status_code, 403) + + + response = self.user_client.get(f"/api/v1/data-disclosure/files/{file_id}/{secret}") + self.assertEqual(response.status_code, 200) + + def test_delete_file(self): + response = self.user_client.get("/api/v1/data-disclosure/list") + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertEqual(len(data), 0) + + response = self.user_client.post("/api/v1/data-disclosure/create") + self.assertEqual(response.status_code, 200) + + + response = self.user_client.get("/api/v1/data-disclosure/list") + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertEqual(len(data), 1) + + file_id = data[0]["id"] + + # get the key + response = self.user_client.get(f"/api/v1/data-disclosure/file_secret/{file_id}") + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertTrue("secret" in data.keys()) + + secret = data["secret"] + + # try with wrong secret + response = self.user_client.get(f"/api/v1/data-disclosure/files/{file_id}/somerandomstring") + self.assertEqual(response.status_code, 403) + + # create zip file to fulfil requirements + empty_zip_data = ContentFile(b'PK\x05\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00') + file_name = default_storage.save( + f"{settings.DATA_DISCLOSURE_LOCATION}/{file_id}.zip", empty_zip_data + ) + + disc = DataDisclosure.objects.get(id=file_id) + disc.filename = file_name + disc.save() + + response = self.user_client.get(f"/api/v1/data-disclosure/files/{file_id}/{secret}") + self.assertEqual(response.status_code, 200) + + # try wrong account for deletion + response = self.provider_client.delete(f"/api/v1/data-disclosure/{file_id}/delete") + self.assertEqual(response.status_code, 403) + + response = self.user_client.delete(f"/api/v1/data-disclosure/{file_id}/delete") + self.assertEqual(response.status_code, 200) + + self.assertFalse(default_storage.exists(file_name)) \ No newline at end of file diff --git a/src/data_removal/tests.py b/src/data_removal/tests.py deleted file mode 100644 index 7ce503c..0000000 --- a/src/data_removal/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/src/data_removal/tests/__init__.py b/src/data_removal/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/data_removal/tests/tests_data_removal.py b/src/data_removal/tests/tests_data_removal.py new file mode 100644 index 0000000..1774e7e --- /dev/null +++ b/src/data_removal/tests/tests_data_removal.py @@ -0,0 +1,130 @@ +import json +import os +from io import StringIO + +from django.conf import settings +from django.core.files.base import ContentFile +from django.core.files.storage import default_storage +from rest_framework.test import APIClient +from rolepermissions.roles import assign_role +from django.core.management import call_command + +from consents.tests.tests_consent_operations import BaseTestCase +from providers.models import ProviderAuthorization, Provider, ProviderVerbGroup, ProviderSchema +from users.models import CustomUser + +from data_removal.models import DataRemovalJob + + +class TestsDataRemoval(BaseTestCase): + verb_groups = [ + { + "provider_id": 1, + "id": "default_group", + "label": "Default group", + "description": "default", + "showVerbDetails": True, + "purposeOfCollection": "Lorem Ipsum", + "requiresConsent": True, + "verbs": [ + {"id": "http://h5p.example.com/expapi/verbs/experienced"}, + {"id": "http://h5p.example.com/expapi/verbs/attempted"}, + {"id": "http://h5p.example.com/expapi/verbs/interacted"}, + {"id": "http://h5p.example.com/expapi/verbs/answered"} + ] + }, + ] + + def setUp(self): + call_command('check_and_apply_migrations') + normal_user = CustomUser.objects.create_user( + self.test_user_email, self.test_user_password + ) + provider_user = CustomUser.objects.create_user( + self.test_provider_email, self.test_provider_password + ) + + assign_role(normal_user, "user") + assign_role(provider_user, "provider_manager") + + response = self.client.post( + "/api/v1/auth/token", + {"email": self.test_user_email, "password": self.test_user_password}, + ) # obtain token + self.assertEqual(response.status_code, 200) + self.assertTrue("access" in response.data) + user_token = response.data["access"] + + response = self.client.post( + "/api/v1/auth/token", + { + "email": self.test_provider_email, + "password": self.test_provider_password, + }, + ) # obtain token + self.assertEqual(response.status_code, 200) + self.assertTrue("access" in response.data) + provider_token = response.data["access"] + + self.user_client = APIClient() + self.user_client.credentials(HTTP_AUTHORIZATION="Bearer " + user_token) + + self.provider_client = APIClient() + self.provider_client.credentials(HTTP_AUTHORIZATION="Bearer " + provider_token) + + # Create provider schema + with StringIO(json.dumps(self.h5p_provider_schema)) as fp: + response = self.provider_client.put( + "/api/v1/consents/provider/create", + {"provider-schema": fp}, + format="multipart", + ) + self.assertEqual(response.status_code, 201) + + # Get application token for created provider + self.application_token = ProviderAuthorization.objects.get( + provider__name="H5P" + ).key + self.third_party_client = APIClient() + self.third_party_client.credentials( + HTTP_AUTHORIZATION="Basic " + self.application_token + ) + + # create some verb groups + for group in self.verb_groups: + response = self.provider_client.post( + "/api/v1/consents/provider/" + str(Provider.objects.latest('id').id) + "/create-verb-group", + group, + format="json", + ) + self.assertEqual(response.status_code, 200) + + + def test_create_job(self): + self.assertEqual(DataRemovalJob.objects.count(), 0) + + response = self.user_client.post("/api/v1/data-removal/create", { + "scope": {}, + "immediately": True, + }, format="json") + self.assertEqual(response.status_code, 200) + + self.assertEqual(DataRemovalJob.objects.count(), 1) + + + def test_list_jobs(self): + response = self.user_client.get("/api/v1/data-removal/list", ) + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertEqual(len(data), 0) + + response = self.user_client.post("/api/v1/data-removal/create", { + "scope": {}, + "immediately": True, + }, format="json") + self.assertEqual(response.status_code, 200) + + response = self.user_client.get("/api/v1/data-removal/list",) + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertEqual(len(data), 1) \ No newline at end of file diff --git a/src/data_removal/views.py b/src/data_removal/views.py index dd3d7f0..a84e922 100644 --- a/src/data_removal/views.py +++ b/src/data_removal/views.py @@ -61,7 +61,7 @@ class CreateDataRemovalJob(APIView): [statement_filter, description] = get_scope_filter( request.user.id, request.data["scope"] ) - if request.data["immedialty"]: + if request.data["immediately"]: DataRemovalJob.objects.create( user=request.user, filter=statement_filter, diff --git a/src/frontend/src/app/data-removal.service.ts b/src/frontend/src/app/data-removal.service.ts index cd518c6..650f6b7 100644 --- a/src/frontend/src/app/data-removal.service.ts +++ b/src/frontend/src/app/data-removal.service.ts @@ -35,7 +35,7 @@ export class DataRemovalService { scope: DeleteAllUserStatements | DeleteVerbStatements | DeleteObjectStatements }): Observable<void> { return this.http.post<void>(`${environment.apiUrl}/api/v1/data-removal/create`, { - immedialty: data.immediately, + immediately: data.immediately, scope: data.scope }) } diff --git a/src/providers/models.py b/src/providers/models.py index ab80e24..c7f7b12 100644 --- a/src/providers/models.py +++ b/src/providers/models.py @@ -71,8 +71,10 @@ class ProviderVerbGroup(models.Model): def __str__(self): return (f"Provider Verb Group {self.id}:\n" + f"- label {self.label}\n" f"- contains {len(self.verbs.all())} verbs\n" f"- group ID: {self.group_id}\n" + f"- provider ID: {self.provider_id}\n" f"- provider Schema ID: {self.provider_schema_id}\n") diff --git a/src/providers/tests/__init__.py b/src/providers/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/providers/tests/tests_analytics_tokens.py b/src/providers/tests/tests_analytics_tokens.py new file mode 100644 index 0000000..3c011d7 --- /dev/null +++ b/src/providers/tests/tests_analytics_tokens.py @@ -0,0 +1,241 @@ +import json +import os + +from django.core.files.uploadedfile import SimpleUploadedFile +from django.test import TestCase +from rest_framework.test import APIClient +from rolepermissions.roles import assign_role +from django.core.management import call_command + +from users.models import CustomUser + +from providers.models import Provider, ProviderAuthorization, AnalyticsToken + +PROJECT_PATH = os.path.abspath(os.path.dirname(__name__)) + +class ProviderTestCase(TestCase): + + test_user_email = "test@mail.com" + test_user_password = "test123" + + test_provider_email = "test2@mail.com" + test_provider_password = "test123" + + def setUp(self): + call_command('check_and_apply_migrations') + normal_user = CustomUser.objects.create_user(self.test_user_email, self.test_user_password) + provider_user = CustomUser.objects.create_user(self.test_provider_email, self.test_provider_password) + + assign_role(normal_user, 'user') + assign_role(provider_user, 'provider_manager') + assign_role(provider_user, 'analyst') # needed to create analytics tokens + + response=self.client.post('/api/v1/auth/token', {"email": self.test_user_email, "password": self.test_user_password}) # obtain token + self.assertEqual(response.status_code, 200) + self.assertTrue('access' in response.data) + user_token = response.data['access'] + + response=self.client.post('/api/v1/auth/token', {"email": self.test_provider_email, "password": self.test_provider_password}) # obtain token + self.assertEqual(response.status_code, 200) + self.assertTrue('access' in response.data) + provider_token = response.data['access'] + + self.user_client = APIClient() + self.user_client.credentials(HTTP_AUTHORIZATION='Bearer ' + user_token) + + self.provider_client = APIClient() + self.provider_client.credentials(HTTP_AUTHORIZATION='Bearer ' + provider_token) + + # we need provider schemas for many of these tests + with open(os.path.join(PROJECT_PATH, "static/provider_schema_h5p_v1.example.json")) as fp: + response=self.provider_client.put('/api/v1/consents/provider/create', { + "provider-schema": fp + }, format='multipart') + self.assertEqual(response.status_code, 201) + self.assertTrue('message' in response.json()) + + def test_create_token(self): + response = self.provider_client.post( + f"/api/v1/provider/analytics-tokens/create", + { + "name": "Test analytics token", + "description": "Test analytics token", + "expires": "12/31/2099", + "can_access": [] + }, format="json" + ) + self.assertEqual(response.status_code, 201) + + + def test_token_name_available(self): + response = self.provider_client.post( + f"/api/v1/provider/analytics-tokens/create", + { + "name": "Test analytics token", + "description": "Test analytics token", + "expires": "12/31/2099", + "can_access": [] + }, format="json" + ) + self.assertEqual(response.status_code, 201) + + # test unavailable + response = self.provider_client.post( + f"/api/v1/provider/analytics-tokens/name-available", + { + "name": "Test analytics token" + }, format="json" + ) + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertFalse(data["is_available"]) + + # test available + response = self.provider_client.post( + f"/api/v1/provider/analytics-tokens/name-available", + { + "name": "Test analytics token other" + }, format="json" + ) + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertTrue(data["is_available"]) + + + # test failure case + response = self.provider_client.post( + f"/api/v1/provider/analytics-tokens/name-available", + { + "name": None + }, format="json" + ) + self.assertEqual(response.status_code, 400) + + def test_delete_token(self): + response = self.provider_client.post( + f"/api/v1/provider/analytics-tokens/create", + { + "name": "Test analytics token", + "description": "Test analytics token", + "expires": "12/31/2099", + "can_access": [] + }, format="json" + ) + self.assertEqual(response.status_code, 201) + self.assertEqual(AnalyticsToken.objects.count(), 1) + + token_id = AnalyticsToken.objects.latest('id').id + + response = self.provider_client.delete(f"/api/v1/provider/analytics-tokens/{token_id}/delete") + self.assertEqual(response.status_code, 204) + self.assertEqual(AnalyticsToken.objects.count(), 0) + + def test_update_token_verbs(self): + provider_id = Provider.objects.latest('id').id + + response = self.provider_client.post( + f"/api/v1/provider/analytics-tokens/create", + { + "name": "Test analytics token", + "description": "Test analytics token", + "expires": "12/31/2099", + "can_access": [] + }, format="json" + ) + self.assertEqual(response.status_code, 201) + + token_id = AnalyticsToken.objects.latest('id').id + + # test wrong token ID + response = self.provider_client.post(f"/api/v1/provider/analytics-tokens/1000000/update-verbs", + { + "active_verbs": [ + {"verb": "http://h5p.example.com/expapi/verbs/experienced", + "provider": provider_id}, + {"verb": "http://h5p.example.com/expapi/verbs/attempted", + "provider": provider_id}, + {"verb": "http://h5p.example.com/expapi/verbs/interacted", + "provider": provider_id}, + {"verb": "http://h5p.example.com/expapi/verbs/answered", + "provider": provider_id}, + ] + }, format="json") + self.assertEqual(response.status_code, 404) + + response = self.provider_client.post(f"/api/v1/provider/analytics-tokens/{token_id}/update-verbs", + { + "active_verbs": [ + {"verb": "http://h5p.example.com/expapi/verbs/experienced", "provider": provider_id}, + {"verb": "http://h5p.example.com/expapi/verbs/attempted", "provider": provider_id}, + {"verb": "http://h5p.example.com/expapi/verbs/interacted", "provider": provider_id}, + {"verb": "http://h5p.example.com/expapi/verbs/answered", "provider": provider_id}, + ] + }, format="json") + self.assertEqual(response.status_code, 200) + + token = AnalyticsToken.objects.filter(pk=token_id).get() + self.assertEqual(token.analyticstokenverb_set.count(), 4) + + def test_update_and_delete_token_image(self): + provider_id = Provider.objects.latest('id').id + + response = self.provider_client.post( + f"/api/v1/provider/analytics-tokens/create", + { + "name": "Test analytics token", + "description": "Test analytics token", + "expires": "12/31/2099", + "can_access": [] + }, format="json" + ) + self.assertEqual(response.status_code, 201) + + token_id = AnalyticsToken.objects.latest('id').id + + response = self.provider_client.post(f"/api/v1/provider/analytics-tokens/{token_id}/update-verbs", + { + "active_verbs": [ + {"verb": "http://h5p.example.com/expapi/verbs/experienced", "provider": provider_id}, + {"verb": "http://h5p.example.com/expapi/verbs/attempted", "provider": provider_id}, + {"verb": "http://h5p.example.com/expapi/verbs/interacted", "provider": provider_id}, + {"verb": "http://h5p.example.com/expapi/verbs/answered", "provider": provider_id}, + ] + }, format="json") + self.assertEqual(response.status_code, 200) + + token = AnalyticsToken.objects.filter(pk=token_id).get() + self.assertEqual(token.analyticstokenverb_set.count(), 4) + + # test upload invalid image + upload_image = SimpleUploadedFile("image.mp4", b"aslkdjnwekjgfnwkgbwekrjgbwgbkwjebg", content_type="video/mp4") + response = self.provider_client.post(f"/api/v1/provider/analytics-tokens/{token_id}/image", + {"image": upload_image}) + self.assertEqual(response.status_code, 400) + + # test upload "valid" image + upload_image = SimpleUploadedFile("image.png", b"aslkdjnwekjgfnwkgbwekrjgbwgbkwjebg", content_type="image/png") + response = self.provider_client.post(f"/api/v1/provider/analytics-tokens/{token_id}/image", + {"image": upload_image}) + self.assertEqual(response.status_code, 200) + + token = AnalyticsToken.objects.filter(pk=token_id).get() + + # test get image + response = self.provider_client.get(f"/api/v1/provider/analytics-tokens/{token_id}/image/{token.image_path}") + self.assertEqual(response.status_code, 200) + + # test get wrong image + response = self.provider_client.get(f"/api/v1/provider/analytics-tokens/{token_id}/image/nonexistent.png") + self.assertEqual(response.status_code, 404) + + # test deletion failure for nonexistent token + response = self.provider_client.delete(f"/api/v1/provider/analytics-tokens/1000000/image") + self.assertEqual(response.status_code, 404) + + # test deletion + response = self.provider_client.delete(f"/api/v1/provider/analytics-tokens/{token_id}/image") + self.assertEqual(response.status_code, 200) + + # test get image + response = self.provider_client.get(f"/api/v1/provider/analytics-tokens/{token_id}/image/{token.image_path}") + self.assertEqual(response.status_code, 404) \ No newline at end of file diff --git a/src/providers/tests.py b/src/providers/tests/tests_provider_management.py similarity index 98% rename from src/providers/tests.py rename to src/providers/tests/tests_provider_management.py index c7d75f0..f17447f 100644 --- a/src/providers/tests.py +++ b/src/providers/tests/tests_provider_management.py @@ -8,7 +8,7 @@ from django.core.management import call_command from users.models import CustomUser -from .models import Provider, ProviderAuthorization +from src.providers.models import Provider, ProviderAuthorization PROJECT_PATH = os.path.abspath(os.path.dirname(__name__)) diff --git a/src/providers/views.py b/src/providers/views.py index 9f6a42c..a210f30 100644 --- a/src/providers/views.py +++ b/src/providers/views.py @@ -238,8 +238,8 @@ class UpdateAnalyticsTokensVerbs(APIView): status=status.HTTP_401_UNAUTHORIZED, ) - # delete all currently assignes verbs - AnalyticsTokenVerb.objects.filter(analytics_token=analytics_token.id).delete(); + # delete all currently assigned verbs + AnalyticsTokenVerb.objects.filter(analytics_token=analytics_token.id).delete() for activeverb in request.data["active_verbs"]: v = AnalyticsTokenVerb() v.verb = activeverb["verb"] @@ -248,7 +248,7 @@ class UpdateAnalyticsTokensVerbs(APIView): v.save() - return Response({"message": "sucess"}, status=status.HTTP_200_OK) + return Response({"message": "success"}, status=status.HTTP_200_OK) except ObjectDoesNotExist: return Response( {"message": "token doesn't exist"}, status=status.HTTP_404_NOT_FOUND diff --git a/src/settings/tests/__init__.py b/src/settings/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/settings/tests/tests_privacy_policy.py b/src/settings/tests/tests_privacy_policy.py new file mode 100644 index 0000000..9a67194 --- /dev/null +++ b/src/settings/tests/tests_privacy_policy.py @@ -0,0 +1,136 @@ +import json +import os +from io import StringIO + +from rest_framework.test import APIClient +from rolepermissions.roles import assign_role +from django.core.management import call_command + +from consents.tests.tests_consent_operations import BaseTestCase +from providers.models import ProviderAuthorization, Provider, ProviderVerbGroup, ProviderSchema +from users.models import CustomUser + +from consents.models import UserConsents + + +class TestUserEndpoints(BaseTestCase): + verb_groups = [ + { + "provider_id": 1, + "id": "default_group", + "label": "Default group", + "description": "default", + "showVerbDetails": True, + "purposeOfCollection": "Lorem Ipsum", + "requiresConsent": True, + "verbs": [ + {"id": "http://h5p.example.com/expapi/verbs/experienced"}, + {"id": "http://h5p.example.com/expapi/verbs/attempted"}, + {"id": "http://h5p.example.com/expapi/verbs/interacted"}, + {"id": "http://h5p.example.com/expapi/verbs/answered"} + ] + }, + ] + + def setUp(self): + call_command('check_and_apply_migrations') + normal_user = CustomUser.objects.create_user( + self.test_user_email, self.test_user_password + ) + provider_user = CustomUser.objects.create_superuser( # super user, since we need extended permissions + self.test_provider_email, self.test_provider_password + ) + + assign_role(normal_user, "user") + assign_role(provider_user, "provider_manager") + assign_role(provider_user, 'polaris_administrator') + + response = self.client.post( + "/api/v1/auth/token", + {"email": self.test_user_email, "password": self.test_user_password}, + ) # obtain token + self.assertEqual(response.status_code, 200) + self.assertTrue("access" in response.data) + user_token = response.data["access"] + + response = self.client.post( + "/api/v1/auth/token", + { + "email": self.test_provider_email, + "password": self.test_provider_password, + }, + ) # obtain token + self.assertEqual(response.status_code, 200) + self.assertTrue("access" in response.data) + provider_token = response.data["access"] + + self.user_client = APIClient() + self.user_client.credentials(HTTP_AUTHORIZATION="Bearer " + user_token) + + self.provider_client = APIClient() + self.provider_client.credentials(HTTP_AUTHORIZATION="Bearer " + provider_token) + + # Create provider schema + with StringIO(json.dumps(self.h5p_provider_schema)) as fp: + response = self.provider_client.put( + "/api/v1/consents/provider/create", + {"provider-schema": fp}, + format="multipart", + ) + self.assertEqual(response.status_code, 201) + + # Get application token for created provider + self.application_token = ProviderAuthorization.objects.get( + provider__name="H5P" + ).key + self.third_party_client = APIClient() + self.third_party_client.credentials( + HTTP_AUTHORIZATION="Basic " + self.application_token + ) + + # create some verb groups + for group in self.verb_groups: + response = self.provider_client.post( + "/api/v1/consents/provider/" + str(Provider.objects.latest('id').id) + "/create-verb-group", + group, + format="json", + ) + self.assertEqual(response.status_code, 200) + + def test_set_privacy_policy(self): + # test unauthorized user + response = self.user_client.post( + f"/api/v1/settings/privacy-policy", + {"content": "test"}, + format="json" + ) + self.assertEqual(response.status_code, 403) + + # test authorized user + response = self.provider_client.post( + f"/api/v1/settings/privacy-policy", + {"content": "test"}, + format="json" + ) + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertEqual(data["content"], "test") + + def test_get_privacy_policy(self): + # test authorized user + response = self.provider_client.post( + f"/api/v1/settings/privacy-policy", + {"content": "test"}, + format="json" + ) + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertEqual(data["content"], "test") + + # test unauthorized user + response = self.user_client.get( + f"/api/v1/settings/privacy-policy", + ) + self.assertEqual(response.status_code, 200) + data = json.loads(response.content.decode()) + self.assertEqual(data["content"], "test") \ No newline at end of file -- GitLab From 3924d744027b4e5fdede0ff3e921d6df354835f5 Mon Sep 17 00:00:00 2001 From: Lennard Strohmeyer <lennard.strohmeyer@digitallearning.gmbh> Date: Thu, 27 Mar 2025 16:53:22 +0100 Subject: [PATCH 02/12] fixed imports in broken test cases --- src/consents/tests/tests_consent_operations.py | 2 +- src/providers/tests/tests_provider_management.py | 2 +- src/users/tests.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/consents/tests/tests_consent_operations.py b/src/consents/tests/tests_consent_operations.py index 13c22c4..72887c8 100644 --- a/src/consents/tests/tests_consent_operations.py +++ b/src/consents/tests/tests_consent_operations.py @@ -11,7 +11,7 @@ from providers.models import Provider, ProviderAuthorization, ProviderSchema, Pr from users.models import CustomUser from django.core.management import call_command -from ..models import UserConsents +from consents.models import UserConsents PROJECT_PATH = os.path.abspath(os.path.dirname(__name__)) diff --git a/src/providers/tests/tests_provider_management.py b/src/providers/tests/tests_provider_management.py index f17447f..a6ea6b0 100644 --- a/src/providers/tests/tests_provider_management.py +++ b/src/providers/tests/tests_provider_management.py @@ -8,7 +8,7 @@ from django.core.management import call_command from users.models import CustomUser -from src.providers.models import Provider, ProviderAuthorization +from providers.models import Provider, ProviderAuthorization PROJECT_PATH = os.path.abspath(os.path.dirname(__name__)) diff --git a/src/users/tests.py b/src/users/tests.py index 6a49d12..012b400 100644 --- a/src/users/tests.py +++ b/src/users/tests.py @@ -3,7 +3,7 @@ from rest_framework.test import APIClient from rolepermissions.roles import assign_role from django.core.management import call_command -from .models import CustomUser +from users.models import CustomUser class UserTests(TestCase): -- GitLab From 53ef3e7f3bdd5a67bb6e800f94c0be96c1706976 Mon Sep 17 00:00:00 2001 From: Lennard Strohmeyer <lennard.strohmeyer@digitallearning.gmbh> Date: Thu, 27 Mar 2025 17:07:29 +0100 Subject: [PATCH 03/12] #152: streamlined third party API, updated documentation --- docs/docs/third_party_access.md | 789 ++++++++++---------------------- src/consents/views.py | 14 - 2 files changed, 235 insertions(+), 568 deletions(-) diff --git a/docs/docs/third_party_access.md b/docs/docs/third_party_access.md index 93328c1..cdd896d 100644 --- a/docs/docs/third_party_access.md +++ b/docs/docs/third_party_access.md @@ -18,7 +18,7 @@ In cases were neither a provider schema is available or the provider hasn't yet ### Get user consent -This endpoint returns the current user consent for a given user along with the provider schema the user accepted and a flag for data recording pausation. +This endpoint returns the current user consent for a given user, along with a flag indicating data recording pausation. - HTTP Method: **GET** - URL `/api/v1/consents/user/status/user1@polaris.com/third-party` @@ -31,11 +31,11 @@ This endpoint returns the current user consent for a given user along with the p ###### Returns -| Code | Type | Description | -| ----- | ---- | -------------------------------------------------------------------------------------------- | -| `200` | dict | dict containing `user_consent`, `provider_schema` and `paused_data_recording` (boolean flag) | -| `401` | dict | invalid token | -| `400` | dict | invalid request data e.g. `{"user_id":["Object with email=user1@polaris does not exist."]}` | +| Code | Type | Description | +| ----- | ---- |------------------------------------------------------------------------------------------| +| `200` | dict | dict containing `consent` and `paused_data_recording` (boolean flag) | +| `401` | dict | invalid token | +| `400` | dict | invalid request data e.g. `{"user_id":["Object with email=user1@polaris does not exist."]}` | ###### Example @@ -45,7 +45,7 @@ $ curl -X GET http://[RIGHT_ENGINE_URL]/api/v1/consents/user/status/user1@polari Response -```js +```json { "consent": { "id": 1, @@ -138,181 +138,13 @@ Response "objects": [] }] }, - "provider_schema": { - "id": 1, - "definition": { - "id": "h5p-0", - "name": "H5P", - "description": "Open-source content collaboration framework", - "groups": [{ - "label": "Default group", - "description": "default", - "verbs": [{ - "id": "http://h5p.example.com/expapi/verbs/experienced", - "label": "Experienced", - "description": "Experienced", - "defaultConsent": true, - "objects": [{ - "id": "http://h5p.example.com/expapi/activity/QKGPPiIhI4zx9YAZZksLKigqyf7yW4WF", - "label": "1.1.1 Funktionen", - "defaultConsent": true, - "definition": { - "name": { - "enUS": "1.1.1 Funktionen" - } - } - }] - }, { - "id": "http://h5p.example.com/expapi/verbs/attempted", - "label": "Attempted", - "description": "Attempted", - "defaultConsent": true, - "objects": [{ - "id": "http://h5p.example.com/expapi/activity/VeH7S8NeGlCRM1myYRDBjHMCknLqDLgm", - "label": "2.3.1 Funktion Zirkulationsleitung", - "defaultConsent": true, - "definition": { - "name": { - "enUS": "2.3.1 Funktion Zirkulationsleitung" - } - } - }] - }], - "isDefault": true - }, { - "label": "Group 2", - "description": "Lorem ipsum", - "verbs": [{ - "id": "http://h5p.example.com/expapi/verbs/interacted", - "label": "Interacted", - "description": "Lorem ipsum", - "defaultConsent": true, - "objects": [{ - "id": "http://h5p.example.com/expapi/activity/ofN2ODcnLRaVu30lUpzrWPqF2AcG7g46", - "label": "1.2.3 Kappenventil", - "defaultConsent": true, - "definition": { - "name": { - "enUS": "1.2.3 Kappenventil" - } - } - }] - }, { - "id": "http://h5p.example.com/expapi/verbs/answered", - "label": "Answered", - "description": "lorem ipsum", - "defaultConsent": false, - "objects": [{ - "id": "http://h5p.example.com/expapi/activity/K34IszYvGE4R0cC72Ean6msLfLCJtQ8b", - "label": "7.2.1 Ventil Basics", - "defaultConsent": false, - "definition": { - "name": { - "enUS": "7.2.1 Ventil Basics" - } - } - }] - }], - "isDefault": false - }], - "essential_verbs": [{ - "id": "http://h5p.example.com/expapi/verbs/liked", - "label": "Liked", - "description": "Like interaction", - "defaultConsent": true, - "objects": [] - }] - }, - "groups": [{ - "label": "Default group", - "description": "default", - "verbs": [{ - "id": "http://h5p.example.com/expapi/verbs/experienced", - "label": "Experienced", - "description": "Experienced", - "defaultConsent": true, - "objects": [{ - "id": "http://h5p.example.com/expapi/activity/QKGPPiIhI4zx9YAZZksLKigqyf7yW4WF", - "label": "1.1.1 Funktionen", - "defaultConsent": true, - "definition": { - "name": { - "enUS": "1.1.1 Funktionen" - } - } - }] - }, { - "id": "http://h5p.example.com/expapi/verbs/attempted", - "label": "Attempted", - "description": "Attempted", - "defaultConsent": true, - "objects": [{ - "id": "http://h5p.example.com/expapi/activity/VeH7S8NeGlCRM1myYRDBjHMCknLqDLgm", - "label": "2.3.1 Funktion Zirkulationsleitung", - "defaultConsent": true, - "definition": { - "name": { - "enUS": "2.3.1 Funktion Zirkulationsleitung" - } - } - }] - }], - "isDefault": true - }, { - "label": "Group 2", - "description": "Lorem ipsum", - "verbs": [{ - "id": "http://h5p.example.com/expapi/verbs/interacted", - "label": "Interacted", - "description": "Lorem ipsum", - "defaultConsent": true, - "objects": [{ - "id": "http://h5p.example.com/expapi/activity/ofN2ODcnLRaVu30lUpzrWPqF2AcG7g46", - "label": "1.2.3 Kappenventil", - "defaultConsent": true, - "definition": { - "name": { - "enUS": "1.2.3 Kappenventil" - } - } - }] - }, { - "id": "http://h5p.example.com/expapi/verbs/answered", - "label": "Answered", - "description": "lorem ipsum", - "defaultConsent": false, - "objects": [{ - "id": "http://h5p.example.com/expapi/activity/K34IszYvGE4R0cC72Ean6msLfLCJtQ8b", - "label": "7.2.1 Ventil Basics", - "defaultConsent": false, - "definition": { - "name": { - "enUS": "7.2.1 Ventil Basics" - } - } - }] - }], - "isDefault": false - }], - "essential_verbs": [{ - "id": "http://h5p.example.com/expapi/verbs/liked", - "label": "Liked", - "description": "Like interaction", - "defaultConsent": true, - "objects": [] - }], - "updated": "2023-02-09T12:45:57.562000Z", - "created": "2023-02-09T12:45:57.562000Z", - "provider": 1, - "superseded_by": null - }, "paused_data_recording": false } ``` ### Get provider details -Allows a third party to query the details of one's provider. Among other things, all created schemas are supplied here. +Allows a third party to query the details of the current provider. Among other things, all associated groups and created schemas are supplied here. - HTTP Method: **GET** - URL `/api/v1/consents/provider-status/third-party` @@ -325,383 +157,254 @@ $ curl -X GET 127.0.0.1:8003/api/v1/consents/provider-status/third-party -H "Con Response -```js +```json { - "id": 1, - "name": "H5P", - "versions": [{ - "id": 5, - "definition": { - "id": "h5p-0", - "name": "H5P", - "description": "Open-source content collaboration framework", - "groups": [{ - "label": "Default group", - "description": "default", - "verbs": [{ - "id": "http://h5p.example.com/expapi/verbs/experienced", - "label": "Experienced", - "description": "Experienced", - "defaultConsent": false, - "objects": [{ - "id": "http://h5p.example.com/expapi/activity/QKGPPiIhI4zx9YAZZksLKigqyf7yW4WF", - "label": "1.1.1 Funktionen", - "defaultConsent": true, - "definition": { - "name": { - "enUS": "1.1.1 Funktionen" - } - } - }] - }, { - "id": "http://h5p.example.com/expapi/verbs/attempted", - "label": "Attempted", - "description": "Attempted", - "defaultConsent": true, - "objects": [{ - "id": "http://h5p.example.com/expapi/activity/VeH7S8NeGlCRM1myYRDBjHMCknLqDLgm", - "label": "2.3.1 Funktion Zirkulationsleitung", - "defaultConsent": true, - "definition": { - "name": { - "enUS": "2.3.1 Funktion Zirkulationsleitung" - } - } - }] - }], - "isDefault": true - }, { - "label": "Group 2", - "description": "Lorem ipsum", - "verbs": [{ - "id": "http://h5p.example.com/expapi/verbs/interacted", - "label": "Interacted", - "description": "Lorem ipsum", - "defaultConsent": true, - "objects": [{ - "id": "http://h5p.example.com/expapi/activity/ofN2ODcnLRaVu30lUpzrWPqF2AcG7g46", - "label": "1.2.3 Kappenventil", - "defaultConsent": true, - "definition": { - "name": { - "enUS": "1.2.3 Kappenventil" - } - } - }] - }, { - "id": "http://h5p.example.com/expapi/verbs/answered", - "label": "Answered", - "description": "lorem ipsum", - "defaultConsent": false, - "objects": [{ - "id": "http://h5p.example.com/expapi/activity/K34IszYvGE4R0cC72Ean6msLfLCJtQ8b", - "label": "7.2.1 Ventil Basics", - "defaultConsent": false, - "definition": { - "name": { - "enUS": "7.2.1 Ventil Basics" - } - } - }] - }], - "isDefault": false - }], - "essential_verbs": [{ - "id": "http://h5p.example.com/expapi/verbs/liked", - "label": "Liked", - "description": "Like interaction", - "defaultConsent": true, - "objects": [] - }, { - "id": "http://h5p.example.com/expapi/verbs/clicked", - "label": "Clicked", - "description": "Clicked interaction", - "defaultConsent": true, - "objects": [] - }] + "id": 1, + "name": "H5P", + "groups": [ + { + "id": 1, + "verbs": [ + { + "id": "http://h5p.example.com/expapi/verbs/experienced", + "label": "Experienced", + "description": "Experienced", + "defaultConsent": true, + "objects": [ + { + "id": "http://h5p.example.com/expapi/activity/QKGPPiIhI4zx9YAZZksLKigqyf7yW4WF", + "objectType": "Activity", + "defaultConsent": true, + "definition": { + "type": "http://h5p.example.com/expapi/activity/QKGPPiIhI4zx9YAZZksLKigqyf7yW4WF", + "name": { + "enUS": "1.1.1 Funktionen" + } + }, + "object_type": "Activity", + "matching": "definitionType", + "label": "" + } + ] }, - "groups": [{ - "label": "Default group", - "description": "default", - "verbs": [{ - "id": "http://h5p.example.com/expapi/verbs/experienced", - "label": "Experienced", - "description": "Experienced", - "defaultConsent": false, - "objects": [{ - "id": "http://h5p.example.com/expapi/activity/QKGPPiIhI4zx9YAZZksLKigqyf7yW4WF", - "label": "1.1.1 Funktionen", - "defaultConsent": true, - "definition": { - "name": { - "enUS": "1.1.1 Funktionen" - } - } - }] - }, { - "id": "http://h5p.example.com/expapi/verbs/attempted", - "label": "Attempted", - "description": "Attempted", - "defaultConsent": true, - "objects": [{ - "id": "http://h5p.example.com/expapi/activity/VeH7S8NeGlCRM1myYRDBjHMCknLqDLgm", - "label": "2.3.1 Funktion Zirkulationsleitung", - "defaultConsent": true, - "definition": { - "name": { - "enUS": "2.3.1 Funktion Zirkulationsleitung" - } - } - }] - }], - "isDefault": true - }, { - "label": "Group 2", - "description": "Lorem ipsum", - "verbs": [{ - "id": "http://h5p.example.com/expapi/verbs/interacted", - "label": "Interacted", - "description": "Lorem ipsum", - "defaultConsent": true, - "objects": [{ - "id": "http://h5p.example.com/expapi/activity/ofN2ODcnLRaVu30lUpzrWPqF2AcG7g46", - "label": "1.2.3 Kappenventil", - "defaultConsent": true, - "definition": { - "name": { - "enUS": "1.2.3 Kappenventil" - } - } - }] - }, { - "id": "http://h5p.example.com/expapi/verbs/answered", - "label": "Answered", - "description": "lorem ipsum", - "defaultConsent": false, - "objects": [{ - "id": "http://h5p.example.com/expapi/activity/K34IszYvGE4R0cC72Ean6msLfLCJtQ8b", - "label": "7.2.1 Ventil Basics", - "defaultConsent": false, - "definition": { - "name": { - "enUS": "7.2.1 Ventil Basics" - } - } - }] - }], - "isDefault": false - }], - "essential_verbs": [{ - "id": "http://h5p.example.com/expapi/verbs/liked", - "label": "Liked", - "description": "Like interaction", - "defaultConsent": true, - "objects": [] - }, { - "id": "http://h5p.example.com/expapi/verbs/clicked", - "label": "Clicked", - "description": "Clicked interaction", - "defaultConsent": true, - "objects": [] - }], - "updated": "2023-02-09T10:19:51.566456Z", - "created": "2023-02-09T10:19:51.566471Z", - "provider": 1, - "superseded_by": null - }, { - "id": 1, - "definition": { - "id": "h5p-0", - "name": "H5P", - "description": "Open-source content collaboration framework", - "groups": [{ - "label": "Default group", - "description": "default", - "verbs": [{ - "id": "http://h5p.example.com/expapi/verbs/experienced", - "label": "Experienced", - "description": "Experienced", - "defaultConsent": true, - "objects": [{ - "id": "http://h5p.example.com/expapi/activity/QKGPPiIhI4zx9YAZZksLKigqyf7yW4WF", - "label": "1.1.1 Funktionen", - "defaultConsent": true, - "definition": { - "name": { - "enUS": "1.1.1 Funktionen" - } - } - }] - }, { - "id": "http://h5p.example.com/expapi/verbs/attempted", - "label": "Attempted", - "description": "Attempted", - "defaultConsent": true, - "objects": [{ - "id": "http://h5p.example.com/expapi/activity/VeH7S8NeGlCRM1myYRDBjHMCknLqDLgm", - "label": "2.3.1 Funktion Zirkulationsleitung", - "defaultConsent": true, - "definition": { - "name": { - "enUS": "2.3.1 Funktion Zirkulationsleitung" - } - } - }] - }], - "isDefault": true - }, { - "label": "Group 2", - "description": "Lorem ipsum", - "verbs": [{ - "id": "http://h5p.example.com/expapi/verbs/interacted", - "label": "Interacted", - "description": "Lorem ipsum", - "defaultConsent": true, - "objects": [{ - "id": "http://h5p.example.com/expapi/activity/ofN2ODcnLRaVu30lUpzrWPqF2AcG7g46", - "label": "1.2.3 Kappenventil", - "defaultConsent": true, - "definition": { - "name": { - "enUS": "1.2.3 Kappenventil" - } - } - }] - }, { - "id": "http://h5p.example.com/expapi/verbs/answered", - "label": "Answered", - "description": "lorem ipsum", - "defaultConsent": false, - "objects": [{ - "id": "http://h5p.example.com/expapi/activity/K34IszYvGE4R0cC72Ean6msLfLCJtQ8b", - "label": "7.2.1 Ventil Basics", - "defaultConsent": false, - "definition": { - "name": { - "enUS": "7.2.1 Ventil Basics" - } - } - }] - }], - "isDefault": false - }], - "essential_verbs": [{ - "id": "http://h5p.example.com/expapi/verbs/liked", - "label": "Liked", - "description": "Like interaction", - "defaultConsent": true, - "objects": [] - }] + { + "id": "http://h5p.example.com/expapi/verbs/attempted", + "label": "Attempted", + "description": "Attempted", + "defaultConsent": true, + "objects": [ + { + "id": "http://h5p.example.com/expapi/activity/VeH7S8NeGlCRM1myYRDBjHMCknLqDLgm", + "objectType": "Activity", + "defaultConsent": true, + "definition": { + "type": "http://h5p.example.com/expapi/activity/VeH7S8NeGlCRM1myYRDBjHMCknLqDLgm", + "name": { + "enUS": "2.3.1 Funktion Zirkulationsleitung" + } + }, + "object_type": "Activity", + "matching": "definitionType", + "label": "" + } + ] }, - "groups": [{ - "label": "Default group", - "description": "default", - "verbs": [{ - "id": "http://h5p.example.com/expapi/verbs/experienced", - "label": "Experienced", - "description": "Experienced", + { + "id": "http://h5p.example.com/expapi/verbs/interacted", + "label": "Interacted", + "description": "Lorem ipsum", + "defaultConsent": true, + "objects": [ + { + "id": "http://h5p.example.com/expapi/activity/ofN2ODcnLRaVu30lUpzrWPqF2AcG7g46", + "objectType": "Activity", + "defaultConsent": true, + "definition": { + "type": "http://h5p.example.com/expapi/activity/ofN2ODcnLRaVu30lUpzrWPqF2AcG7g46", + "name": { + "enUS": "1.2.3 Kappenventil" + } + }, + "object_type": "Activity", + "matching": "definitionType", + "label": "" + } + ] + }, + { + "id": "http://h5p.example.com/expapi/verbs/answered", + "label": "Answered", + "description": "lorem ipsum", + "defaultConsent": false, + "objects": [ + { + "id": "http://h5p.example.com/expapi/activity/K34IszYvGE4R0cC72Ean6msLfLCJtQ8b", + "objectType": "Activity", + "defaultConsent": false, + "definition": { + "type": "http://h5p.example.com/expapi/activity/K34IszYvGE4R0cC72Ean6msLfLCJtQ8b", + "name": { + "enUS": "7.2.1 Ventil Basics" + } + }, + "object_type": "Activity", + "matching": "definitionType", + "label": "" + } + ] + } + ], + "purposeOfCollection": "Lorem Ipsum", + "group_id": "372f255714963f49f0f8e7acca75e0fc55140d00cae2d4328f5a9a6bf090c359", + "label": "Default group", + "description": "default", + "requires_consent": true, + "updated": "2025-03-27T16:01:20.025590Z", + "created": "2025-03-27T16:01:20.025602Z", + "provider": 1, + "provider_schema": 1 + } + ], + "versions": [ + { + "id": 1, + "definition": { + "id": "h5p-0", + "name": "H5P", + "description": "Open-source content collaboration framework", + "verbs": [ + { + "id": "http://h5p.example.com/expapi/verbs/experienced", + "label": "Experienced", + "description": "Experienced", + "defaultConsent": true, + "objects": [ + { + "id": "http://h5p.example.com/expapi/activity/QKGPPiIhI4zx9YAZZksLKigqyf7yW4WF", + "objectType": "Activity", "defaultConsent": true, - "objects": [{ - "id": "http://h5p.example.com/expapi/activity/QKGPPiIhI4zx9YAZZksLKigqyf7yW4WF", - "label": "1.1.1 Funktionen", - "defaultConsent": true, - "definition": { - "name": { - "enUS": "1.1.1 Funktionen" - } - } - }] - }, { - "id": "http://h5p.example.com/expapi/verbs/attempted", - "label": "Attempted", - "description": "Attempted", + "definition": { + "type": "http://h5p.example.com/expapi/activity/QKGPPiIhI4zx9YAZZksLKigqyf7yW4WF", + "name": { + "enUS": "1.1.1 Funktionen" + } + }, + "object_type": "Activity", + "matching": "definitionType", + "label": "" + } + ] + }, + { + "id": "http://h5p.example.com/expapi/verbs/attempted", + "label": "Attempted", + "description": "Attempted", + "defaultConsent": true, + "objects": [ + { + "id": "http://h5p.example.com/expapi/activity/VeH7S8NeGlCRM1myYRDBjHMCknLqDLgm", + "objectType": "Activity", "defaultConsent": true, - "objects": [{ - "id": "http://h5p.example.com/expapi/activity/VeH7S8NeGlCRM1myYRDBjHMCknLqDLgm", - "label": "2.3.1 Funktion Zirkulationsleitung", - "defaultConsent": true, - "definition": { - "name": { - "enUS": "2.3.1 Funktion Zirkulationsleitung" - } - } - }] - }], - "isDefault": true - }, { - "label": "Group 2", + "definition": { + "type": "http://h5p.example.com/expapi/activity/VeH7S8NeGlCRM1myYRDBjHMCknLqDLgm", + "name": { + "enUS": "2.3.1 Funktion Zirkulationsleitung" + } + }, + "object_type": "Activity", + "matching": "definitionType", + "label": "" + } + ] + }, + { + "id": "http://h5p.example.com/expapi/verbs/interacted", + "label": "Interacted", "description": "Lorem ipsum", - "verbs": [{ - "id": "http://h5p.example.com/expapi/verbs/interacted", - "label": "Interacted", - "description": "Lorem ipsum", + "defaultConsent": true, + "objects": [ + { + "id": "http://h5p.example.com/expapi/activity/ofN2ODcnLRaVu30lUpzrWPqF2AcG7g46", + "objectType": "Activity", "defaultConsent": true, - "objects": [{ - "id": "http://h5p.example.com/expapi/activity/ofN2ODcnLRaVu30lUpzrWPqF2AcG7g46", - "label": "1.2.3 Kappenventil", - "defaultConsent": true, - "definition": { - "name": { - "enUS": "1.2.3 Kappenventil" - } - } - }] - }, { - "id": "http://h5p.example.com/expapi/verbs/answered", - "label": "Answered", - "description": "lorem ipsum", + "definition": { + "type": "http://h5p.example.com/expapi/activity/ofN2ODcnLRaVu30lUpzrWPqF2AcG7g46", + "name": { + "enUS": "1.2.3 Kappenventil" + } + }, + "object_type": "Activity", + "matching": "definitionType", + "label": "" + } + ] + }, + { + "id": "http://h5p.example.com/expapi/verbs/answered", + "label": "Answered", + "description": "lorem ipsum", + "defaultConsent": false, + "objects": [ + { + "id": "http://h5p.example.com/expapi/activity/K34IszYvGE4R0cC72Ean6msLfLCJtQ8b", + "objectType": "Activity", "defaultConsent": false, - "objects": [{ - "id": "http://h5p.example.com/expapi/activity/K34IszYvGE4R0cC72Ean6msLfLCJtQ8b", - "label": "7.2.1 Ventil Basics", - "defaultConsent": false, - "definition": { - "name": { - "enUS": "7.2.1 Ventil Basics" - } - } - }] - }], - "isDefault": false - }], - "essential_verbs": [{ + "definition": { + "type": "http://h5p.example.com/expapi/activity/K34IszYvGE4R0cC72Ean6msLfLCJtQ8b", + "name": { + "enUS": "7.2.1 Ventil Basics" + } + }, + "object_type": "Activity", + "matching": "definitionType", + "label": "" + } + ] + } + ], + "essential_verbs": [ + { "id": "http://h5p.example.com/expapi/verbs/liked", "label": "Liked", "description": "Like interaction", "defaultConsent": true, "objects": [] - }], - "updated": "2023-02-09T10:19:51.568203Z", - "created": "2023-02-09T09:45:15.405764Z", - "provider": 1, - "superseded_by": 5 - }] + } + ], + "additional_lrs": [] + }, + "additional_lrs": [], + "updated": "2025-03-27T16:01:20.013790Z", + "created": "2025-03-27T16:01:20.013799Z", + "provider": 1, + "superseded_by": null + } + ] } ``` ### Save user consent -The provider schema id for each provider must always refer to the latest schema version of the respective provider, otherwise the request will be rejected +Save consent declarations for one or multiple verb groups for a user. - HTTP Method: **POST** - URL `/api/v1/consents/user/save/third-party` ###### Parameters -| Name | Type | Required | Default | -| -------------------- | ---------- | -------- | ------- | -| `user_id` | str | Yes | None | -| `provider_schema_id` | int | Yes | None | -| `verbs` | list[dict] | Yes | None | +| Name | Type | Required | Default | +|----------------------|-----------| -------- | ------- | +| `user_id` | str | Yes | None | +| `groups` | list[int] | Yes | None | ###### Returns -| Code | Type | Description | -| ----- | ---- | ----------------------------------------------------------------------------------------------------- | -| `200` | dict | dict with entry for each user contain list of consented verb and object ids | -| `401` | dict | invalid token | -| `403` | dict | reaised if a verb for a provider other than the one associated with the application token is provided | -| `400` | dict | invalid request data e.g. `{"user_id":["Object with email=user1@polaris does not exist."]}` | +| Code | Type | Description | +| ----- | ---- |------------------------------------------------------------------------------------------------------| +| `200` | dict | dict with the key "message" stating that the consent was saved | +| `401` | dict | invalid token | +| `403` | dict | raised if a verb for a provider other than the one associated with the application token is provided | +| `400` | dict | invalid request data e.g. `{"user_id":["Object with email=user1@polaris does not exist."]}` | ###### Examples @@ -710,36 +413,14 @@ curl -X POST 127.0.0.1:8003/api/v1/consents/user/save/third-party --data '{ "use ``` -Formatted request payload -```js +Formatted request payload (group IDs have to be retrieved beforehand) +```json { "user_id": "user1@polaris.com", - "provider_schema_id": 3, - "verbs": [ - { - "provider": 1, - "id": "http://h5p.example.com/expapi/verbs/experienced", - "consented": false, - "objects": "[{\"id\":\"http://h5p.example.com/expapi/activity/QKGPPiIhI4zx9YAZZksLKigqyf7yW4WF\",\"label\":\"1.1.1 Funktionen\",\"defaultConsent\":true,\"definition\":{\"name\":{\"enUS\":\"1.1.1 Funktionen\"}},\"consented\":true}]" - }, - { - "provider": 1, - "id": "http://h5p.example.com/expapi/verbs/attempted", - "consented": true, - "objects": "[{\"id\":\"http://h5p.example.com/expapi/activity/VeH7S8NeGlCRM1myYRDBjHMCknLqDLgm\",\"label\":\"2.3.1 Funktion Zirkulationsleitung\",\"defaultConsent\":true,\"definition\":{\"name\":{\"enUS\":\"2.3.1 Funktion Zirkulationsleitung\"}},\"consented\":true}]" - }, - { - "provider": 1, - "id": "http://h5p.example.com/expapi/verbs/interacted", - "consented": true, - "objects": "[{\"id\":\"http://h5p.example.com/expapi/activity/ofN2ODcnLRaVu30lUpzrWPqF2AcG7g46\",\"label\":\"1.2.3 Kappenventil\",\"defaultConsent\":true,\"definition\":{\"name\":{\"enUS\":\"1.2.3 Kappenventil\"}},\"consented\":true}]" - }, - { - "provider": 1, - "id": "http://h5p.example.com/expapi/verbs/answered", - "consented": false, - "objects": "[{\"id\":\"http://h5p.example.com/expapi/activity/K34IszYvGE4R0cC72Ean6msLfLCJtQ8b\",\"label\":\"7.2.1 Ventil Basics\",\"defaultConsent\":false,\"definition\":{\"name\":{\"enUS\":\"7.2.1 Ventil Basics\"}},\"consented\":false}]" - } + "groups": [ + 1, + 2, + 5 ] } ``` diff --git a/src/consents/views.py b/src/consents/views.py index c4731ba..dc09367 100644 --- a/src/consents/views.py +++ b/src/consents/views.py @@ -806,23 +806,9 @@ class GetUsersConsentsThirdPartyView(APIView): status=status.HTTP_200_OK, ) else: - accepted_provider_schemas = user.accepted_provider_schemas.all() - provider_schemas = list( - filter( - lambda schema: schema.provider.id == int(provider.provider.id), - accepted_provider_schemas, - ) - ) - - provider_schema = ( - ProviderSchemaSerializer(provider_schemas, many=True).data[0] - if len(provider_schemas) > 0 - else None - ) return JsonResponse( { "consent": user_consent, - "provider_schema": provider_schema, "paused_data_recording": user.paused_data_recording, }, safe=False, -- GitLab From ef4ae00a9c2f87afb8bcccec2d4d1ce77a5e25ba Mon Sep 17 00:00:00 2001 From: Lennard Strohmeyer <lennard.strohmeyer@digitallearning.gmbh> Date: Thu, 27 Mar 2025 17:17:06 +0100 Subject: [PATCH 04/12] #155: fixed data disclosure tests for environments where no mongodb is available or the LRS is empty --- src/data_disclosure/tests/tests_data_disclosure.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/data_disclosure/tests/tests_data_disclosure.py b/src/data_disclosure/tests/tests_data_disclosure.py index d338e4f..88d0add 100644 --- a/src/data_disclosure/tests/tests_data_disclosure.py +++ b/src/data_disclosure/tests/tests_data_disclosure.py @@ -1,6 +1,7 @@ import json import os from io import StringIO +from unittest.mock import patch from django.conf import settings from django.core.files.base import ContentFile @@ -14,6 +15,8 @@ from providers.models import ProviderAuthorization, Provider, ProviderVerbGroup, from users.models import CustomUser from data_disclosure.models import DataDisclosure +from data_disclosure.tasks import DataDisclosureProcessor + class TestsDataDisclosure(BaseTestCase): @@ -35,6 +38,10 @@ class TestsDataDisclosure(BaseTestCase): }, ] + @patch.object(DataDisclosureProcessor, 'get_xapi_statements') + def mock_get_xapi_statements(self, user_email: str): + return [] + def setUp(self): call_command('check_and_apply_migrations') normal_user = CustomUser.objects.create_user( -- GitLab From ca0c26355f99ee1a2c748ce4c81721d417d7ed22 Mon Sep 17 00:00:00 2001 From: Lennard Strohmeyer <lennard.strohmeyer@digitallearning.gmbh> Date: Mon, 31 Mar 2025 08:27:47 +0200 Subject: [PATCH 05/12] pipeline fix --- .../tests/tests_data_disclosure.py | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/data_disclosure/tests/tests_data_disclosure.py b/src/data_disclosure/tests/tests_data_disclosure.py index 88d0add..d514510 100644 --- a/src/data_disclosure/tests/tests_data_disclosure.py +++ b/src/data_disclosure/tests/tests_data_disclosure.py @@ -15,7 +15,6 @@ from providers.models import ProviderAuthorization, Provider, ProviderVerbGroup, from users.models import CustomUser from data_disclosure.models import DataDisclosure -from data_disclosure.tasks import DataDisclosureProcessor @@ -38,10 +37,6 @@ class TestsDataDisclosure(BaseTestCase): }, ] - @patch.object(DataDisclosureProcessor, 'get_xapi_statements') - def mock_get_xapi_statements(self, user_email: str): - return [] - def setUp(self): call_command('check_and_apply_migrations') normal_user = CustomUser.objects.create_user( @@ -106,16 +101,19 @@ class TestsDataDisclosure(BaseTestCase): ) self.assertEqual(response.status_code, 200) - - def test_create_job(self): + @patch('celery.app.task.Task.apply_async') + def test_create_job(self, celery_mock): self.assertEqual(DataDisclosure.objects.count(), 0) response = self.user_client.post("/api/v1/data-disclosure/create") self.assertEqual(response.status_code, 200) self.assertEqual(DataDisclosure.objects.count(), 1) + self.assertTrue(celery_mock.called) + - def test_list_jobs(self): + @patch('celery.app.task.Task.apply_async') + def test_list_jobs(self, celery_mock): response = self.user_client.get("/api/v1/data-disclosure/list") self.assertEqual(response.status_code, 200) data = json.loads(response.content.decode()) @@ -123,13 +121,15 @@ class TestsDataDisclosure(BaseTestCase): response = self.user_client.post("/api/v1/data-disclosure/create") self.assertEqual(response.status_code, 200) + self.assertTrue(celery_mock.called) response = self.user_client.get("/api/v1/data-disclosure/list") self.assertEqual(response.status_code, 200) data = json.loads(response.content.decode()) self.assertEqual(len(data), 1) - def test_get_secret(self): + @patch('celery.app.task.Task.apply_async') + def test_get_secret(self, celery_mock): response = self.user_client.get("/api/v1/data-disclosure/list") self.assertEqual(response.status_code, 200) data = json.loads(response.content.decode()) @@ -137,6 +137,7 @@ class TestsDataDisclosure(BaseTestCase): response = self.user_client.post("/api/v1/data-disclosure/create") self.assertEqual(response.status_code, 200) + self.assertTrue(celery_mock.called) # create second job from another account response = self.provider_client.post("/api/v1/data-disclosure/create") @@ -162,8 +163,8 @@ class TestsDataDisclosure(BaseTestCase): response = self.user_client.get(f"/api/v1/data-disclosure/file_secret/{data[0]['id']}") # note the "user_client" self.assertEqual(response.status_code, 403) - - def test_get_file(self): + @patch('celery.app.task.Task.apply_async') + def test_get_file(self, celery_mock): response = self.user_client.get("/api/v1/data-disclosure/list") self.assertEqual(response.status_code, 200) data = json.loads(response.content.decode()) @@ -171,7 +172,7 @@ class TestsDataDisclosure(BaseTestCase): response = self.user_client.post("/api/v1/data-disclosure/create") self.assertEqual(response.status_code, 200) - + self.assertTrue(celery_mock.called) response = self.user_client.get("/api/v1/data-disclosure/list") self.assertEqual(response.status_code, 200) @@ -206,7 +207,8 @@ class TestsDataDisclosure(BaseTestCase): response = self.user_client.get(f"/api/v1/data-disclosure/files/{file_id}/{secret}") self.assertEqual(response.status_code, 200) - def test_delete_file(self): + @patch('celery.app.task.Task.apply_async') + def test_delete_file(self, celery_mock): response = self.user_client.get("/api/v1/data-disclosure/list") self.assertEqual(response.status_code, 200) data = json.loads(response.content.decode()) @@ -214,7 +216,7 @@ class TestsDataDisclosure(BaseTestCase): response = self.user_client.post("/api/v1/data-disclosure/create") self.assertEqual(response.status_code, 200) - + self.assertTrue(celery_mock.called) response = self.user_client.get("/api/v1/data-disclosure/list") self.assertEqual(response.status_code, 200) -- GitLab From d8f50410fac57a6fac75a122b922b49bdda973cb Mon Sep 17 00:00:00 2001 From: Lennard Strohmeyer <lennard.strohmeyer@digitallearning.gmbh> Date: Mon, 31 Mar 2025 08:40:34 +0200 Subject: [PATCH 06/12] pipeline fix --- src/data_disclosure/tests/tests_data_disclosure.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/data_disclosure/tests/tests_data_disclosure.py b/src/data_disclosure/tests/tests_data_disclosure.py index d514510..b32ca89 100644 --- a/src/data_disclosure/tests/tests_data_disclosure.py +++ b/src/data_disclosure/tests/tests_data_disclosure.py @@ -190,9 +190,12 @@ class TestsDataDisclosure(BaseTestCase): secret = data["secret"] # create zip file to fulfil requirements + data_disclosure_location = settings.DATA_DISCLOSURE_LOCATION + if not data_disclosure_location or data_disclosure_location is None or data_disclosure_location == "": + data_disclosure_location = "./data_disclosure_zips" empty_zip_data = ContentFile(b'PK\x05\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00') file_name = default_storage.save( - f"{settings.DATA_DISCLOSURE_LOCATION}/{file_id}.zip", empty_zip_data + f"{data_disclosure_location}/{file_id}.zip", empty_zip_data ) disc = DataDisclosure.objects.get(id=file_id) @@ -238,9 +241,12 @@ class TestsDataDisclosure(BaseTestCase): self.assertEqual(response.status_code, 403) # create zip file to fulfil requirements + data_disclosure_location = settings.DATA_DISCLOSURE_LOCATION + if not data_disclosure_location or data_disclosure_location is None or data_disclosure_location == "": + data_disclosure_location = "./data_disclosure_zips" empty_zip_data = ContentFile(b'PK\x05\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00') file_name = default_storage.save( - f"{settings.DATA_DISCLOSURE_LOCATION}/{file_id}.zip", empty_zip_data + f"{data_disclosure_location}/{file_id}.zip", empty_zip_data ) disc = DataDisclosure.objects.get(id=file_id) -- GitLab From edcfd8459532e9d73cc82acb2b92f0a6a1d9ab95 Mon Sep 17 00:00:00 2001 From: Lennard Strohmeyer <lennard.strohmeyer@digitallearning.gmbh> Date: Mon, 31 Mar 2025 08:59:32 +0200 Subject: [PATCH 07/12] pipeline fix --- src/backend/settings.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/backend/settings.py b/src/backend/settings.py index 9dcb9c9..31a0c99 100644 --- a/src/backend/settings.py +++ b/src/backend/settings.py @@ -11,6 +11,7 @@ https://docs.djangoproject.com/en/4.1/ref/settings/ """ import mimetypes import os +import sys from pathlib import Path import environ @@ -135,11 +136,12 @@ else: # Cache(s) # https://docs.djangoproject.com/en/4.1/topics/cache/ +use_file_based_cache = 'test' in sys.argv or os.getenv('DJANGO_TEST_ENV') or env("CACHE_BACKEND", default="file") != "redis" CACHES = { 'default': { - 'BACKEND': 'django.core.cache.backends.redis.RedisCache' if env("CACHE_BACKEND", default="file") == "redis" + 'BACKEND': 'django.core.cache.backends.redis.RedisCache' if not use_file_based_cache else 'django.core.cache.backends.filebased.FileBasedCache', - 'LOCATION': '/tmp/django_cache' if env("CACHE_BACKEND", default="file") == 'file' else env("CACHE_URI", default='redis://127.0.0.1:6379') , + 'LOCATION': '/tmp/django_cache' if use_file_based_cache else env("CACHE_URI", default='redis://127.0.0.1:6379') , } } -- GitLab From a862d36ed6e614e05e80cb74e8fe96f8b3156db3 Mon Sep 17 00:00:00 2001 From: Benjamin Ledel <benjamin@schule-plus.com> Date: Mon, 31 Mar 2025 14:00:00 +0200 Subject: [PATCH 08/12] *make worker adjustable #164 --- docs/docs/deployment.md | 15 +++++++++++++++ src/entrypoint.sh | 16 ++++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/docs/docs/deployment.md b/docs/docs/deployment.md index e60de0f..d6d1f5c 100644 --- a/docs/docs/deployment.md +++ b/docs/docs/deployment.md @@ -372,3 +372,18 @@ Content `traefik.toml` statusCodes = ["200", "300-302"] ``` +## Adjust deployment + +### Number of gunicorn workers +The number of Gunicorn workers can be adjusted using the WORKERS environment variable. +By default, the application starts with 4 workers. You can override this value during deployment: + +```docker-compose.yml +services: + web: + image: registry.git.rwth-aachen.de/polaris/rights-engine/rights-engine:${POLARIS_VERSION} + ports: + - "80:80" + environment: + - WORKERS=10 +``` \ No newline at end of file diff --git a/src/entrypoint.sh b/src/entrypoint.sh index 1fd51c8..7be3fd2 100644 --- a/src/entrypoint.sh +++ b/src/entrypoint.sh @@ -1,7 +1,19 @@ #!/bin/sh +# Set default number of workers to 4 if not provided via environment +: "${WORKERS:=4}" + echo "Running database migrations and role setup..." python manage.py check_and_apply_migrations -echo "Starting Gunicorn server..." -exec env "$@" gunicorn --bind :80 --workers 3 --timeout 240 --access-logfile - --error-logfile - backend.wsgi +echo "Starting Gunicorn server with $WORKERS workers..." +exec env "$@" gunicorn \ + --bind :80 \ + --workers "$WORKERS" \ + --timeout 300 \ + --graceful-timeout 300 \ + --max-requests 1000 \ + --max-requests-jitter 50 \ + --access-logfile - \ + --error-logfile - \ + backend.wsgi -- GitLab From f5e7091c4a3344ed2151d2212741d69d0e9916f7 Mon Sep 17 00:00:00 2001 From: Lennard Strohmeyer <lennard.strohmeyer@digitallearning.gmbh> Date: Mon, 31 Mar 2025 15:15:41 +0200 Subject: [PATCH 09/12] #165: only update relevant fields, excluding `updated`-timestamp, in migration --- ...005_remove_userconsents_provider_schema_and_more.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/consents/migrations/0005_remove_userconsents_provider_schema_and_more.py b/src/consents/migrations/0005_remove_userconsents_provider_schema_and_more.py index 2dffac1..feb994a 100644 --- a/src/consents/migrations/0005_remove_userconsents_provider_schema_and_more.py +++ b/src/consents/migrations/0005_remove_userconsents_provider_schema_and_more.py @@ -1,4 +1,4 @@ -from django.db import migrations, models +from django.db import migrations, models, connection import django.db.models.deletion def migrate_user_consent_to_verb(apps, schema_editor): @@ -8,10 +8,12 @@ def migrate_user_consent_to_verb(apps, schema_editor): schema = consent.provider_schema verb_id = consent.verb verb = Verb.objects.get(verb_id=verb_id, provider_schema=schema) - consent.verb = verb.id + #consent.verb = verb.id group = verb.providerverbgroup_set.first() - consent.verb_group_id = group.id - consent.save() + #consent.verb_group_id = group.id + #consent.save() + with connection.cursor() as cursor: + cursor.execute("UPDATE consents_userconsents SET verb = %s, verb_group_id = %s WHERE id = %s", [verb.id, group.id, consent.id]) class Migration(migrations.Migration): -- GitLab From 0025c1c5a4d2b7ecf94572829e4db0f143f47d42 Mon Sep 17 00:00:00 2001 From: Benjamin Ledel <benjamin@schule-plus.com> Date: Mon, 31 Mar 2025 18:05:03 +0200 Subject: [PATCH 10/12] * adjust index creation for lrs --- src/backend/management/commands/create_mongo_index.py | 9 +++++++++ src/consents/tests/tests_consent_operations.py | 1 + src/entrypoint.sh | 3 +++ 3 files changed, 13 insertions(+) diff --git a/src/backend/management/commands/create_mongo_index.py b/src/backend/management/commands/create_mongo_index.py index a5edf0a..848a24d 100644 --- a/src/backend/management/commands/create_mongo_index.py +++ b/src/backend/management/commands/create_mongo_index.py @@ -14,6 +14,7 @@ class Command(BaseCommand): {"key": [("name", ASCENDING)], "name": "name_1"}, {"key": [("created_at", DESCENDING)], "name": "created_at_-1"}, {"key": [("name", ASCENDING), ("created_at", DESCENDING)], "name": "name_1_created_at_-1"}, + {"key": [("context_id", HASHED), ("name", ASCENDING), ("created_at", DESCENDING)], "name": "context_name_1_created_at_-1"}, ], "statement": [ {"key": [("_id", ASCENDING)], "name": "_id_"}, @@ -25,6 +26,14 @@ class Command(BaseCommand): # Iterate through each collection and create indexes for collection_name, indexes in collections_indexes.items(): + if collection_name not in lrs_db.list_collection_names(): + self.stdout.write(self.style.WARNING( + f"Collection '{collection_name}' does not exist. Creating collection..." + )) + # Force collection creation by inserting and deleting a dummy doc + lrs_db[collection_name].insert_one({"_init": True}) + lrs_db[collection_name].delete_one({"_init": True}) + collection = lrs_db[collection_name] self.stdout.write(self.style.SUCCESS(f"Creating indexes for collection: {collection_name}")) for index in indexes: diff --git a/src/consents/tests/tests_consent_operations.py b/src/consents/tests/tests_consent_operations.py index 72887c8..93dc869 100644 --- a/src/consents/tests/tests_consent_operations.py +++ b/src/consents/tests/tests_consent_operations.py @@ -244,6 +244,7 @@ class BaseTestCase(TransactionTestCase): def setUp(self): call_command('check_and_apply_migrations') + call_command('create_mongo_index') normal_user = CustomUser.objects.create_user( self.test_user_email, self.test_user_password ) diff --git a/src/entrypoint.sh b/src/entrypoint.sh index 7be3fd2..47930e5 100644 --- a/src/entrypoint.sh +++ b/src/entrypoint.sh @@ -6,6 +6,9 @@ echo "Running database migrations and role setup..." python manage.py check_and_apply_migrations +echo "Running mongodb check..." +python manage.py create_mongo_index + echo "Starting Gunicorn server with $WORKERS workers..." exec env "$@" gunicorn \ --bind :80 \ -- GitLab From 73973b481c23934c270db0f912575a9a5e95f97f Mon Sep 17 00:00:00 2001 From: Benjamin Ledel <benjamin@schule-plus.com> Date: Mon, 31 Mar 2025 18:17:40 +0200 Subject: [PATCH 11/12] * fix tests --- src/consents/tests/tests_consent_operations.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/consents/tests/tests_consent_operations.py b/src/consents/tests/tests_consent_operations.py index 93dc869..72887c8 100644 --- a/src/consents/tests/tests_consent_operations.py +++ b/src/consents/tests/tests_consent_operations.py @@ -244,7 +244,6 @@ class BaseTestCase(TransactionTestCase): def setUp(self): call_command('check_and_apply_migrations') - call_command('create_mongo_index') normal_user = CustomUser.objects.create_user( self.test_user_email, self.test_user_password ) -- GitLab From 5ec6eaac7294a4ac9f7c774f0d5d01c97948c3cc Mon Sep 17 00:00:00 2001 From: Lennard Strohmeyer <lennard.strohmeyer@digitallearning.gmbh> Date: Tue, 1 Apr 2025 12:46:08 +0200 Subject: [PATCH 12/12] #162: return empty groups list to frontend when user has not declared any consents --- src/consents/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/consents/views.py b/src/consents/views.py index dc09367..b6e5703 100644 --- a/src/consents/views.py +++ b/src/consents/views.py @@ -506,6 +506,7 @@ class GetUserConsentHistoryView(APIView): except ObjectDoesNotExist: return JsonResponse( { + "groups": [], "message": "User has no consent declaration record.", "no_consent_record": True, }, -- GitLab