diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 40860361dfee52f3d279c22eb84e59c144931f83..5efce1f4f8c9c131a9cb974f732f9e7d85ec5786 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -71,8 +71,13 @@ deploy_rights-engine: - exec_ssh "cd $BASE_DIR/$URL/rights-engine; echo 'JWT_PUBLIC_KEY_PATH=backend/id_rsa.pub' >> .env" - exec_ssh "cd $BASE_DIR/$URL/rights-engine; echo 'JWT_PRIVATE_KEY_PATH=backend/id_rsa' >> .env" - exec_ssh "cd $BASE_DIR/$URL/rights-engine; echo 'IDP_SERVER=$IDP_SERVER' >> .env" - - exec_ssh "cd $BASE_DIR/$URL/rights-engine; echo 'IDP_ENABLED=false' >> .env" - - exec_ssh "cd $BASE_DIR/$URL/rights-engine; echo 'THIRD_PARTY_ACCESS_TOKEN=$THIRD_PARTY_ACCESS_TOKEN' >> .env" + - exec_ssh "cd $BASE_DIR/$URL/rights-engine; echo 'IDP_ENABLED=False' >> .env" + - exec_ssh "cd $BASE_DIR/$URL/rights-engine; echo 'SP_HOST=$SP_HOST' >> .env" + - exec_ssh "cd $BASE_DIR/$URL/rights-engine; echo 'PROVIDER_CONTEXTS_PULL_ENABLED=True' >> .env" + - exec_ssh "cd $BASE_DIR/$URL/rights-engine; echo 'PROVIDER_CONTEXTS_PULL_URL=$PROVIDER_CONTEXTS_PULL_URL' >> .env" + - exec_ssh "cd $BASE_DIR/$URL/rights-engine; echo 'PROVIDER_CONTEXTS_PULL_TOKEN=$PROVIDER_CONTEXTS_PULL_TOKEN' >> .env" + - exec_ssh "cd $BASE_DIR/$URL/rights-engine; echo 'ANONYMIZATION_DEFAULT_MINIMUM_COUNT=10' >> .env" + - exec_ssh "cd $BASE_DIR/$URL/rights-engine; echo 'ANONYMIZATION_HASH_PREFIX=change_me' >> .env" script: - function exec_ssh () { ssh -o StrictHostKeyChecking=no $SSH_USER@$DEPLOYMENT_URL $1; } @@ -118,6 +123,8 @@ deploy_analytics-engine: - exec_ssh "cd $BASE_DIR/$URL/analytics-engine; echo 'POSTGRES_USER=postgres' >> .env" - exec_ssh "cd $BASE_DIR/$URL/analytics-engine; echo 'POSTGRES_DATABASE=analytics-engine' >> .env" - exec_ssh "cd $BASE_DIR/$URL/analytics-engine; echo 'POSTGRES_PASSWORD=$POSTGRES_PASSWORD' >> .env" + - exec_ssh "cd $BASE_DIR/$URL/analytics-engine; echo 'UID=$UID' >> .env" + - exec_ssh "cd $BASE_DIR/$URL/analytics-engine; echo 'GID=$GID' >> .env" # Image Pull & Docker Restart - exec_ssh "cd $BASE_DIR/$URL/analytics-engine; docker compose pull && docker compose down && docker compose up -d && sleep 30 && docker compose exec -it scheduler sh -c 'scheduler create-db' && docker compose exec -it scheduler sh -c 'scheduler read-configs'" diff --git a/README.md b/README.md index 1c436dbb7554ff36fd01ac81bafb83c382e23b2c..9cf8ca9e5b79c2da0b7fe9c19ee00e860d762c07 100644 --- a/README.md +++ b/README.md @@ -21,71 +21,121 @@ end # Getting started Each folder contains one of the components of -## Login in the docker registry +## Local Deployment + +### Login in the docker registry If you are not already logged in, you need to authenticate to the Container Registry by using your GitLab username and password. If you have Two-Factor Authentication enabled, use a Personal Access Token instead of a password. ```bash docker login registry.digitallearning.gmbh ``` -## LRS / MongoDB -First we need a database or LRS to store the xAPI Statements. In a local setup, you can use the docker container in the mongodb folder. -Rename or copy the sample configuration. Please change all passwords in the configuration file. +### Create Docker network +Different containers communicate via a Docker network `web`, which must be created before Docker compose files are executed. +```bash +docker network create web +``` + +### Create private/public keys and .env + +Please change all passwords in the configuration file and leave the passphrase empty. ```bash -cd mongodb +cd single-compose +ssh-keygen -b 4096 -f id_rsa cp .env.sample .env ``` -Now you can start the mongodb with the following command + +### Start containers + ```bash docker compose up -d ``` -Please check that the database is created: +Please check whether all services started successfully. ```bash docker compose ps ``` +After that you should be able to visit http://localhost:8004/ and see the rights engine. -## Rights engine -First create some jwt keys with the following command: +### Migrate rights-engine database (only required once after first start) ```bash -cd rights-engine -ssh-keygen -t rsa -b 4096 -m PEM -f id_rsa -openssl rsa -in id_rsa -pubout -outform PEM -out id_rsa.pub +docker compose exec -it rights-engine sh -c 'python3 manage.py sqlflush | sed s/TRUNCATE/DROP\ TABLE\ IF\ EXISTS/g | python3 manage.py dbshell && echo DROP\ TABLE\ IF\ EXISTS\ django_migrations\; | python3 manage.py dbshell && python3 manage.py migrate && python3 manage.py loaddata fixtures/initial_db.json' ``` -Leave the passphrase empty. -Rename or copy the sample configuration. Please change all passwords in the configuration file. +### Migrate analytics-engine database (only required once after first start) + ```bash -cp .env.sample .env +docker compose exec -it scheduler sh -c 'scheduler create-db' ``` -Now you can start the rights engine with the following command -```bash -docker compose up -d +### Adding analytics engine jobs + +Analytics engines jobs are configured via yml files and read from the `configuration` directory, which is a volume. + +Example configuration file `configuration/h5p_engines.yml` +```yml +h5p_statements_count_engine: + crontab: "*/1 * * * *" + repo: "https://scheduler:glpat-MsDsrHMH-k3-DzEfNRgk@gitlab.digitallearning.gmbh/polaris/engines/dummy-engine.git" + analytics_token: "b6a4ec069ef9f688e781161d46c2a85c14a761a4eaf0074099656c7de44a65d9" +``` + +Example configuration file `configuration/moodle_engines.yml` +```yml +moodle_statements_count_engine: + crontab: "*/1 * * * *" + repo: "https://scheduler:glpat-MsDsrHMH-k3-DzEfNRgk@gitlab.digitallearning.gmbh/polaris/engines/dummy-engine.git" + analytics_token: "0482a0f3259c966dfddb38de867573a95995ee5e10898bb71f9ae0e99efe54ef" ``` -Please check that all services are started: +Update analytics engine scheduler ```bash -docker compose ps +docker compose exec -it scheduler sh -c 'scheduler read-configs' ``` -After that you should be able to visit http://localhost:80/ and see the rights engine. +### Create visualization token -## Analytics engine -Rename or copy the sample configuration. Please change all passwords in the configuration file. ```bash -cd analytics-engine -cp .env.sample .env +curl -X POST http://localhost:8004/api/v1/provider/visualization-tokens/create --data '{"engines": ["count_h5p_statements", "count_moodle_statements"]}' -H "Content-Type: application/json" ``` -Now you can start the analytics engine with the following command -```bash -docker compose up -d +Returns JWT Token for dashboard +``` +{"token":[JWT_TOKEN]"} +``` + +### Loading JSON statements + +It is recommended to import a set of sample statements into the LRS so that the analytics engines can work on this data. Furhtermore, users (user1@polaris.com and user2@polaris.com) can test the data disclousure and data deletion process. + +```console +$ docker compose exec -it mongodb_container sh -c 'mongoimport --authenticationDatabase admin --username root --password CHANGE_ME --db lrs --collection statements --drop --file ./lrs_statements_dump.json' ``` +### Start dashboard + +1. Clone  + - `cd dashboard-example` +2. Download latest `@polaris/dashboard-sdk-X.X.X.tgz` from registry and `npm install @polaris/dashboard-sdk-1.0.2.tgz` it (TODO: improve with npm login) +2. Update `TOKEN` in `dashboard-example/src/js/app.js` +3. Run `npm run dev` +3. Visit http://localhost:8005/ + +### (Optional) Filling DB with random statements + +1. Clone rights-engine +2. Create provider config +```bash +$ cd rights-engine/tools/xapi-statement-generator +$ cp provider_config.json.example provider_config.json +```` +3. Open `provider_config.json` and insert Application Tokens (visible at http://localhost:8004 (Rights Engine UI), if you login as a provider) +4. Run `python generator.py -t http://localhost:8003/xapi/statements -r` + + ## Update Docker Images ```bash docker compose pull -``` \ No newline at end of file +``` diff --git a/analytics-engine/docker-compose.yml b/analytics-engine/docker-compose.yml index 70bdcaa660b5e000f32c7935bab6dcb7443196eb..2e83d97505048f1a35f9c53cade19c5547b80627 100644 --- a/analytics-engine/docker-compose.yml +++ b/analytics-engine/docker-compose.yml @@ -20,7 +20,7 @@ services: scheduler: user: "${UID}:${GID}" - image: registry.git.rwth-aachen.de/polaris/analytics-engine/scheduler:latest + image: registry.digitallearning.gmbh/polaris/analytics-engine/scheduler:latest restart: unless-stopped expose: - 8000 @@ -42,7 +42,7 @@ services: beat: user: "${UID}:${GID}" - image: registry.git.rwth-aachen.de/polaris/analytics-engine/scheduler:latest + image: registry.digitallearning.gmbh/polaris/analytics-engine/scheduler:latest restart: unless-stopped entrypoint: celery command: -A scheduler.worker beat -l info --scheduler celery_sqlalchemy_scheduler.schedulers:DatabaseScheduler @@ -57,7 +57,7 @@ services: worker: user: "${UID}:${GID}" - image: registry.git.rwth-aachen.de/polaris/analytics-engine/scheduler:latest + image: registry.digitallearning.gmbh/polaris/analytics-engine/scheduler:latest restart: unless-stopped entrypoint: celery command: -A scheduler.worker worker -l info diff --git a/rights-engine/.env.sample b/rights-engine/.env.sample index af67df1bbaf6dae1fac14b57ab01800685c9b112..1741657df8dc7686d484ef05840b59cf7a066a33 100644 --- a/rights-engine/.env.sample +++ b/rights-engine/.env.sample @@ -24,6 +24,6 @@ JWT_PRIVATE_KEY_PATH=backend/id_rsa IDP_SERVER=https://aai-test-v3.ruhr-uni-bochum.de IDP_ENABLED=false SP_HOST=polaris.digitallearning.gmbh -PROVIDER_CONTEXTS_PULL_ENABLED=False -PROVIDER_CONTEXTS_PULL_URL=https://example.com -PROVIDER_CONTEXTS_PULL_TOKEN=CHANGE_ME +ANONYMIZATION_HASH_PREFIX=anon +ANONYMIZATION_DEFAULT_MINIMUM_COUNT=10 + diff --git a/rights-engine/docker-compose.yml b/rights-engine/docker-compose.yml index 33534b15f3ebcc5494ab6c35e53cb423735d9fb5..b446484a39140e657916a1df21df6d772c59626d 100644 --- a/rights-engine/docker-compose.yml +++ b/rights-engine/docker-compose.yml @@ -20,7 +20,7 @@ services: - internal rights-engine: - image: registry.git.rwth-aachen.de/polaris/rights-engine/rights-engine:latest + image: registry.digitallearning.gmbh/polaris/rights-engine/rights-engine restart: unless-stopped expose: - 80 @@ -45,7 +45,7 @@ services: - "./data_disclosure_zips/:/usr/src/app/backend/data_disclosure_zips" beat: - image: registry.git.rwth-aachen.de/polaris/rights-engine/rights-engine:latest + image: registry.digitallearning.gmbh/polaris/rights-engine/rights-engine restart: unless-stopped entrypoint: celery command: -A backend beat -l info @@ -63,7 +63,7 @@ services: - "./id_rsa:/usr/src/app/backend/id_rsa" worker: - image: registry.git.rwth-aachen.de/polaris/rights-engine/rights-engine:latest + image: registry.digitallearning.gmbh/polaris/rights-engine/rights-engine restart: unless-stopped entrypoint: celery command: -A backend worker --loglevel=debug diff --git a/single-compose/.env.sample b/single-compose/.env.sample index 7537613085232c1522a1f58c4760755a67a35013..202af3f284a4c77c298d7a5d94471392aff2c80b 100644 --- a/single-compose/.env.sample +++ b/single-compose/.env.sample @@ -33,6 +33,6 @@ POSTGRES_USER=postgres POSTGRES_DATABASE=analytics-engine POSTGRES_PASSWORD=CHANGE_ME SP_HOST=polaris.ruhr-uni-bochum.de -PROVIDER_CONTEXTS_PULL_ENABLED=False -PROVIDER_CONTEXTS_PULL_URL=https://example.com -PROVIDER_CONTEXTS_PULL_TOKEN=CHANGE_ME +ANONYMIZATION_HASH_PREFIX=anon +ANONYMIZATION_DEFAULT_MINIMUM_COUNT=10 + diff --git a/single-compose/configuration/h5p_engines.yml b/single-compose/configuration/h5p_engines.yml index a594286fe2792c9bbe7d2c5335f116e135160e08..85eb85b823685a08abbd56663259fd87035b7365 100644 --- a/single-compose/configuration/h5p_engines.yml +++ b/single-compose/configuration/h5p_engines.yml @@ -1,4 +1,4 @@ h5p_statements_count_engine: crontab: "*/1 * * * *" - repo: "https://scheduler:glpat-MsDsrHMH-k3-DzEfNRgk@gitlab.digitallearning.gmbh/polaris/engines/activities.git" + repo: "https://scheduler:glpat-MsDsrHMH-k3-DzEfNRgk@gitlab.digitallearning.gmbh/polaris/engines/dummy-engine.git" analytics_token: "b6a4ec069ef9f688e781161d46c2a85c14a761a4eaf0074099656c7de44a65d9" diff --git a/single-compose/docker-compose.yml b/single-compose/docker-compose.yml index 8bf50a41bdff073a5ed7a8ed3db09cd70445e750..2362f0190405fe8609a91469fea9b8dc62481441 100644 --- a/single-compose/docker-compose.yml +++ b/single-compose/docker-compose.yml @@ -27,7 +27,7 @@ services: command: redis-server --requirepass ${REDIS_PASSWORD} rights-engine: - image: registry.git.rwth-aachen.de/polaris/entwicklung/rights-engine/rights-engine:latest + image: registry.digitallearning.gmbh/polaris/rights-engine/rights-engine:latest restart: always ports: - 8004:80 @@ -44,7 +44,7 @@ services: - "./data_disclosure_zips/:/usr/src/app/backend/data_disclosure_zips" beat: - image: registry.git.rwth-aachen.de/polaris/entwicklung/rights-engine/rights-engine:latest + image: registry.digitallearning.gmbh/polaris/rights-engine/rights-engine:latest restart: always entrypoint: celery command: -A backend beat -l info @@ -60,7 +60,7 @@ services: - "./id_rsa:/usr/src/app/backend/id_rsa" worker: - image: registry.git.rwth-aachen.de/polaris/entwicklung/rights-engine/rights-engine:latest + image: registry.digitallearning.gmbh/polaris/rights-engine/rights-engine:latest restart: always entrypoint: celery command: -A backend worker --loglevel=debug @@ -94,7 +94,7 @@ services: command: redis-server --requirepass ${REDIS_PASSWORD} scheduler: - image: registry.git.rwth-aachen.de/polaris/entwicklung/analytics-engine/scheduler:latest + image: registry.digitallearning.gmbh/polaris/analytics-engine/scheduler:latest restart: always ports: - 8000:8000 @@ -108,7 +108,7 @@ services: - "./configuration:/app/configuration" beat_analytics: - image: registry.git.rwth-aachen.de/polaris/entwicklung/analytics-engine/scheduler:latest + image: registry.digitallearning.gmbh/polaris/analytics-engine/scheduler:latest restart: always entrypoint: celery command: -A scheduler.worker beat -l info --scheduler celery_sqlalchemy_scheduler.schedulers:DatabaseScheduler @@ -122,7 +122,7 @@ services: - "./configuration:/app/configuration" worker_analytics: - image: registry.git.rwth-aachen.de/polaris/entwicklung/analytics-engine/scheduler:latest + image: registry.digitallearning.gmbh/polaris/analytics-engine/scheduler:latest restart: always entrypoint: celery command: -A scheduler.worker worker -l info @@ -137,7 +137,7 @@ services: - "./configuration:/app/configuration" dashboard: - image: registry.git.rwth-aachen.de/polaris/dashboard-example/dashboard-example:latest + image: registry.digitallearning.gmbh/polaris/dashboard-example/dashboard-example:latest restart: always ports: - 8005:80