diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 27a83241908afd47acd09a26d29384029d7287c9..c1c6843900e2a0af2a9c0d6e5116a1d7ec00c82f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,7 @@ stages: # - test - deploy -variables: +variables: &global-variables URL: $URL DEPLOYMENT_URL: "polaris.servicechain.cloud" IDP_SERVER: "https://aai-test-v3.ruhr-uni-bochum.de" @@ -11,12 +11,25 @@ variables: SSH_USER: "root" SSH_PRIVATE_KEY: $SSH_PRIVATE_KEY BRANCH : "main" + +.deploy: &deploy + stage: deploy + + before_script: + - echo "setting up ssh and rsync" + # Install ssh-agent if not already installed + - which ssh-agent || (apk --update add openssh-client rsync) + - eval `ssh-agent -s` + - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null + - function exec_ssh () { ssh -o StrictHostKeyChecking=no $SSH_USER@$DEPLOYMENT_URL $1; } + # Login in Container Registry + - exec_ssh "docker login -u \"${CI_REGISTRY_USER}\" -p \"${CI_REGISTRY_PASSWORD}\" \"${CI_REGISTRY}\"" -deploy_app: +deploy_rights-engine: stage: deploy allow_failure: false environment: - name: prod + name: rights-engine url: $URL before_script: - echo "setting up ssh" @@ -25,39 +38,39 @@ deploy_app: - eval `ssh-agent -s` - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null - function exec_ssh () { ssh -o StrictHostKeyChecking=no $SSH_USER@$DEPLOYMENT_URL $1; } - - exec_ssh "if [ -d '$BASE_DIR' ]; then cd $BASE_DIR; git remote set-url origin https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.digitallearning.gmbh/polaris/deployment; git pull origin $BRANCH; else cd $BASE_DIR; git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.digitallearning.gmbh/polaris/deployment; cd $BASE_DIR; git checkout $BRANCH; git pull origin $BRANCH; fi;" + - exec_ssh "if [ -d '$BASE_DIR/$WEBSITE' ]; then cd $BASE_DIR/$WEBSITE; git remote set-url origin ${CI_REPOSITORY_URL}; git pull; else cd $BASE_DIR; git clone ${CI_REPOSITORY_URL} $WEBSITE; cd $BASE_DIR/$WEBSITE; git checkout ${CI_COMMIT_BRANCH}; git pull; fi;" # ENV Docker-Compose - - exec_ssh "mkdir -p data_disclosure_zips" - - exec_ssh "rm -rf data_disclosure_zips/*" - - exec_ssh "echo 'URL=$URL' > $BASE_DIR/.env" - - exec_ssh "echo 'MYSQL_ROOT_PASSWORD=$MYSQL_ROOT_PASSWORD' >> $BASE_DIR/.env" - - exec_ssh "echo 'WEBSITE=polaris' | sed 's/\./-/g' >> $BASE_DIR/.env" - - exec_ssh "echo 'DB_PASSWORD=$DB_PASSWORD' >> $BASE_DIR/.env" - - exec_ssh "echo 'DJANGO_DEVELOPMENT=false' >> $BASE_DIR/.env" - - exec_ssh "echo 'DATABASE_NAME=polaris' | sed 's/\./-/g' >> $BASE_DIR/.env" - - exec_ssh "echo 'DATABASE_USER=root' >> $BASE_DIR/.env" - - exec_ssh "echo 'DATABASE_PASSWORD=$DB_PASSWORD' >> $BASE_DIR/.env" - - exec_ssh "echo 'DATABASE_HOST=database' >> $BASE_DIR/.env" - - exec_ssh "echo 'DEBUG=False' >> $BASE_DIR/.env" - - exec_ssh "echo 'LRS_HOST=$LRS_HOST' >> $BASE_DIR/.env" - - exec_ssh "echo 'LRS_TOKEN=$LRS_TOKEN' >> $BASE_DIR/.env" - - exec_ssh "echo 'LRS_CONNECTION_STRING=mongodb://root:$MONGO_PASSWORT@polaris.servicechain.cloud:27017' >> $BASE_DIR/.env" - - exec_ssh "echo 'LRS_MONGO_DB_NAME=lrs' >> $BASE_DIR/.env" - - exec_ssh "echo 'DATA_DISCLOSURE_EXPIRATION=30' >> $BASE_DIR/.env" - - exec_ssh "echo 'DATA_DISCLOSURE_LOCATION=/usr/src/app/backend/data_disclosure_zips' >> $BASE_DIR/.env" - - exec_ssh "echo 'EMAIL_HOST=$EMAIL_HOST' >> $BASE_DIR/.env" - - exec_ssh "echo 'EMAIL_PORT=587' >> $BASE_DIR/.env" - - exec_ssh "echo 'EMAIL_HOST_USER=$EMAIL_HOST_USER' >> $BASE_DIR/.env" - - exec_ssh "echo 'EMAIL_HOST_PASSWORD=$EMAIL_HOST_PASSWORD' >> $BASE_DIR/.env" - - exec_ssh "echo 'REDIS_PASSWORD=$REDIS_PASSWORD' >> $BASE_DIR/.env" - - exec_ssh "echo 'CELERY_BROKER_URL=redis://:$REDIS_PASSWORD@redis:6379/0' >> $BASE_DIR/.env" - - exec_ssh "echo '$JWT_PUBLIC_KEY' > $BASE_DIR/id_rsa.pub" - - exec_ssh "echo '$JWT_PRIVATE_KEY' > $BASE_DIR/id_rsa" - - exec_ssh "echo 'JWT_PUBLIC_KEY_PATH=backend/id_rsa.pub' >> $BASE_DIR/.env" - - exec_ssh "echo 'JWT_PRIVATE_KEY_PATH=backend/id_rsa' >> $BASE_DIR/.env" - - exec_ssh "echo 'IDP_SERVER=$IDP_SERVER' >> $BASE_DIR/.env" - - exec_ssh "echo 'IDP_ENABLED=false' >> $BASE_DIR/.env" - - exec_ssh "echo 'THIRD_PARTY_ACCESS_TOKEN=$THIRD_PARTY_ACCESS_TOKEN' >> $BASE_DIR/.env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; mkdir -p data_disclosure_zips" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; rm -rf data_disclosure_zips/*" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; echo 'URL=$URL' > .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; echo 'MYSQL_ROOT_PASSWORD=$MYSQL_ROOT_PASSWORD' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; echo 'WEBSITE=polaris' | sed 's/\./-/g' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; echo 'DB_PASSWORD=$DB_PASSWORD' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; echo 'DJANGO_DEVELOPMENT=false' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; echo 'DATABASE_NAME=polaris' | sed 's/\./-/g' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; echo 'DATABASE_USER=root' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; echo 'DATABASE_PASSWORD=$DB_PASSWORD' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; echo 'DATABASE_HOST=database' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; echo 'DEBUG=False' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; echo 'LRS_HOST=$LRS_HOST' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; echo 'LRS_TOKEN=$LRS_TOKEN' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; echo 'LRS_CONNECTION_STRING=mongodb://root:$MONGO_PASSWORT@polaris.servicechain.cloud:27017' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; echo 'LRS_MONGO_DB_NAME=lrs' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; echo 'DATA_DISCLOSURE_EXPIRATION=30' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; echo 'DATA_DISCLOSURE_LOCATION=/usr/src/app/backend/data_disclosure_zips' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; echo 'EMAIL_HOST=$EMAIL_HOST' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; echo 'EMAIL_PORT=587' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; echo 'EMAIL_HOST_USER=$EMAIL_HOST_USER' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; echo 'EMAIL_HOST_PASSWORD=$EMAIL_HOST_PASSWORD' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; echo 'REDIS_PASSWORD=$REDIS_PASSWORD' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; echo 'CELERY_BROKER_URL=redis://:$REDIS_PASSWORD@redis:6379/0' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; echo '$JWT_PUBLIC_KEY' > id_rsa.pub" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; echo '$JWT_PRIVATE_KEY' > id_rsa" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; echo 'JWT_PUBLIC_KEY_PATH=backend/id_rsa.pub' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; echo 'JWT_PRIVATE_KEY_PATH=backend/id_rsa' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; echo 'IDP_SERVER=$IDP_SERVER' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; echo 'IDP_ENABLED=false' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/rights-engine; echo 'THIRD_PARTY_ACCESS_TOKEN=$THIRD_PARTY_ACCESS_TOKEN' >> .env" script: - function exec_ssh () { ssh -o StrictHostKeyChecking=no $SSH_USER@$DEPLOYMENT_URL $1; } @@ -69,3 +82,56 @@ deploy_app: - exec_ssh "sleep 20 && cd $BASE_DIR; docker compose exec -it rights-engine sh -c 'python3 manage.py sqlflush | sed s/TRUNCATE/DROP\ TABLE\ IF\ EXISTS/g | python3 manage.py dbshell && echo DROP\ TABLE\ IF\ EXISTS\ django_migrations\; | python3 manage.py dbshell && python3 manage.py migrate && python3 manage.py loaddata fixtures/initial_db.json'" # ohne Datenbank Flush #- exec_ssh "sleep 20 && cd $BASE_DIR; docker compose exec -it backend sh -c 'python3 manage.py migrate" + +deploy_analytics-engine: + stage: deploy + allow_failure: false + variables: + <<: *global-variables + DEPLOYMENT_URL: $URL + environment: + name: analytics-engine + url: $URL + script: + - function exec_ssh () { ssh -o StrictHostKeyChecking=no $SSH_USER@$DEPLOYMENT_URL $1; } + # ENV Docker Compose + - exec_ssh "cd $BASE_DIR/$WEBSITE/analytics-engine; echo 'DYNACONF_SQLALCHEMY_DATABASE_URI=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@database/${POSTGRES_DATABASE}' > .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/analytics-engine; echo 'DYNACONF_CELERY_BROKER_URL=redis://:${REDIS_PASSWORD}@redis_db:6379/0' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/analytics-engine; echo 'DYNACONF_CELERY_RESULT_BACKEND=redis://:${REDIS_PASSWORD}@redis_db:6379/1' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/analytics-engine; echo 'DYNACONF_ANALYTICS_BACKEND_URL=http://scheduler:5000' >> .env" + # TODO fix backend url + - exec_ssh "cd $BASE_DIR/$WEBSITE/analytics-engine; echo 'DYNACONF_RIGHTS_ENGINE_BACKEND_URL=https://$URL' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/analytics-engine; echo 'WEBSITE=analytics' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/analytics-engine; echo 'URL=$DEPLOYMENT_URL' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/analytics-engine; echo 'REDIS_PASSWORD=$REDIS_PASSWORD' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/analytics-engine; echo 'POSTGRES_USER=$POSTGRES_USER' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/analytics-engine; echo 'POSTGRES_DATABASE=$POSTGRES_DATABASE' >> .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/analytics-engine; echo 'POSTGRES_PASSWORD=$POSTGRES_PASSWORD' >> .env" + + # Image Pull & Docker Restart + - exec_ssh "cd $BASE_DIR/$WEBSITE/analytics-engine; docker compose pull && docker compose down && docker compose up -d && sleep 30 && docker compose exec -it scheduler sh -c 'scheduler create-db' && docker compose exec -it scheduler sh -c 'scheduler read-configs'" + + # Initale Datenbank + #- exec_ssh "sleep 30 && cd $BASE_DIR/deployment/rights-engine; docker compose exec -it backend sh -c 'python3 manage.py migrate && python3 manage.py loaddata fixtures/initial_db.json'" +# Mit Datenbank Flush + #- exec_ssh "sleep 30 && cd $BASE_DIR/deployment/rights-engine; docker compose exec -it backend sh -c 'python3 manage.py sqlflush | sed s/TRUNCATE/DROP\ TABLE\ IF\ EXISTS/g | python3 manage.py dbshell && echo DROP\ TABLE\ IF\ EXISTS\ django_migrations\; | python3 manage.py dbshell && python3 manage.py migrate && python3 manage.py loaddata fixtures/initial_db.json'" +# ohne Datenbank Flush + #- exec_ssh "sleep 30 && cd $BASE_DIR/deployment/rights-engine; docker compose exec -it backend sh -c 'python3 manage.py migrate" + +deploy_mongodb: + stage: deploy + allow_failure: false + variables: + <<: *global-variables + DEPLOYMENT_URL: $URL + environment: + name: mongodb + url: $URL + + script: + - function exec_ssh () { ssh -o StrictHostKeyChecking=no $SSH_USER@$DEPLOYMENT_URL $1; } + # ENV Docker Compose + - exec_ssh "cd $BASE_DIR/$WEBSITE/mongodb; echo 'MONGO_USER=root' > .env" + - exec_ssh "cd $BASE_DIR/$WEBSITE/mongodb; echo 'MONGO_PASSWORT=$MONGO_PASSWORT' >> .env" + # Image Pull & Docker Restart + - exec_ssh "cd $BASE_DIR/$WEBSITE/mongodb; docker compose pull && docker compose down && docker compose up -d" \ No newline at end of file