Commit d11c1c5f authored by Sonja Happ's avatar Sonja Happ
Browse files

Merge branch 'large-files' into 'master'

S3 storage for large files

See merge request !22
parents 44e919b5 86b7a702
services:
- postgres:latest
- rabbitmq:latest
variables:
DOCKER_TAG: ${CI_COMMIT_SHORT_SHA}
DOCKER_TAG: ${CI_COMMIT_BRANCH}
DOCKER_IMAGE: ${CI_REGISTRY_IMAGE}
POSTGRES_DB: testvillasdb
POSTGRES_USER: villas
POSTGRES_PASSWORD: villas
POSTGRES_HOST: postgres
RABBITMQ_DEFAULT_USER: villas
RABBITMQ_DEFAULT_PASS: villas
AMQP_HOST: rabbitmq:5672
AMQP_USER: villas
AMQP_PASS: villas
PORT: 4000
GO_IMAGE: golang:1.13-buster
# Template
.go:
image: ${GO_IMAGE}
variables:
GOPATH: $CI_PROJECT_DIR/.go
before_script:
- mkdir -p .go
cache:
paths:
- .go/pkg/mod/
stages:
- build
- test
......@@ -27,11 +24,11 @@ stages:
build:
stage: build
image: ${GO_IMAGE}
extends: .go
script:
- go mod tidy
- go install github.com/swaggo/swag/cmd/swag
- swag init -p pascalcase -g "start.go" -o "./doc/api/"
- ${GOPATH}/bin/swag init -p pascalcase -g "start.go" -o "./doc/api/"
- go build
artifacts:
paths:
......@@ -43,26 +40,83 @@ build:
test:
stage: test
image: ${GO_IMAGE}
extends: .go
variables:
POSTGRES_DB: testvillasdb
POSTGRES_USER: villas
POSTGRES_PASSWORD: villas
POSTGRES_HOST: postgres
RABBITMQ_DEFAULT_USER: villas
RABBITMQ_DEFAULT_PASS: villas
MINIO_ROOT_USER: minio-villas
MINIO_ROOT_PASSWORD: minio-villas
MINIO_REGION_NAME: default
AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER}
AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD}
S3_BUCKET: villas-web
S3_ENDPOINT: http://minio:9000
S3_PATHSTYLE: 'true'
S3_NOSSL: 'false'
S3_REGION: ${MINIO_REGION_NAME}
AMQP_HOST: rabbitmq:5672
AMQP_USER: villas
AMQP_PASS: villas
PORT: 4000
DB_NAME: ${POSTGRES_DB}
DB_HOST: ${POSTGRES_HOST}
DB_USER: ${POSTGRES_USER}
DB_PASS: ${POSTGRES_PASSWORD}
BASE_PATH: /api
MODE: test
MODE: release
ADMIN_USER: User_0
ADMIN_PASS: xyz789
services:
- postgres:latest
- rabbitmq:latest
- name: minio/minio:RELEASE.2021-01-16T02-19-44Z
command: ['server', '/minio']
alias: minio
before_script:
- wget -qO /usr/bin/mc https://dl.min.io/client/mc/release/linux-amd64/mc && chmod +x /usr/bin/mc
- mc alias set gitlab http://minio:9000 ${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD}
- mc mb gitlab/${S3_BUCKET}
script:
- go mod tidy
- go test $(go list ./... )
-p 1
-v
-covermode=count
-coverprofile ./testcover.txt
- go tool cover -func=testcover.txt
dependencies:
- build
test:files_postgres:
stage: test
extends: .go
variables:
POSTGRES_DB: testvillasdb
POSTGRES_USER: villas
POSTGRES_PASSWORD: villas
POSTGRES_HOST: postgres
PORT: 4000
DB_NAME: ${POSTGRES_DB}
DB_HOST: ${POSTGRES_HOST}
DB_USER: ${POSTGRES_USER}
DB_PASS: ${POSTGRES_PASSWORD}
BASE_PATH: /api
MODE: release
ADMIN_USER: User_0
ADMIN_PASS: xyz789
services:
- postgres:latest
script:
- go mod tidy
- cd routes/file
- go test -v
dependencies:
- build
# Stage: deploy
##############################################################################
......@@ -80,7 +134,5 @@ deploy:
--dockerfile ${CI_PROJECT_DIR}/Dockerfile
--destination ${DOCKER_IMAGE}:${DOCKER_TAG}
--snapshotMode=redo
--cache=true
--cache-ttl=12h
dependencies:
- test
......@@ -38,59 +38,84 @@ func InitConfig() error {
}
var (
dbHost = flag.String("db-host", "/var/run/postgresql", "Host of the PostgreSQL database (default is /var/run/postgresql for localhost DB on Ubuntu systems)")
dbName = flag.String("db-name", "villasdb", "Name of the database to use (default is villasdb)")
dbUser = flag.String("db-user", "", "Username of database connection (default is <empty>)")
dbPass = flag.String("db-pass", "", "Password of database connection (default is <empty>)")
dbSSLMode = flag.String("db-ssl-mode", "disable", "SSL mode of DB (default is disable)") // TODO: change default for production
amqpHost = flag.String("amqp-host", "", "If set, use this as host for AMQP broker (default is disabled)")
amqpUser = flag.String("amqp-user", "", "Username for AMQP broker")
amqpPass = flag.String("amqp-pass", "", "Password for AMQP broker")
configFile = flag.String("config", "", "Path to YAML configuration file")
mode = flag.String("mode", "release", "Select debug/release/test mode (default is release)")
port = flag.String("port", "4000", "Port of the backend (default is 4000)")
baseHost = flag.String("base-host", "localhost:4000", "The host at which the backend is hosted (default: localhost)")
basePath = flag.String("base-path", "/api/v2", "The path at which the API routes are located (default /api/v2)")
adminUser = flag.String("admin-user", "", "Initial admin username")
adminPass = flag.String("admin-pass", "", "Initial admin password")
adminMail = flag.String("admin-mail", "", "Initial admin mail address")
dbHost = flag.String("db-host", "/var/run/postgresql", "Host of the PostgreSQL database (default is /var/run/postgresql for localhost DB on Ubuntu systems)")
dbName = flag.String("db-name", "villasdb", "Name of the database to use (default is villasdb)")
dbUser = flag.String("db-user", "", "Username of database connection (default is <empty>)")
dbPass = flag.String("db-pass", "", "Password of database connection (default is <empty>)")
dbSSLMode = flag.String("db-ssl-mode", "disable", "SSL mode of DB (default is disable)") // TODO: change default for production
amqpHost = flag.String("amqp-host", "", "If set, use this as host for AMQP broker (default is disabled)")
amqpUser = flag.String("amqp-user", "", "Username for AMQP broker")
amqpPass = flag.String("amqp-pass", "", "Password for AMQP broker")
configFile = flag.String("config", "", "Path to YAML configuration file")
mode = flag.String("mode", "release", "Select debug/release/test mode (default is release)")
port = flag.String("port", "4000", "Port of the backend (default is 4000)")
baseHost = flag.String("base-host", "localhost:4000", "The host at which the backend is hosted (default: localhost)")
basePath = flag.String("base-path", "/api/v2", "The path at which the API routes are located (default /api/v2)")
adminUser = flag.String("admin-user", "", "Initial admin username")
adminPass = flag.String("admin-pass", "", "Initial admin password")
adminMail = flag.String("admin-mail", "", "Initial admin mail address")
s3Bucket = flag.String("s3-bucket", "", "S3 Bucket for uploading files")
s3Endpoint = flag.String("s3-endpoint", "", "Endpoint of S3 API for file uploads")
s3Region = flag.String("s3-region", "default", "S3 Region for file uploads")
s3NoSSL = flag.Bool("s3-nossl", false, "Use encrypted connections to the S3 API")
s3PathStyle = flag.Bool("s3-pathstyle", false, "Use path-style S3 API")
)
flag.Parse()
static := map[string]string{
"db.host": *dbHost,
"db.name": *dbName,
"db.user": *dbUser,
"db.pass": *dbPass,
"db.ssl": *dbSSLMode,
"amqp.host": *amqpHost,
"amqp.user": *amqpUser,
"amqp.pass": *amqpPass,
"mode": *mode,
"port": *port,
"base.host": *baseHost,
"base.path": *basePath,
"admin.user": *adminUser,
"admin.pass": *adminPass,
"admin.mail": *adminMail,
"db.host": *dbHost,
"db.name": *dbName,
"db.user": *dbUser,
"db.pass": *dbPass,
"db.ssl": *dbSSLMode,
"amqp.host": *amqpHost,
"amqp.user": *amqpUser,
"amqp.pass": *amqpPass,
"mode": *mode,
"port": *port,
"base.host": *baseHost,
"base.path": *basePath,
"admin.user": *adminUser,
"admin.pass": *adminPass,
"admin.mail": *adminMail,
"s3.bucket": *s3Bucket,
"s3.endpoint": *s3Endpoint,
"s3.region": *s3Region,
}
if *s3NoSSL == true {
static["s3.nossl"] = "true"
} else {
static["s3.nossl"] = "false"
}
if *s3PathStyle == true {
static["s3.pathstyle"] = "true"
} else {
static["s3.pathstyle"] = "false"
}
mappings := map[string]string{
"DB_HOST": "db.host",
"DB_NAME": "db.name",
"DB_USER": "db.user",
"DB_PASS": "db.pass",
"DB_SSLMODE": "db.ssl",
"AMQP_HOST": "amqp.host",
"AMQP_USER": "amqp.user",
"AMQP_PASS": "amqp.pass",
"BASE_HOST": "base.host",
"BASE_PATH": "base.path",
"MODE": "mode",
"PORT": "port",
"ADMIN_USER": "admin.user",
"ADMIN_PASS": "admin.pass",
"ADMIN_MAIL": "admin.mail",
"DB_HOST": "db.host",
"DB_NAME": "db.name",
"DB_USER": "db.user",
"DB_PASS": "db.pass",
"DB_SSLMODE": "db.ssl",
"AMQP_HOST": "amqp.host",
"AMQP_USER": "amqp.user",
"AMQP_PASS": "amqp.pass",
"BASE_HOST": "base.host",
"BASE_PATH": "base.path",
"MODE": "mode",
"PORT": "port",
"ADMIN_USER": "admin.user",
"ADMIN_PASS": "admin.pass",
"ADMIN_MAIL": "admin.mail",
"S3_BUCKET": "s3.bucket",
"S3_ENDPOINT": "s3.endpoint",
"S3_REGION": "s3.region",
"S3_NOSSL": "s3.nossl",
"S3_PATHSTYLE": "s3.pathstyle",
}
defaults := config.NewStatic(static)
......
......@@ -22,9 +22,10 @@
package database
import (
"github.com/lib/pq"
"time"
"github.com/lib/pq"
"github.com/jinzhu/gorm/dialects/postgres"
)
......@@ -199,6 +200,8 @@ type File struct {
Model
// Name of file
Name string `json:"name" gorm:"not null"`
// Key of file in S3 bucket
Key string `json:"key"`
// Type of file (MIME type)
Type string `json:"type"`
// Size of file (in byte)
......
......@@ -2,6 +2,7 @@ module git.rwth-aachen.de/acs/public/villas/web-backend-go
require (
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751
github.com/aws/aws-sdk-go v1.35.35
github.com/chenjiandongx/ginprom v0.0.0-20191022035802-6f3da3c84986
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/gin-gonic/gin v1.4.0
......@@ -19,7 +20,7 @@ require (
github.com/swaggo/gin-swagger v1.2.0
github.com/swaggo/swag v1.6.3
github.com/zpatrick/go-config v0.0.0-20191104215613-50bc2709703f
golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
gopkg.in/go-playground/validator.v9 v9.30.0
gopkg.in/ini.v1 v1.51.0 // indirect
)
......
......@@ -18,6 +18,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/aws/aws-sdk-go v1.35.35 h1:o/EbgEcIPWga7GWhJhb3tiaxqk4/goTdo5YEMdnVxgE=
github.com/aws/aws-sdk-go v1.35.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
......@@ -105,6 +107,10 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M=
github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo=
......@@ -151,6 +157,7 @@ github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJ
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
......@@ -207,8 +214,8 @@ go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708 h1:pXVtWnwHkrWD9ru3sDxY/qFK/bfc0egRovX91EjWjf4=
golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
......@@ -228,6 +235,8 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
......@@ -248,10 +257,14 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
......@@ -288,6 +301,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
......@@ -25,11 +25,12 @@ import (
"bytes"
"encoding/json"
"fmt"
"github.com/gin-gonic/gin"
"github.com/nsf/jsondiff"
"log"
"net/http"
"net/http/httptest"
"github.com/gin-gonic/gin"
"github.com/nsf/jsondiff"
)
// data type used in testing
......@@ -169,8 +170,47 @@ func TestEndpoint(router *gin.Engine, token string, url string,
}
req.Header.Set("Content-Type", "application/json")
req.Header.Add("Authorization", "Bearer "+token)
router.ServeHTTP(w, req)
return handleRedirect(w, req)
}
func handleRedirect(w *httptest.ResponseRecorder, req *http.Request) (int, *bytes.Buffer, error) {
if w.Code == http.StatusMovedPermanently ||
w.Code == http.StatusFound ||
w.Code == http.StatusTemporaryRedirect ||
w.Code == http.StatusPermanentRedirect {
// Follow external redirect
redirURL, err := w.Result().Location()
if err != nil {
return 0, nil, fmt.Errorf("invalid location header")
}
log.Println("redirecting request to", redirURL.String())
req, err := http.NewRequest(req.Method, redirURL.String(), req.Body)
if err != nil {
return 0, nil, fmt.Errorf("handle redirect: failed to create new request: %v", err)
}
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return 0, nil, fmt.Errorf("handle redirect: failed to follow redirect: %v", err)
}
buf := new(bytes.Buffer)
_, err = buf.ReadFrom(resp.Body)
if err != nil {
return 0, nil, fmt.Errorf("handle redirect: failed to follow redirect: %v", err)
}
return resp.StatusCode, buf, nil
}
// No redirect
return w.Code, w.Body, nil
}
......
#!/bin/bash
kubectl -n services port-forward svc/postgres-postgresql 5432:5432 & PF1=$!
kubectl -n services port-forward svc/broker 5672:5672 & PF2=$!
set -e
NS=villas-demo
NS=${NS:-villas}
MK="minikube"
KCTL="kubectl -n${NS}"
HELM="helm -n${NS}"
# CHART="villas"
CHART="~/workspace/rwth/acs/public/catalogue/charts/villas"
POSTGRES_USER=$(kubectl -n ${NS} get secret postgres-credentials -o json | jq -r .data.username | base64 -d)
POSTGRES_PASS=$(kubectl -n ${NS} get secret postgres-credentials -o json | jq -r .data.password | base64 -d)
RABBITMQ_USER=$(kubectl -n ${NS} get secret postgres-credentials -o json | jq -r .data.username | base64 -d)
RABBITMQ_PASS=$(kubectl -n ${NS} get secret postgres-credentials -o json | jq -r .data.password | base64 -d)
CONFIG=$(mktemp)
NAME="villas"
go test ./routes/healthz -p 1 --args \
-mode test \
-db-host localhost -db-name villas \
-db-user ${POSTGRES_USER} -db-pass ${POSTGRES_PASS} \
-amqp-host localhost \
-amqp-user ${RABBITMQ_USER} -amqp-pass ${RABBITMQ_PASS}
function cleanup() {
kill $PF1 $PF2 $PF3
wait $PF1 $PF2 $PF3
kill $PF1 $PF2
wait $PF1 $PF2
rm ${CONFIG}
echo "Goodbye"
}
trap cleanup EXIT
if [ -n "${USE_MINIKUBE}" ]; then
MK_START_OPTS="--addons=ingress"
if [ $(uname -s) == Darwin ]; then
# https://github.com/kubernetes/minikube/issues/7332
MK_START_OPTS+="--vm=true"
fi
${MK} start ${MK_START_OPTS}
kubectl -n kube-system expose deployment ingress-nginx-controller --type=LoadBalancer || true
IP=$(minikube ip)
PORT=$(kubectl -n kube-system get service ingress-nginx-controller --output='jsonpath={.spec.ports[0].nodePort}')
# Add pseudo hostname to /etc/hosts
echo "Please provide your root password for modifiying /etc/hosts"
sudo sed -in "/^${IP}.*/d" /etc/hosts
echo "${IP} minikube" | sudo tee -a /etc/hosts
HELM_OPTS="--set web.backend.enabled=false \
--set web.backend.external.ip=${IP} \
--set web.backend.external.port=4000 \
--set web.admin.password=admin \
--set ingress.host=minikube"
# Check if chart has already been deployed before
if helm get values villas > /dev/null; then
RABBITMQ_ERLANG_COOKIE=$(kubectl get secret --namespace default villas-broker -o jsonpath="{.data.rabbitmq-erlang-cookie}" | base64 --decode)
RABBITMQ_PASSWORD=$(kubectl get secret --namespace default villas-broker -o jsonpath="{.data.rabbitmq-password}" | base64 --decode)
HELM_OPTS+=" --set broker.auth.password=${RABBITMQ_PASSWORD} --set broker.auth.erlangCookie=${RABBITMQ_ERLANG_COOKIE}"
fi
${HELM} upgrade \
${HELM_OPTS} \
--install \
--create-namespace \
--wait \
--repo https://packages.fein-aachen.org/helm/charts \
${NAME} ${CHART}
fi
# Get backend config from cluster
${KCTL} get cm ${NAME}-web -o 'jsonpath={.data.config\.yaml}' > ${CONFIG}
# Enable access of backend to db, broker and s3
export DB_HOST=localhost
export DB_PASS=$(${KCTL} get secret ${NAME}-database -o 'jsonpath={.data.postgresql-password}' | base64 -d)
export AMQP_HOST=localhost
export AMQP_PASS=$(${KCTL} get secret ${NAME}-broker -o 'jsonpath={.data.rabbitmq-password}' | base64 -d)
export AWS_ACCESS_KEY_ID=$(${KCTL} get secret ${NAME}-s3 -o 'jsonpath={.data.accesskey}' | base64 -d)
export AWS_SECRET_ACCESS_KEY=$(${KCTL} get secret ${NAME}-s3 -o 'jsonpath={.data.secretkey}' | base64 -d)
# Setup port forwards for backend
if [ -n "${USE_EXTERNAL_POSTGRESQL}" ]; then
kubectl -n postgresql port-forward svc/postgresql 5432:5432 & PF1=$!
else
${KCTL} port-forward svc/${NAME}-database 5432:5432 & PF1=$!
fi
${KCTL} port-forward svc/${NAME}-broker 5672:5672 & PF2=$!
sleep 2
if [ -n "${USE_MINIKUBE}" ]; then
# python -mwebbrowser http://minikube:${PORT}
echo
echo "==========================================="
echo "Access VILLASweb at http://minikube:${PORT}"
echo "==========================================="
echo
fi
go run start.go -config ${CONFIG}
......@@ -97,14 +97,14 @@ func addFile(c *gin.Context) {
}
// Extract file from POST request form
file_header, err := c.FormFile("file")
fileHeader, err := c.FormFile("file")
if err != nil {
helper.BadRequestError(c, fmt.Sprintf("Get form error: %s", err.Error()))
return
}
var newFile File
err = newFile.Register(file_header, so.ID)
err = newFile.Register(fileHeader, so.ID)
if !helper.DBError(c, err) {
c.JSON(http.StatusOK, gin.H{"file": newFile.File})
}
......
......@@ -22,8 +22,7 @@
package file
import (
"git.rwth-aachen.de/acs/public/villas/web-backend-go/routes/scenario"
"github.com/gin-gonic/gin"
"fmt"
"image"
_ "image/gif"
_ "image/jpeg"
......@@ -33,10 +32,13 @@ import (
"mime/multipart"
"net/http"
"path/filepath"
"strconv"
"strings"
"time"
"git.rwth-aachen.de/acs/public/villas/web-backend-go/configuration"
"git.rwth-aachen.de/acs/public/villas/web-backend-go/routes/scenario"
"github.com/gin-gonic/gin"
"git.rwth-aachen.de/acs/public/villas/web-backend-go/database"
)
......@@ -62,14 +64,20 @@ func (f *File) save() error {
func (f *File) download(c *gin.Context) error {
// create unique file name
filename := "file_" + strconv.FormatUint(uint64(f.ID), 10) + "_" + f.Name
// detect the content type of the file
contentType := http.DetectContentType(f.FileData)
//Seems this headers needed for some browsers (for example without this headers Chrome will download files as txt)
c.Header("Content-Description", "File Transfer")
c.Header("Content-Disposition", "attachment; filename="+filename)
c.Data(http.StatusOK, contentType, f.FileData)
if f.Key == "" {