Commit d9c25152 authored by Petar Hristov's avatar Petar Hristov 💬
Browse files

Merge branch 'Hotfix/1485-policyFix' into 'master'

Hotfix/1485 policy fix

See merge request !9
parents a4050fba f281460f
...@@ -24,6 +24,8 @@ namespace Coscine.ResourceTypeWaterbutlerRdsS3.Test ...@@ -24,6 +24,8 @@ namespace Coscine.ResourceTypeWaterbutlerRdsS3.Test
private Guid _guid; private Guid _guid;
private string _bucketName; private string _bucketName;
private string _endpoint;
private string _secretKey; private string _secretKey;
private string _readUser; private string _readUser;
...@@ -42,6 +44,7 @@ namespace Coscine.ResourceTypeWaterbutlerRdsS3.Test ...@@ -42,6 +44,7 @@ namespace Coscine.ResourceTypeWaterbutlerRdsS3.Test
_secretKey = "VERY_S3cr3t_Key!!!"; _secretKey = "VERY_S3cr3t_Key!!!";
_readUser = $"{_testPrefix}.read_{_guid}"; _readUser = $"{_testPrefix}.read_{_guid}";
_writeUser = $"{_testPrefix}.write_{_guid}"; _writeUser = $"{_testPrefix}.write_{_guid}";
_endpoint = _configuration.GetString("coscine/global/rds/ecs-rwth/rds-s3/s3_endpoint");
_rdsS3EcsManager = new EcsManager _rdsS3EcsManager = new EcsManager
{ {
...@@ -102,6 +105,7 @@ namespace Coscine.ResourceTypeWaterbutlerRdsS3.Test ...@@ -102,6 +105,7 @@ namespace Coscine.ResourceTypeWaterbutlerRdsS3.Test
{ "secretKeyRead", _secretKey }, { "secretKeyRead", _secretKey },
{ "accessKeyWrite", _writeUser }, { "accessKeyWrite", _writeUser },
{ "secretKeyWrite", _secretKey }, { "secretKeyWrite", _secretKey },
{ "endpoint", _endpoint },
}).Wait(); }).Wait();
// No implementation for deletion // No implementation for deletion
......
...@@ -33,7 +33,7 @@ namespace Coscine.ResourceTypeWaterbutlerRdsS3 ...@@ -33,7 +33,7 @@ namespace Coscine.ResourceTypeWaterbutlerRdsS3
private readonly List<string> _readRights; private readonly List<string> _readRights;
private readonly List<string> _writeRights; private readonly List<string> _writeRights;
public ResourceTypeWaterbutlerRdsS3(string name, IConfiguration gConfig, ResourceTypeConfigurationObject resourceTypeConfiguration) : base(name, gConfig, resourceTypeConfiguration) public ResourceTypeWaterbutlerRdsS3(string name, IConfiguration gConfig, ResourceTypeConfigurationObject resourceTypeConfiguration) : base(name, gConfig, resourceTypeConfiguration)
{ {
_waterbutlerInterface = new WaterbutlerInterface(Configuration, new DataSourceService(new HttpClient())); _waterbutlerInterface = new WaterbutlerInterface(Configuration, new DataSourceService(new HttpClient()));
_rdsS3EcsManager = new EcsManager _rdsS3EcsManager = new EcsManager
...@@ -119,7 +119,7 @@ public ResourceTypeWaterbutlerRdsS3(string name, IConfiguration gConfig, Resourc ...@@ -119,7 +119,7 @@ public ResourceTypeWaterbutlerRdsS3(string name, IConfiguration gConfig, Resourc
// Not found, upload new // Not found, upload new
if (infos == null) if (infos == null)
{ {
var filename = key.Substring(key.LastIndexOf("/") + 1); var filename = key[(key.LastIndexOf("/") + 1)..];
var rootPath = key.Substring(0, key.Length - filename.Length); var rootPath = key.Substring(0, key.Length - filename.Length);
HandleResponse(await _waterbutlerInterface.UploadFileAsync(rootPath, filename, _provider, authHeader, body, contentLength)); HandleResponse(await _waterbutlerInterface.UploadFileAsync(rootPath, filename, _provider, authHeader, body, contentLength));
...@@ -212,6 +212,49 @@ public ResourceTypeWaterbutlerRdsS3(string name, IConfiguration gConfig, Resourc ...@@ -212,6 +212,49 @@ public ResourceTypeWaterbutlerRdsS3(string name, IConfiguration gConfig, Resourc
await _rdsS3EcsManager.SetUserAcl(options["accessKeyRead"], options["bucketname"], _readRights); await _rdsS3EcsManager.SetUserAcl(options["accessKeyRead"], options["bucketname"], _readRights);
await _rdsS3EcsManager.SetUserAcl(options["accessKeyWrite"], options["bucketname"], _writeRights); await _rdsS3EcsManager.SetUserAcl(options["accessKeyWrite"], options["bucketname"], _writeRights);
var amazonConfig = new AmazonS3Config
{
ServiceURL = options["endpoint"],
ForcePathStyle = true
};
using var client = new AmazonS3Client(_accessKey, _secretKey, amazonConfig);
var policy = $@"{{
""Version"": ""2012-10-17"",
""Id"": ""null"",
""Statement"":[
{{
""Action"": [""s3:PutObject"", ""s3:GetObject"", ""s3:GetObjectAcl"", ""s3:GetObjectVersion"", ""s3:DeleteObject"", ""s3:DeleteObjectVersion""],
""Effect"": ""Allow"",
""Resource"": [""{options["bucketname"]}/*""],
""Principal"": [""{options["accessKeyWrite"]}""]
}},
{{
""Action"": [""s3:GetObject"", ""s3:GetObjectAcl"", ""s3:GetObjectVersion""],
""Effect"": ""Allow"",
""Resource"": [""{options["bucketname"]}/*""],
""Principal"": [""{options["accessKeyRead"]}""]
}}
]
}}";
var putRequest = new PutBucketPolicyRequest
{
BucketName = options["bucketname"],
Policy = policy
};
// Exception will be thrown on dev systems.
// Works on live.
try
{
await client.PutBucketPolicyAsync(putRequest);
}
catch (Exception)
{
}
} }
public override async Task<long> GetResourceQuotaUsed(string id, Dictionary<string, string> options = null) public override async Task<long> GetResourceQuotaUsed(string id, Dictionary<string, string> options = null)
...@@ -223,27 +266,25 @@ public ResourceTypeWaterbutlerRdsS3(string name, IConfiguration gConfig, Resourc ...@@ -223,27 +266,25 @@ public ResourceTypeWaterbutlerRdsS3(string name, IConfiguration gConfig, Resourc
ForcePathStyle = true ForcePathStyle = true
}; };
using (var client = new AmazonS3Client(_accessKey, _secretKey, amazonConfig)) using var client = new AmazonS3Client(_accessKey, _secretKey, amazonConfig);
long totalFileSize = 0;
long fileCount = 0;
var listRequest = new ListObjectsRequest()
{ {
long totalFileSize = 0; BucketName = options["bucketname"]
long fileCount = 0; };
var listRequest = new ListObjectsRequest()
{
BucketName = options["bucketname"]
};
ListObjectsResponse listResponse; ListObjectsResponse listResponse;
do do
{ {
listResponse = await client.ListObjectsAsync(listRequest); listResponse = await client.ListObjectsAsync(listRequest);
fileCount += listResponse.S3Objects.Count; fileCount += listResponse.S3Objects.Count;
totalFileSize += listResponse.S3Objects.Sum(x => x.Size); totalFileSize += listResponse.S3Objects.Sum(x => x.Size);
listRequest.Marker = listResponse.NextMarker; listRequest.Marker = listResponse.NextMarker;
} while (listResponse.IsTruncated); } while (listResponse.IsTruncated);
return totalFileSize; return totalFileSize;
}
} }
public override async Task<long> GetResourceQuotaAvailable(string id, Dictionary<string, string> options = null) public override async Task<long> GetResourceQuotaAvailable(string id, Dictionary<string, string> options = null)
...@@ -265,19 +306,17 @@ public ResourceTypeWaterbutlerRdsS3(string name, IConfiguration gConfig, Resourc ...@@ -265,19 +306,17 @@ public ResourceTypeWaterbutlerRdsS3(string name, IConfiguration gConfig, Resourc
ServiceURL = options["endpoint"], ServiceURL = options["endpoint"],
ForcePathStyle = true ForcePathStyle = true
}; };
using (var client = new AmazonS3Client(_accessKey, _secretKey, amazonConfig)) using var client = new AmazonS3Client(_accessKey, _secretKey, amazonConfig);
var presignedUrl = client.GetPreSignedURL(new GetPreSignedUrlRequest()
{ {
var presignedUrl = client.GetPreSignedURL(new GetPreSignedUrlRequest() BucketName = options["bucketname"],
{ Key = key,
BucketName = options["bucketname"], Verb = HttpVerb.GET,
Key = key, Protocol = Protocol.HTTP,
Verb = HttpVerb.GET, // For now, expiry of a day is set, but this might be up to debate
Protocol = Protocol.HTTP, Expires = DateTime.UtcNow.AddHours(24)
// For now, expiry of a day is set, but this might be up to debate });
Expires = DateTime.UtcNow.AddHours(24) return new Uri(presignedUrl);
});
return new Uri(presignedUrl);
}
}); });
} }
...@@ -290,19 +329,17 @@ public ResourceTypeWaterbutlerRdsS3(string name, IConfiguration gConfig, Resourc ...@@ -290,19 +329,17 @@ public ResourceTypeWaterbutlerRdsS3(string name, IConfiguration gConfig, Resourc
ServiceURL = options["endpoint"], ServiceURL = options["endpoint"],
ForcePathStyle = true ForcePathStyle = true
}; };
using (var client = new AmazonS3Client(_accessKey, _secretKey, amazonConfig)) using var client = new AmazonS3Client(_accessKey, _secretKey, amazonConfig);
var presignedUrl = client.GetPreSignedURL(new GetPreSignedUrlRequest()
{ {
var presignedUrl = client.GetPreSignedURL(new GetPreSignedUrlRequest() BucketName = options["bucketname"],
{ Key = key,
BucketName = options["bucketname"], Verb = HttpVerb.PUT,
Key = key, Protocol = Protocol.HTTP,
Verb = HttpVerb.PUT, // For now, expiry of a day is set, but this might be up to debate
Protocol = Protocol.HTTP, Expires = DateTime.UtcNow.AddHours(24)
// For now, expiry of a day is set, but this might be up to debate });
Expires = DateTime.UtcNow.AddHours(24) return new Uri(presignedUrl);
});
return new Uri(presignedUrl);
}
}); });
} }
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment