Commit 90940116 authored by L. Ellenbeck's avatar L. Ellenbeck Committed by Petar Hristov
Browse files

Fix: fix for ude (rpdm/issues#71)

parent bec2978e
......@@ -14,4 +14,4 @@
<PackageReference Include="NUnit" Version="3.13.1" />
<PackageReference Include="NUnit3TestAdapter" Version="3.17.0" />
</ItemGroup>
</Project>
\ No newline at end of file
</Project>
......@@ -5,7 +5,7 @@ using Coscine.ResourceTypeBase;
using NUnit.Framework;
using System;
using System.Collections.Generic;
using System.Linq;
using System.IO;
using System.Reflection;
namespace Coscine.ResourceTypeWaterbutlerRdsS3.Test
......@@ -14,15 +14,24 @@ namespace Coscine.ResourceTypeWaterbutlerRdsS3.Test
public class ResourceTypeWaterbutlerRdsS3Tests
{
private readonly string _testPrefix = "Coscine-ResourceTypeWaterbutlerS3Tests-Tests";
private string _rdsS3KeyPrefix;
private string _userKeyPrefix;
private readonly ResourceTypeConfigurationObject _resourceTypeConfiguration = new ResourceTypeConfigurationObject();
private readonly string _userKeyPrefix = "coscine/global/rds/ecs-rwth/users";
private readonly IConfiguration _configuration = new ConsulConfiguration();
private EcsManager _rdsS3EcsManager;
private EcsManager _userEcsManager;
private static readonly Dictionary<string, string> _configs = new Dictionary<string, string>
{
{ "rdss3", "coscine/global/rds/ecs-rwth/rds-s3" },
{ "rdss3ude", "coscine/global/rds/ecs-ude/rds-s3" }
};
private static IEnumerable<object[]> GetTestConfigs()
{
foreach (var kv in _configs)
{
yield return new object[] { kv.Key, kv.Value };
}
}
private string _resourceType;
private string _localAssemblyName;
private long _quota;
......@@ -30,117 +39,165 @@ namespace Coscine.ResourceTypeWaterbutlerRdsS3.Test
private Guid _guid;
private string _bucketName;
private string _endpoint;
private string _secretKey;
private string _readWriteSecretKey;
private string _readUser;
private string _writeUser;
private string _accessKey;
[OneTimeSetUp]
public void OneTimeSetUp()
[SetUp]
public void SetUp()
{
_resourceTypeConfiguration.Config = new Dictionary<string, string>
{
{ "rdss3Key", "coscine/global/rds/ecs-rwth/rds-s3" },
{ "userKey", "coscine/global/rds/ecs-rwth/users"}
};
_resourceType = "rdss3";
_localAssemblyName = "Coscine.ResourceTypeWaterbutlerRdsS3";
_rdsS3KeyPrefix = _resourceTypeConfiguration.Config["rdss3Key"];
_userKeyPrefix = _resourceTypeConfiguration.Config["userKey"];
_quota = 1;
_guid = Guid.NewGuid();
_bucketName = $"{_testPrefix}.{_guid}";
_secretKey = "VERY_S3cr3t_Key!!!";
_readWriteSecretKey = "VERY_S3cr3t_Key!!!";
_readUser = $"{_testPrefix}.read_{_guid}";
_writeUser = $"{_testPrefix}.write_{_guid}";
_endpoint = _configuration.GetString($"{_rdsS3KeyPrefix}/s3_endpoint");
_accessKey = _configuration.GetString($"{_rdsS3KeyPrefix}/object_user_name");
_rdsS3EcsManager = new EcsManager
}
private EcsManager CreateManager(string keyPrefix)
{
return new EcsManager
{
EcsManagerConfiguration = new EcsManagerConfiguration
{
ManagerApiEndpoint = _configuration.GetString($"{_rdsS3KeyPrefix}/manager_api_endpoint"),
NamespaceName = _configuration.GetString($"{_rdsS3KeyPrefix}/namespace_name"),
NamespaceAdminName = _configuration.GetString($"{_rdsS3KeyPrefix}/namespace_admin_name"),
NamespaceAdminPassword = _configuration.GetString($"{_rdsS3KeyPrefix}/namespace_admin_password"),
ObjectUserName = _configuration.GetString($"{_rdsS3KeyPrefix}/object_user_name"),
ReplicationGroupId = _configuration.GetString($"{_rdsS3KeyPrefix}/replication_group_id"),
ManagerApiEndpoint = _configuration.GetString($"{keyPrefix}/manager_api_endpoint"),
NamespaceName = _configuration.GetString($"{keyPrefix}/namespace_name"),
NamespaceAdminName = _configuration.GetString($"{keyPrefix}/namespace_admin_name"),
NamespaceAdminPassword = _configuration.GetString($"{keyPrefix}/namespace_admin_password"),
ObjectUserName = _configuration.GetString($"{keyPrefix}/object_user_name"),
ReplicationGroupId = _configuration.GetString($"{keyPrefix}/replication_group_id"),
}
};
}
private ResourceTypeDefinition GetLocalResourceType(string type)
{
// Load the current assembly from the ref and not the locally installed.
var assemblyName = Array.Find(Assembly.GetExecutingAssembly().GetReferencedAssemblies(), x => x.Name == _localAssemblyName);
var assembly = Assembly.Load(assemblyName);
var resourceType = ResourceTypeFactory.CreateResourceTypeObject(type, new ConsulConfiguration(), assembly);
return resourceType;
}
_userEcsManager = new EcsManager
private Dictionary<string, string> GetResourceOptions(string prefix)
{
var dictionary = new Dictionary<string, string>
{
EcsManagerConfiguration = new EcsManagerConfiguration
{
ManagerApiEndpoint = _configuration.GetString($"{_userKeyPrefix}/manager_api_endpoint"),
NamespaceName = _configuration.GetString($"{_userKeyPrefix}/namespace_name"),
NamespaceAdminName = _configuration.GetString($"{_userKeyPrefix}/namespace_admin_name"),
NamespaceAdminPassword = _configuration.GetString($"{_userKeyPrefix}/namespace_admin_password"),
ObjectUserName = _configuration.GetString($"{_userKeyPrefix}/object_user_name"),
ReplicationGroupId = _configuration.GetString($"{_userKeyPrefix}/replication_group_id"),
}
{ "accessKey", _configuration.GetString($"{prefix}/object_user_name") },
{ "secretKey", _configuration.GetString($"{prefix}/object_user_secretkey") },
{ "accessKeyRead", _readUser },
{ "secretKeyRead", _readWriteSecretKey },
{ "accessKeyWrite", _writeUser },
{ "secretKeyWrite", _readWriteSecretKey },
{ "bucketname", _bucketName },
{ "endpoint", _configuration.GetString($"{prefix}/s3_endpoint") },
{ "size", $"{_quota}" }
};
return dictionary;
}
[OneTimeTearDown]
public void OneTimeTearDown()
[TearDown]
public void TearDown()
{
foreach (var kv in _configs)
{
var ecsManager = CreateManager(kv.Value);
try
{
ecsManager.DeleteBucket(_bucketName).Wait();
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
var userEcsManager = CreateManager(_userKeyPrefix);
try
{
_userEcsManager.DeleteObjectUser(_readUser).Wait();
_userEcsManager.DeleteObjectUser(_writeUser).Wait();
_rdsS3EcsManager.DeleteBucket(_bucketName).Wait();
userEcsManager.DeleteObjectUser(_readUser).Wait();
}
catch (Exception)
catch (Exception e)
{
Console.WriteLine(e);
}
try
{
userEcsManager.DeleteObjectUser(_writeUser).Wait();
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
[Test]
public void TestConstructor()
[TestCaseSource(nameof(GetTestConfigs))]
public void TestConstructor(string type, string configPrefix)
{
_ = new ResourceTypeWaterbutlerRdsS3(_resourceType, _configuration, _resourceTypeConfiguration);
var resourceTypeConfiguration = new ResourceTypeConfigurationObject
{
Config = new Dictionary<string, string>
{
{ "rdss3Key", configPrefix },
{ "userKey", _userKeyPrefix }
}
};
var resourceTpye = new ResourceTypeWaterbutlerRdsS3(type, _configuration, resourceTypeConfiguration);
Assert.NotNull(resourceTpye);
}
[Test]
public void TestCreate()
[TestCaseSource(nameof(GetTestConfigs))]
public void TestCreate(string type, string configPrefix)
{
var resourceType = new ResourceTypeWaterbutlerRdsS3(_resourceType, _configuration, _resourceTypeConfiguration);
resourceType.CreateResource(new Dictionary<string, string> {
{ "bucketname", _bucketName },
{ "size", $"{_quota}" },
{ "accessKey", _accessKey },
{ "accessKeyRead", _readUser },
{ "secretKeyRead", _secretKey },
{ "accessKeyWrite", _writeUser },
{ "secretKeyWrite", _secretKey },
{ "endpoint", _endpoint },
{ "readonly", "false" },
}).Wait();
Assert.True(_userEcsManager.DeleteObjectUser(_readUser).Result);
Assert.True(_userEcsManager.DeleteObjectUser(_writeUser).Result);
Assert.True(_rdsS3EcsManager.DeleteBucket(_bucketName).Result);
var rdsS3EcsManager = CreateManager(configPrefix);
var userEcsManager = CreateManager(_userKeyPrefix);
var resourceType = GetLocalResourceType(type);
var options = GetResourceOptions(configPrefix);
resourceType.CreateResource(options).Wait();
var randomFileName = Guid.NewGuid();
var memoryStream = new MemoryStream();
byte[] testData = { (byte)'C', (byte)'o', (byte)'S', (byte)'I', (byte)'n', (byte)'e' };
memoryStream.Write(testData, 0, testData.Length);
memoryStream.Position = 0;
options.Add("ContentLength", memoryStream.Length.ToString());
resourceType.StoreEntry(null, $"/{randomFileName}", memoryStream, options).Wait();
var entry = resourceType.GetEntry(null, $"/{randomFileName}", null, options).Result;
Assert.NotNull(entry);
var entries = resourceType.ListEntries(null, "/", options).Result;
Assert.NotNull(entries);
Assert.IsNotEmpty(entries);
Assert.IsTrue(entries.Count == 1);
Assert.IsTrue(entries[0].Key == entry.Key);
resourceType.DeleteEntry(null, entry.Key, options).Wait();
Assert.True(userEcsManager.DeleteObjectUser(_readUser).Result);
Assert.True(userEcsManager.DeleteObjectUser(_writeUser).Result);
Assert.True(rdsS3EcsManager.DeleteBucket(_bucketName).Result);
}
[Test]
public void TestResourceTypeInformation()
[TestCaseSource(nameof(GetTestConfigs))]
public void TestResourceTypeInformation(string type, string _)
{
// Load the current assembly from the ref and not the locally installed.
var assemblyName = Assembly.GetExecutingAssembly().GetReferencedAssemblies().Where(x => x.Name == _localAssemblyName).FirstOrDefault();
var assembly = Assembly.Load(assemblyName);
var resourceType = ResourceTypeFactory.CreateResourceTypeObject(_resourceType, new ConsulConfiguration(), assembly);
var resourceType = GetLocalResourceType(type);
var resourceTypeInformation = resourceType.GetResourceTypeInformation().Result;
Assert.IsTrue(resourceTypeInformation.IsQuotaAvailable);
Assert.IsFalse(resourceTypeInformation.IsQuotaAdjustable);
}
}
}
}
\ No newline at end of file
......@@ -12,7 +12,6 @@ using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
namespace Coscine.ResourceTypeWaterbutlerRdsS3
{
public class ResourceTypeWaterbutlerRdsS3 : ResourceTypeDefinition
......@@ -79,7 +78,7 @@ namespace Coscine.ResourceTypeWaterbutlerRdsS3
// Not found
if (infos == null)
{
throw new Exception($"Found nothing in waterbutler under \"{prefix}\".");
throw new Exception($"Found nothing in Waterbutler under \"{prefix}\".");
}
var entries = new List<ResourceEntry>();
......@@ -104,12 +103,12 @@ namespace Coscine.ResourceTypeWaterbutlerRdsS3
}
// Not a file
if (infos.Count > 1 || !infos.First().IsFile)
if (infos.Count > 1 || !infos[0].IsFile)
{
throw new Exception("Not a file.");
}
return new ResourceEntry(key, infos.First().IsFile, (long)infos.First().Size, null, null, infos.First().Created == null ? new DateTime() : (DateTime)infos.First().Created, infos.First().Modified == null ? new DateTime() : (DateTime)infos.First().Modified);
return new ResourceEntry(key, infos[0].IsFile, (long)infos[0].Size, null, null, infos[0].Created == null ? new DateTime() : (DateTime)infos[0].Created, infos[0].Modified == null ? new DateTime() : (DateTime)infos[0].Modified);
}
public override async Task StoreEntry(string id, string key, Stream body, Dictionary<string, string> options = null)
......@@ -125,17 +124,16 @@ namespace Coscine.ResourceTypeWaterbutlerRdsS3
var rootPath = key.Substring(0, key.Length - filename.Length);
HandleResponse(await _waterbutlerInterface.UploadFileAsync(rootPath, filename, _provider, authHeader, body, contentLength));
}
else
{
// update existing file
// Not a file
if (infos.Count > 1 || !infos.First().IsFile)
if (infos.Count > 1 || !infos[0].IsFile)
{
throw new Exception("Not an updateable or new file.");
throw new Exception("Not an updatable or new file.");
}
HandleResponse(await _waterbutlerInterface.UploadObjectAsync(infos.First(), authHeader, body, contentLength));
HandleResponse(await _waterbutlerInterface.UploadObjectAsync(infos[0], authHeader, body, contentLength));
}
}
......@@ -151,12 +149,12 @@ namespace Coscine.ResourceTypeWaterbutlerRdsS3
}
// Not a file
if (infos.Count > 1 || !infos.First().IsFile)
if (infos.Count > 1 || !infos[0].IsFile)
{
throw new Exception($"Not a file.");
throw new Exception("Not a file.");
}
HandleResponse(await _waterbutlerInterface.DeleteObjectAsync(infos.First(), authHeader));
HandleResponse(await _waterbutlerInterface.DeleteObjectAsync(infos[0], authHeader));
}
public override async Task<Stream> LoadEntry(string id, string key, string version = null, Dictionary<string, string> options = null)
......@@ -172,7 +170,7 @@ namespace Coscine.ResourceTypeWaterbutlerRdsS3
}
// Not a file
if (infos.Count > 1 || !infos.First().IsFile)
if (infos.Count > 1 || !infos[0].IsFile)
{
throw new Exception("Not a file.");
}
......@@ -180,7 +178,7 @@ namespace Coscine.ResourceTypeWaterbutlerRdsS3
// Do not dispose the response!
// The stream is later accessed by the MVC, to deliver the file.
// When disposed, the stream becomes invalid and can't be read.
var response = await _waterbutlerInterface.DownloadObjectAsync(infos.First(), authHeader);
var response = await _waterbutlerInterface.DownloadObjectAsync(infos[0], authHeader);
if (response.IsSuccessStatusCode)
{
return await response.Content.ReadAsStreamAsync();
......@@ -215,7 +213,7 @@ namespace Coscine.ResourceTypeWaterbutlerRdsS3
BucketName = id,
Policy = policy
};
var amazonConfig = new AmazonS3Config
{
ServiceURL = options["endpoint"],
......@@ -224,13 +222,15 @@ namespace Coscine.ResourceTypeWaterbutlerRdsS3
using var client = new AmazonS3Client(_accessKey, _secretKey, amazonConfig);
// Exception will be thrown on dev systems.
// Exception will be thrown on development systems.
// Works on live.
try
{
await client.PutBucketPolicyAsync(putRequest);
}
#pragma warning disable RCS1075 // Avoid empty catch clause that catches System.Exception.
catch (Exception)
#pragma warning restore RCS1075 // Avoid empty catch clause that catches System.Exception.
{
}
}
......@@ -244,8 +244,8 @@ namespace Coscine.ResourceTypeWaterbutlerRdsS3
await _rdsS3EcsManager.SetUserAcl(options["accessKeyRead"], options["bucketname"], _readRights);
await _rdsS3EcsManager.SetUserAcl(options["accessKeyWrite"], options["bucketname"], _writeRights);
// Set to the readonly value, if present.
if (options != null && options.ContainsKey("readonly") && bool.TryParse(options["readonly"], out var result))
// Set to the read only value, if present.
if (options?.ContainsKey("readonly") == true && bool.TryParse(options["readonly"], out var result))
{
await SetResourceReadonly(options["bucketname"], result, options);
}
......@@ -257,7 +257,6 @@ namespace Coscine.ResourceTypeWaterbutlerRdsS3
public override async Task<long> GetResourceQuotaUsed(string id, Dictionary<string, string> options = null)
{
var amazonConfig = new AmazonS3Config
{
ServiceURL = options["endpoint"],
......@@ -279,7 +278,6 @@ namespace Coscine.ResourceTypeWaterbutlerRdsS3
fileCount += listResponse.S3Objects.Count;
totalFileSize += listResponse.S3Objects.Sum(x => x.Size);
listRequest.Marker = listResponse.NextMarker;
} while (listResponse.IsTruncated);
return totalFileSize;
......@@ -349,11 +347,6 @@ namespace Coscine.ResourceTypeWaterbutlerRdsS3
return await Task.FromResult(resourceTypeInformation);
}
private static string GenerateAccessPolicy(Dictionary<string, string> options, bool isReadonly)
{
return GenerateAccessPolicy(options["accessKey"], options["accessKeyWrite"], options["accessKeyRead"], options["bucketname"], isReadonly);
}
private static string GenerateAccessPolicy(string accessKey, string writeKey, string accessKeyRead, string bucketname, bool isReadonly)
{
if (isReadonly)
......@@ -400,4 +393,4 @@ namespace Coscine.ResourceTypeWaterbutlerRdsS3
}
}
}
}
}
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment