using Coscine.Api.Project.Models; using Coscine.Api.Project.ReturnObjects; using Coscine.ApiCommons; using Coscine.ApiCommons.Factories; using Coscine.ApiCommons.Utils; using Coscine.Configuration; using Coscine.Database.Model; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Newtonsoft.Json.Linq; using System; using System.Collections.Generic; using System.IdentityModel.Tokens.Jwt; using System.IO; using System.Linq; using System.Net.Http; using System.Net.Http.Headers; using System.Text.RegularExpressions; using System.Threading.Tasks; using System.Web; #region DupFinder Exclusion namespace Coscine.Api.Project.Controllers { [Authorize] public class DataSourceController : Controller { private readonly IConfiguration _configuration; private readonly JWTHandler _jwtHandler; // make to lazy property private static readonly HttpClient Client; private readonly Authenticator _authenticator; private readonly ResourceModel _resourceModel; private readonly ProjectModel _projectModel; static DataSourceController() { Client = new HttpClient { Timeout = TimeSpan.FromMinutes(30) }; } public DataSourceController() { _configuration = Program.Configuration; _jwtHandler = new JWTHandler(_configuration); _authenticator = new Authenticator(this, _configuration); _resourceModel = new ResourceModel(); _projectModel = new ProjectModel(); } // inferring a ../ (urlencoded) can manipulate the url. // However the constructed signature for s3 won't match and it will not be resolved. // This may be a problem for other provider! [HttpGet("[controller]/{resourceId}/{path}")] public async Task<IActionResult> GetWaterButlerFolder(string resourceId, string path) { var user = _authenticator.GetUser(); if (!string.IsNullOrWhiteSpace(path)) { path = HttpUtility.UrlDecode(path); } var check = CheckResourceIdAndPath(resourceId, path, out Resource resource); if (check != null) { return check; } if (!_resourceModel.HasAccess(user, resource, UserRoles.Owner, UserRoles.Member)) { return BadRequest("User does not have permission to the resource."); } var authHeader = BuildAuthHeader(resource); if (authHeader == null) { return BadRequest($"No provider for: \"{resource.Type.DisplayName}\"."); } else { // If the path is null, an empty string is added. string url = $"{_configuration.GetString("coscine/global/waterbutler_url")}{GetResourceTypeName(resource)}{path}"; var request = new HttpRequestMessage(HttpMethod.Get, url); request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", authHeader); // Thread safe according to msdn and HttpCompletionOption sets it to get only headers first. var response = await Client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); if (response.IsSuccessStatusCode) { if (response.Content.Headers.Contains("Content-Disposition")) { return File(await response.Content.ReadAsStreamAsync(), response.Content.Headers.GetValues("Content-Type").First()); } else { var data = JObject.Parse(await response.Content.ReadAsStringAsync())["data"]; return Ok(new WaterbutlerObject(path, data)); } } else { return FailedRequest(response, path); } } } // inferring a ../ (urlencoded) can manipulate the url. // However the constructed signature for s3 won't match and it will not be resolved. // This may be a problem for other provider! [HttpPut("[controller]/{resourceId}/{path}")] [DisableRequestSizeLimit] public async Task<IActionResult> PutUploadFile(string resourceId, string path) { var user = _authenticator.GetUser(); if (!string.IsNullOrWhiteSpace(path)) { path = HttpUtility.UrlDecode(path); } var check = CheckResourceIdAndPath(resourceId, path, out Resource resource); if (check != null) { return check; } if(!_resourceModel.HasAccess(user, resource, UserRoles.Owner, UserRoles.Member)) { return BadRequest("User does not have permission to the resource."); } var authHeader = BuildAuthHeader(resource, new string[] { "gitlab" }); if (authHeader == null) { return BadRequest($"No provider for: \"{resource.Type.DisplayName}\"."); } else { // If the path is null, an empty string is added. string url = $"{_configuration.GetString("coscine/global/waterbutler_url")}{GetResourceTypeName(resource)}/?kind=file&name={path}"; try { var response = await UploadFile(url, authHeader, Request.Body); if (response.IsSuccessStatusCode) { return NoContent(); } else { return FailedRequest(response, path); } } catch (Exception e) { Console.WriteLine(e); return BadRequest(e); } } } // inferring a ../ (urlencoded) can manipulate the url. // However the constructed signature for s3 won't match and it will not be resolved. // This may be a problem for other provider! [HttpPut("[controller]/{resourceId}/{path}/update")] [DisableRequestSizeLimit] public async Task<IActionResult> PutUpdateFile(string resourceId, string path) { var user = _authenticator.GetUser(); if (!string.IsNullOrWhiteSpace(path)) { path = HttpUtility.UrlDecode(path); } var check = CheckResourceIdAndPath(resourceId, path, out Resource resource); if (check != null) { return check; } if (!_resourceModel.HasAccess(user, resource, UserRoles.Owner, UserRoles.Member)) { return BadRequest("User does not have permission to the resource."); } var authHeader = BuildAuthHeader(resource, new string[] { "gitlab" }); if (authHeader == null) { return BadRequest($"No provider for: \"{resource.Type.DisplayName}\"."); } else { // If the path is null, an empty string is added. string url = $"{_configuration.GetString("coscine/global/waterbutler_url")}{GetResourceTypeName(resource)}/{path}?kind=file"; try { var response = await UploadFile(url, authHeader, Request.Body); if (response.IsSuccessStatusCode) { return NoContent(); } else { return FailedRequest(response, path); } } catch (Exception e) { Console.WriteLine(e); return BadRequest(e); } } } private string GetResourceTypeName(Resource resource) { if (resource.Type.DisplayName.ToLower().Equals("s3")) { return "rds"; } else { return resource.Type.DisplayName.ToLower(); } } private string GetResourceTypeName(JToken resource) { if (resource["type"]["displayName"].ToString().ToLower().Equals("s3")) { return "rds"; } else { return resource["type"]["displayName"].ToString().ToLower(); } } public async Task<HttpResponseMessage> UploadFile(string url, string authHeader, Stream stream) { var request = new HttpRequestMessage(HttpMethod.Put, url); request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", authHeader); request.Content = new StreamContent(stream); return await Client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); } [HttpDelete("[controller]/{resourceId}/{path}")] public async Task<IActionResult> Delete(string resourceId, string path) { var user = _authenticator.GetUser(); if (!string.IsNullOrWhiteSpace(path)) { path = HttpUtility.UrlDecode(path); } var check = CheckResourceIdAndPath(resourceId, path, out Resource resource); if (check != null) { return check; } if (!_resourceModel.HasAccess(user, resource, UserRoles.Owner, UserRoles.Member)) { return BadRequest("User does not have permission to the resource."); } var authHeader = BuildAuthHeader(resource, new string[] { "gitlab" }); if (authHeader == null) { return BadRequest($"No provider for: \"{resource.Type.DisplayName}\"."); } else { // If the path is null, an empty string is added. string url = $"{_configuration.GetString("coscine/global/waterbutler_url")}{GetResourceTypeName(resource)}{path}"; var request = new HttpRequestMessage(HttpMethod.Delete, url); request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", authHeader); // Thread safe according to msdn and HttpCompletionOption sets it to get only headers first. try { var response = await Client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); if (response.IsSuccessStatusCode) { return NoContent(); } else { return FailedRequest(response, path); } } catch (Exception e) { Console.WriteLine(e); return BadRequest(e); } } } [HttpPost("[controller]/validate")] public async Task<IActionResult> IsResourceValid() { var path = "/"; JToken resource = ObjectFactory<JToken>.DeserializeFromStream(Request.Body); string authHeader = null; if (resource["type"]["displayName"].ToString().ToLower() == "s3") { S3ResourceType s3ResourceType = new S3ResourceType(); s3ResourceType.BucketName = resource["resourceTypeOption"]["BucketName"].ToString(); s3ResourceType.AccessKey = resource["resourceTypeOption"]["AccessKey"].ToString(); s3ResourceType.SecretKey = resource["resourceTypeOption"]["SecretKey"].ToString(); authHeader = BuildS3AuthHeader(s3ResourceType); } else if (resource["type"]["displayName"].ToString().ToLower() == "gitlab") { GitlabResourceType gitlabResourceType = new GitlabResourceType { RepositoryNumber = (int)resource["resourceTypeOption"]["RepositoryNumber"], RepositoryUrl = resource["resourceTypeOption"]["RepositoryUrl"].ToString(), Token = resource["resourceTypeOption"]["Token"].ToString() }; authHeader = BuildGitlabAuthHeader(gitlabResourceType); } if (authHeader == null) { return BadRequest($"No provider for: \"{resource["type"]["displayName"].ToString()}\"."); } else { // If the path is null, an empty string is added. string url = $"{_configuration.GetString("coscine/global/waterbutler_url")}{GetResourceTypeName(resource)}{path}"; var request = new HttpRequestMessage(HttpMethod.Get, url); request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", authHeader); // Thread safe according to msdn and HttpCompletionOption sets it to get only headers first. var response = await Client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); if (response.IsSuccessStatusCode) { if (response.Content.Headers.Contains("Content-Disposition")) { return File(await response.Content.ReadAsStreamAsync(), response.Content.Headers.GetValues("Content-Type").First()); } else { var data = JObject.Parse(await response.Content.ReadAsStringAsync())["data"]; return Ok(new WaterbutlerObject(path, data)); } } else { return FailedRequest(response, path); } } } private IActionResult FailedRequest(HttpResponseMessage response, string path) { if (response.StatusCode == System.Net.HttpStatusCode.NotFound) { return NotFound($"Could not find object for: \"{path}\"."); } else if (response.StatusCode == System.Net.HttpStatusCode.Forbidden) { return Forbid("Not allowed to access the datasource."); } else { return BadRequest($"Error in communication with waterbutler: {response.StatusCode}"); } } private IActionResult CheckResourceIdAndPath(string resourceId, string path, out Resource resource) { resource = null; if (string.IsNullOrWhiteSpace(path)) { return BadRequest($"Your path \"{path}\" is empty."); } Regex rgx = new Regex(@"^[0-9a-zA-Z_\-/. ]+$"); if (!rgx.IsMatch(path)) { return BadRequest($"Your path \"{path}\" contains bad chars. Only {@"^[0-9a-zA-Z_\-./ ]+"} are allowed as chars."); } if (!Guid.TryParse(resourceId, out Guid resourceGuid)) { return BadRequest($"{resourceId} is not a guid."); } try { resource = _resourceModel.GetById(resourceGuid); if (resource == null) { return NotFound($"Could not find resource with id: {resourceId}"); } } catch (Exception) { return NotFound($"Could not find resource with id: {resourceId}"); } if (resource.Type == null) { ResourceTypeModel resourceTypeModel = new ResourceTypeModel(); resource.Type = resourceTypeModel.GetById(resource.TypeId); } // All good return null; } private string BuildWaterbutlerPayload(Dictionary<string, object> auth, Dictionary<string, object> credentials, Dictionary<string, object> settings) { var data = new Dictionary<string, object> { { "auth", auth }, { "credentials", credentials }, { "settings", settings }, { "callback_url", "rwth-aachen.de" } }; var payload = new JwtPayload { { "data", data } }; return _jwtHandler.GenerateJwtToken(payload); } private string BuildAuthHeader(Resource resource, IEnumerable<string> exclude = null) { if (exclude != null && exclude.Contains(resource.Type.DisplayName.ToLower())) { return null; } string authHeader = null; if (resource.Type.DisplayName.ToLower() == "rds") { RDSResourceTypeModel rdsResourceTypeModel = new RDSResourceTypeModel(); var rdsResourceType = rdsResourceTypeModel.GetById(resource.ResourceTypeOptionId.Value); authHeader = BuildRdsAuthHeader(rdsResourceType); } else if (resource.Type.DisplayName.ToLower() == "s3") { S3ResourceTypeModel s3ResourceTypeModel = new S3ResourceTypeModel(); var s3ResourceType = s3ResourceTypeModel.GetById(resource.ResourceTypeOptionId.Value); authHeader = BuildS3AuthHeader(s3ResourceType); } else if (resource.Type.DisplayName.ToLower() == "gitlab") { GitlabResourceTypeModel gitlabResourceTypeModel = new GitlabResourceTypeModel(); var gitlabResourceType = gitlabResourceTypeModel.GetById(resource.ResourceTypeOptionId.Value); authHeader = BuildGitlabAuthHeader(gitlabResourceType); } return authHeader; } private string BuildRdsAuthHeader(RDSResourceType rdsResourceType) { var auth = new Dictionary<string, object>(); var credentials = new Dictionary<string, object> { { "access_key", _configuration.GetStringAndWait("coscine/global/buckets/accessKey") }, { "secret_key", _configuration.GetStringAndWait("coscine/global/buckets/secretKey") } }; var settings = new Dictionary<string, object> { { "bucket", rdsResourceType.BucketName } }; return BuildWaterbutlerPayload(auth, credentials, settings); } private string BuildS3AuthHeader(S3ResourceType s3ResourceType) { var auth = new Dictionary<string, object>(); var credentials = new Dictionary<string, object> { { "access_key", s3ResourceType.AccessKey }, { "secret_key", s3ResourceType.SecretKey } }; var settings = new Dictionary<string, object> { { "bucket", s3ResourceType.BucketName } }; return BuildWaterbutlerPayload(auth, credentials, settings); } private string BuildGitlabAuthHeader(GitlabResourceType gitlabResourceType) { var auth = new Dictionary<string, object>(); var credentials = new Dictionary<string, object> { { "token", gitlabResourceType.Token } }; var settings = new Dictionary<string, object> { { "owner", "Tester"}, { "repo", gitlabResourceType.RepositoryUrl}, { "repo_id", gitlabResourceType.RepositoryNumber.ToString()}, { "host", "https://git.rwth-aachen.de"} }; return BuildWaterbutlerPayload(auth, credentials, settings); } } } #endregion