DataSourceController.cs 19.9 KB
Newer Older
1 2 3
using Coscine.Api.Project.Models;
using Coscine.Api.Project.ReturnObjects;
using Coscine.ApiCommons;
4
using Coscine.ApiCommons.Factories;
5 6
using Coscine.ApiCommons.Utils;
using Coscine.Configuration;
7
using Coscine.Database.Model;
8
using Microsoft.AspNetCore.Authorization;
9 10
using Microsoft.AspNetCore.Mvc;
using Newtonsoft.Json.Linq;
11
using System;
12 13
using System.Collections.Generic;
using System.IdentityModel.Tokens.Jwt;
14
using System.IO;
15 16 17
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
18
using System.Text.RegularExpressions;
19
using System.Threading.Tasks;
20 21 22
using System.Web;

#region DupFinder Exclusion
23 24 25

namespace Coscine.Api.Project.Controllers
{
26
    [Authorize]
27 28 29 30
    public class DataSourceController : Controller
    {
        private readonly IConfiguration _configuration;
        private readonly JWTHandler _jwtHandler;
31 32
        // make to lazy property
        private static readonly HttpClient Client;
33 34
        private readonly Authenticator _authenticator;
        private readonly ResourceModel _resourceModel;
35
        private readonly ProjectModel _projectModel;
36

37 38 39 40 41 42 43 44
        static DataSourceController()
        {
            Client = new HttpClient
            {
                Timeout = TimeSpan.FromMinutes(30)
            };
        }

45 46
        public DataSourceController()
        {
47
            _configuration = Program.Configuration;
48
            _jwtHandler = new JWTHandler(_configuration);
49 50
            _authenticator = new Authenticator(this, _configuration);
            _resourceModel = new ResourceModel();
51
            _projectModel = new ProjectModel();
52 53
        }

54 55 56
        // inferring a ../ (urlencoded) can manipulate the url.
        // However the constructed signature for s3 won't match and it will not be resolved.
        // This may be a problem for other provider!
57 58
        [HttpGet("[controller]/{resourceId}/{path}")]
        public async Task<IActionResult> GetWaterButlerFolder(string resourceId, string path)
59
        {
60 61
            var user = _authenticator.GetUser();

62
            if (!string.IsNullOrWhiteSpace(path))
63
            {
64
                path = HttpUtility.UrlDecode(path);
65
            }
66

67 68
            var check = CheckResourceIdAndPath(resourceId, path, out Resource resource);
            if (check != null)
69
            {
70
                return check;
71 72
            }

73 74 75 76 77
            if (!_resourceModel.HasAccess(user, resource, UserRoles.Owner, UserRoles.Member))
            {
                return BadRequest("User does not have permission to the resource.");
            }

78
            var authHeader = BuildAuthHeader(resource);
79

80
            if (authHeader == null)
81
            {
82
                return BadRequest($"No provider for: \"{resource.Type.DisplayName}\".");
83
            }
84
            else
85 86
            {
                // If the path is null, an empty string is added.
87
                string url = $"{_configuration.GetString("coscine/global/waterbutler_url")}{GetResourceTypeName(resource)}{path}";
88 89 90 91 92

                var request = new HttpRequestMessage(HttpMethod.Get, url);
                request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", authHeader);

                // Thread safe according to msdn and HttpCompletionOption sets it to get only headers first.
93
                var response = await Client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
94
                if (response.IsSuccessStatusCode)
95
                {
96 97 98 99 100 101 102 103 104 105
                    if (response.Content.Headers.Contains("Content-Disposition"))
                    {
                        return File(await response.Content.ReadAsStreamAsync(),
                            response.Content.Headers.GetValues("Content-Type").First());
                    }
                    else
                    {
                        var data = JObject.Parse(await response.Content.ReadAsStringAsync())["data"];
                        return Ok(new WaterbutlerObject(path, data));
                    }
106 107 108
                }
                else
                {
109
                    return FailedRequest(response, path);
110 111 112 113 114 115 116 117 118 119 120
                }
            }
        }

        // inferring a ../ (urlencoded) can manipulate the url.
        // However the constructed signature for s3 won't match and it will not be resolved.
        // This may be a problem for other provider!
        [HttpPut("[controller]/{resourceId}/{path}")]
        [DisableRequestSizeLimit]
        public async Task<IActionResult> PutUploadFile(string resourceId, string path)
        {
121 122 123
            var user = _authenticator.GetUser();


124 125 126 127 128 129 130 131 132 133 134
            if (!string.IsNullOrWhiteSpace(path))
            {
                path = HttpUtility.UrlDecode(path);
            }

            var check = CheckResourceIdAndPath(resourceId, path, out Resource resource);
            if (check != null)
            {
                return check;
            }

135 136 137 138 139
            if(!_resourceModel.HasAccess(user, resource, UserRoles.Owner, UserRoles.Member))
            {
                return BadRequest("User does not have permission to the resource.");
            }

140 141 142 143 144 145 146 147 148
            var authHeader = BuildAuthHeader(resource, new string[] { "gitlab" });

            if (authHeader == null)
            {
                return BadRequest($"No provider for: \"{resource.Type.DisplayName}\".");
            }
            else
            {
                // If the path is null, an empty string is added.
149
                string url = $"{_configuration.GetString("coscine/global/waterbutler_url")}{GetResourceTypeName(resource)}/?kind=file&name={path}";
150 151 152 153 154

                try
                {
                    var response = await UploadFile(url, authHeader, Request.Body);
                    if (response.IsSuccessStatusCode)
155
                    {
156
                        return NoContent();
157 158 159
                    }
                    else
                    {
160
                        return FailedRequest(response, path);
161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177
                    }
                }
                catch (Exception e)
                {
                    Console.WriteLine(e);
                    return BadRequest(e);
                }
            }
        }

        // inferring a ../ (urlencoded) can manipulate the url.
        // However the constructed signature for s3 won't match and it will not be resolved.
        // This may be a problem for other provider!
        [HttpPut("[controller]/{resourceId}/{path}/update")]
        [DisableRequestSizeLimit]
        public async Task<IActionResult> PutUpdateFile(string resourceId, string path)
        {
178 179
            var user = _authenticator.GetUser();

180 181 182 183 184 185 186 187 188 189 190
            if (!string.IsNullOrWhiteSpace(path))
            {
                path = HttpUtility.UrlDecode(path);
            }

            var check = CheckResourceIdAndPath(resourceId, path, out Resource resource);
            if (check != null)
            {
                return check;
            }

191 192 193 194 195
            if (!_resourceModel.HasAccess(user, resource, UserRoles.Owner, UserRoles.Member))
            {
                return BadRequest("User does not have permission to the resource.");
            }

196 197 198 199 200 201 202 203 204
            var authHeader = BuildAuthHeader(resource, new string[] { "gitlab" });

            if (authHeader == null)
            {
                return BadRequest($"No provider for: \"{resource.Type.DisplayName}\".");
            }
            else
            {
                // If the path is null, an empty string is added.
205
                string url = $"{_configuration.GetString("coscine/global/waterbutler_url")}{GetResourceTypeName(resource)}/{path}?kind=file";
206 207 208 209 210 211 212

                try
                {
                    var response = await UploadFile(url, authHeader, Request.Body);
                    if (response.IsSuccessStatusCode)
                    {
                        return NoContent();
213 214 215
                    }
                    else
                    {
216
                        return FailedRequest(response, path);
217
                    }
218
                }
219 220 221 222 223
                catch (Exception e)
                {
                    Console.WriteLine(e);
                    return BadRequest(e);
                }
224
            }
225 226
        }

227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248
        private string GetResourceTypeName(Resource resource)
        {
            if (resource.Type.DisplayName.ToLower().Equals("s3")) {
                return "rds";
            }
            else
            {
                return resource.Type.DisplayName.ToLower();
            }
        }

        private string GetResourceTypeName(JToken resource)
        {
            if (resource["type"]["displayName"].ToString().ToLower().Equals("s3"))
            {
                return "rds";
            }
            else
            {
                return resource["type"]["displayName"].ToString().ToLower();
            }
        }
249

250 251 252 253 254 255 256 257 258 259 260 261

        public async Task<HttpResponseMessage> UploadFile(string url, string authHeader, Stream stream)
        {
            var request = new HttpRequestMessage(HttpMethod.Put, url);
            request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", authHeader);
            request.Content = new StreamContent(stream);
            return await Client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
        }

        [HttpDelete("[controller]/{resourceId}/{path}")]
        public async Task<IActionResult> Delete(string resourceId, string path)
        {
262 263
            var user = _authenticator.GetUser();

264 265 266 267 268 269 270 271 272 273 274
            if (!string.IsNullOrWhiteSpace(path))
            {
                path = HttpUtility.UrlDecode(path);
            }

            var check = CheckResourceIdAndPath(resourceId, path, out Resource resource);
            if (check != null)
            {
                return check;
            }

275 276 277 278 279
            if (!_resourceModel.HasAccess(user, resource, UserRoles.Owner, UserRoles.Member))
            {
                return BadRequest("User does not have permission to the resource.");
            }

280 281 282
            var authHeader = BuildAuthHeader(resource, new string[] { "gitlab" });

            if (authHeader == null)
283 284 285
            {
                return BadRequest($"No provider for: \"{resource.Type.DisplayName}\".");
            }
286 287 288
            else
            {
                // If the path is null, an empty string is added.
289
                string url = $"{_configuration.GetString("coscine/global/waterbutler_url")}{GetResourceTypeName(resource)}{path}";
290 291 292 293 294 295 296 297 298 299 300 301 302 303

                var request = new HttpRequestMessage(HttpMethod.Delete, url);
                request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", authHeader);

                // Thread safe according to msdn and HttpCompletionOption sets it to get only headers first.
                try
                {
                    var response = await Client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
                    if (response.IsSuccessStatusCode)
                    {
                        return NoContent();
                    }
                    else
                    {
304
                        return FailedRequest(response, path);
305 306 307 308 309 310 311 312 313 314
                    }
                }
                catch (Exception e)
                {
                    Console.WriteLine(e);
                    return BadRequest(e);
                }
            }
        }

315 316 317 318 319 320 321 322
        [HttpPost("[controller]/validate")]
        public async Task<IActionResult> IsResourceValid()
        {
            var path = "/";

            JToken resource = ObjectFactory<JToken>.DeserializeFromStream(Request.Body);

            string authHeader = null;
323
            if (resource["type"]["displayName"].ToString().ToLower() == "s3")
324
            {
325 326 327 328 329
                S3ResourceType s3ResourceType = new S3ResourceType();
                s3ResourceType.BucketName = resource["resourceTypeOption"]["BucketName"].ToString();
                s3ResourceType.AccessKey = resource["resourceTypeOption"]["AccessKey"].ToString();
                s3ResourceType.SecretKey = resource["resourceTypeOption"]["SecretKey"].ToString();
                authHeader = BuildS3AuthHeader(s3ResourceType);
330 331 332
            }
            else if (resource["type"]["displayName"].ToString().ToLower() == "gitlab")
            {
333 334 335 336 337 338
                GitlabResourceType gitlabResourceType = new GitlabResourceType
                {
                    RepositoryNumber = (int)resource["resourceTypeOption"]["RepositoryNumber"],
                    RepositoryUrl = resource["resourceTypeOption"]["RepositoryUrl"].ToString(),
                    Token = resource["resourceTypeOption"]["Token"].ToString()
                };
339 340 341 342 343 344 345 346 347 348
                authHeader = BuildGitlabAuthHeader(gitlabResourceType);
            }

            if (authHeader == null)
            {
                return BadRequest($"No provider for: \"{resource["type"]["displayName"].ToString()}\".");
            }
            else
            {
                // If the path is null, an empty string is added.
349
                string url = $"{_configuration.GetString("coscine/global/waterbutler_url")}{GetResourceTypeName(resource)}{path}";
350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375

                var request = new HttpRequestMessage(HttpMethod.Get, url);
                request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", authHeader);

                // Thread safe according to msdn and HttpCompletionOption sets it to get only headers first.
                var response = await Client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
                if (response.IsSuccessStatusCode)
                {
                    if (response.Content.Headers.Contains("Content-Disposition"))
                    {
                        return File(await response.Content.ReadAsStreamAsync(),
                            response.Content.Headers.GetValues("Content-Type").First());
                    }
                    else
                    {
                        var data = JObject.Parse(await response.Content.ReadAsStringAsync())["data"];
                        return Ok(new WaterbutlerObject(path, data));
                    }
                }
                else
                {
                    return FailedRequest(response, path);
                }
            }
        }

376
        private IActionResult FailedRequest(HttpResponseMessage response, string path)
377 378 379 380 381
        {
            if (response.StatusCode == System.Net.HttpStatusCode.NotFound)
            {
                return NotFound($"Could not find object for: \"{path}\".");
            }
382
            else if (response.StatusCode == System.Net.HttpStatusCode.Forbidden)
383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406
            {
                return Forbid("Not allowed to access the datasource.");
            }
            else
            {
                return BadRequest($"Error in communication with waterbutler: {response.StatusCode}");
            }
        }

        private IActionResult CheckResourceIdAndPath(string resourceId, string path, out Resource resource)
        {
            resource = null;

            if (string.IsNullOrWhiteSpace(path))
            {
                return BadRequest($"Your path \"{path}\" is empty.");
            }

            Regex rgx = new Regex(@"^[0-9a-zA-Z_\-/. ]+$");
            if (!rgx.IsMatch(path))
            {
                return BadRequest($"Your path \"{path}\" contains bad chars. Only {@"^[0-9a-zA-Z_\-./ ]+"} are allowed as chars.");
            }

407
            if (!Guid.TryParse(resourceId, out Guid resourceGuid))
408 409 410
            {
                return BadRequest($"{resourceId} is not a guid.");
            }
411

412 413
            try
            {
414
                resource = _resourceModel.GetById(resourceGuid);
415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432
                if (resource == null)
                {
                    return NotFound($"Could not find resource with id: {resourceId}");
                }
            }
            catch (Exception)
            {
                return NotFound($"Could not find resource with id: {resourceId}");
            }

            if (resource.Type == null)
            {
                ResourceTypeModel resourceTypeModel = new ResourceTypeModel();
                resource.Type = resourceTypeModel.GetById(resource.TypeId);
            }

            // All good
            return null;
433 434
        }

435
        private string BuildWaterbutlerPayload(Dictionary<string, object> auth, Dictionary<string, object> credentials, Dictionary<string, object> settings)
436 437
        {
            var data = new Dictionary<string, object>
438 439 440 441 442 443
            {
                { "auth", auth },
                { "credentials", credentials },
                { "settings", settings },
                { "callback_url", "rwth-aachen.de" }
            };
444 445

            var payload = new JwtPayload
446 447 448
            {
                { "data", data }
            };
449 450 451

            return _jwtHandler.GenerateJwtToken(payload);
        }
452

453 454 455 456 457 458 459 460 461 462 463 464 465 466 467
        private string BuildAuthHeader(Resource resource, IEnumerable<string> exclude = null)
        {
            if (exclude != null && exclude.Contains(resource.Type.DisplayName.ToLower()))
            {
                return null;
            }

            string authHeader = null;
            if (resource.Type.DisplayName.ToLower() == "rds")
            {
                RDSResourceTypeModel rdsResourceTypeModel = new RDSResourceTypeModel();
                var rdsResourceType = rdsResourceTypeModel.GetById(resource.ResourceTypeOptionId.Value);

                authHeader = BuildRdsAuthHeader(rdsResourceType);
            }
468 469 470 471 472 473 474
            else if (resource.Type.DisplayName.ToLower() == "s3")
            {
                S3ResourceTypeModel s3ResourceTypeModel = new S3ResourceTypeModel();
                var s3ResourceType = s3ResourceTypeModel.GetById(resource.ResourceTypeOptionId.Value);

                authHeader = BuildS3AuthHeader(s3ResourceType);
            }
475 476 477 478 479 480 481 482 483 484 485
            else if (resource.Type.DisplayName.ToLower() == "gitlab")
            {
                GitlabResourceTypeModel gitlabResourceTypeModel = new GitlabResourceTypeModel();
                var gitlabResourceType = gitlabResourceTypeModel.GetById(resource.ResourceTypeOptionId.Value);

                authHeader = BuildGitlabAuthHeader(gitlabResourceType);
            }

            return authHeader;
        }

486
        private string BuildRdsAuthHeader(RDSResourceType rdsResourceType)
487 488 489 490 491
        {
            var auth = new Dictionary<string, object>();

            var credentials = new Dictionary<string, object>
            {
492 493
                { "access_key", _configuration.GetStringAndWait("coscine/global/buckets/accessKey") },
                { "secret_key", _configuration.GetStringAndWait("coscine/global/buckets/secretKey") }
494 495 496 497
            };

            var settings = new Dictionary<string, object>
            {
498
                { "bucket", rdsResourceType.BucketName }
499 500 501 502 503
            };

            return BuildWaterbutlerPayload(auth, credentials, settings);
        }

504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521
        private string BuildS3AuthHeader(S3ResourceType s3ResourceType)
        {
            var auth = new Dictionary<string, object>();

            var credentials = new Dictionary<string, object>
            {
                { "access_key", s3ResourceType.AccessKey },
                { "secret_key", s3ResourceType.SecretKey }
            };

            var settings = new Dictionary<string, object>
            {
                { "bucket", s3ResourceType.BucketName }
            };

            return BuildWaterbutlerPayload(auth, credentials, settings);
        }

522
        private string BuildGitlabAuthHeader(GitlabResourceType gitlabResourceType)
523
        {
524 525 526 527 528

            var auth = new Dictionary<string, object>();

            var credentials = new Dictionary<string, object>
            {
529
                { "token", gitlabResourceType.Token }
530 531 532 533
            };

            var settings = new Dictionary<string, object>
            {
534
                { "owner", "Tester"},
535
                { "repo", gitlabResourceType.RepositoryUrl},
536
                { "repo_id", gitlabResourceType.RepositoryNumber.ToString()},
537 538 539
                { "host", "https://git.rwth-aachen.de"}
            };

540
            return BuildWaterbutlerPayload(auth, credentials, settings);
541
        }
542 543
    }
}
544
#endregion