DataSourceController.cs 19.8 KB
Newer Older
1 2 3
using Coscine.Api.Project.Models;
using Coscine.Api.Project.ReturnObjects;
using Coscine.ApiCommons;
4
using Coscine.ApiCommons.Factories;
5 6
using Coscine.ApiCommons.Utils;
using Coscine.Configuration;
7
using Coscine.Database.Model;
8
using Microsoft.AspNetCore.Authorization;
9 10
using Microsoft.AspNetCore.Mvc;
using Newtonsoft.Json.Linq;
11
using System;
12 13
using System.Collections.Generic;
using System.IdentityModel.Tokens.Jwt;
14
using System.IO;
15 16 17
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
18
using System.Text.RegularExpressions;
19
using System.Threading.Tasks;
20 21 22
using System.Web;

#region DupFinder Exclusion
23 24 25

namespace Coscine.Api.Project.Controllers
{
26
    [Authorize]
27 28 29 30
    public class DataSourceController : Controller
    {
        private readonly IConfiguration _configuration;
        private readonly JWTHandler _jwtHandler;
31 32
        // make to lazy property
        private static readonly HttpClient Client;
33 34
        private readonly Authenticator _authenticator;
        private readonly ResourceModel _resourceModel;
35
        private readonly ProjectModel _projectModel;
36

37 38 39 40 41 42 43 44
        static DataSourceController()
        {
            Client = new HttpClient
            {
                Timeout = TimeSpan.FromMinutes(30)
            };
        }

45 46
        public DataSourceController()
        {
47
            _configuration = Program.Configuration;
48
            _jwtHandler = new JWTHandler(_configuration);
49 50
            _authenticator = new Authenticator(this, _configuration);
            _resourceModel = new ResourceModel();
51
            _projectModel = new ProjectModel();
52 53
        }

54 55 56
        // inferring a ../ (urlencoded) can manipulate the url.
        // However the constructed signature for s3 won't match and it will not be resolved.
        // This may be a problem for other provider!
57 58
        [HttpGet("[controller]/{resourceId}/{path}")]
        public async Task<IActionResult> GetWaterButlerFolder(string resourceId, string path)
59
        {
60 61
            var user = _authenticator.GetUser();

62
            path = FormatPath(path);
63

64 65
            var check = CheckResourceIdAndPath(resourceId, path, out Resource resource);
            if (check != null)
66
            {
67
                return check;
68 69
            }

70 71 72 73 74
            if (!_resourceModel.HasAccess(user, resource, UserRoles.Owner, UserRoles.Member))
            {
                return BadRequest("User does not have permission to the resource.");
            }

75
            var authHeader = BuildAuthHeader(resource);
76

77
            if (authHeader == null)
78
            {
79
                return BadRequest($"No provider for: \"{resource.Type.DisplayName}\".");
80
            }
81
            else
82 83
            {
                // If the path is null, an empty string is added.
84
                string url = $"{_configuration.GetString("coscine/global/waterbutler_url")}{GetResourceTypeName(resource)}{path}";
85 86 87 88 89

                var request = new HttpRequestMessage(HttpMethod.Get, url);
                request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", authHeader);

                // Thread safe according to msdn and HttpCompletionOption sets it to get only headers first.
90
                var response = await Client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
91
                if (response.IsSuccessStatusCode)
92
                {
93 94 95 96 97 98 99 100 101 102
                    if (response.Content.Headers.Contains("Content-Disposition"))
                    {
                        return File(await response.Content.ReadAsStreamAsync(),
                            response.Content.Headers.GetValues("Content-Type").First());
                    }
                    else
                    {
                        var data = JObject.Parse(await response.Content.ReadAsStringAsync())["data"];
                        return Ok(new WaterbutlerObject(path, data));
                    }
103 104 105
                }
                else
                {
106
                    return FailedRequest(response, path);
107 108 109 110 111 112 113 114 115 116 117
                }
            }
        }

        // inferring a ../ (urlencoded) can manipulate the url.
        // However the constructed signature for s3 won't match and it will not be resolved.
        // This may be a problem for other provider!
        [HttpPut("[controller]/{resourceId}/{path}")]
        [DisableRequestSizeLimit]
        public async Task<IActionResult> PutUploadFile(string resourceId, string path)
        {
118 119
            var user = _authenticator.GetUser();

120
            path = FormatPath(path);
121 122 123 124 125 126 127

            var check = CheckResourceIdAndPath(resourceId, path, out Resource resource);
            if (check != null)
            {
                return check;
            }

128 129 130 131 132
            if(!_resourceModel.HasAccess(user, resource, UserRoles.Owner, UserRoles.Member))
            {
                return BadRequest("User does not have permission to the resource.");
            }

133 134 135 136 137 138 139 140 141
            var authHeader = BuildAuthHeader(resource, new string[] { "gitlab" });

            if (authHeader == null)
            {
                return BadRequest($"No provider for: \"{resource.Type.DisplayName}\".");
            }
            else
            {
                // If the path is null, an empty string is added.
142
                string url = $"{_configuration.GetString("coscine/global/waterbutler_url")}{GetResourceTypeName(resource)}/?kind=file&name={path}";
143 144 145 146 147

                try
                {
                    var response = await UploadFile(url, authHeader, Request.Body);
                    if (response.IsSuccessStatusCode)
148
                    {
149
                        return NoContent();
150 151 152
                    }
                    else
                    {
153
                        return FailedRequest(response, path);
154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170
                    }
                }
                catch (Exception e)
                {
                    Console.WriteLine(e);
                    return BadRequest(e);
                }
            }
        }

        // inferring a ../ (urlencoded) can manipulate the url.
        // However the constructed signature for s3 won't match and it will not be resolved.
        // This may be a problem for other provider!
        [HttpPut("[controller]/{resourceId}/{path}/update")]
        [DisableRequestSizeLimit]
        public async Task<IActionResult> PutUpdateFile(string resourceId, string path)
        {
171 172
            var user = _authenticator.GetUser();

173
            path = FormatPath(path);
174 175 176 177 178 179 180

            var check = CheckResourceIdAndPath(resourceId, path, out Resource resource);
            if (check != null)
            {
                return check;
            }

181 182 183 184 185
            if (!_resourceModel.HasAccess(user, resource, UserRoles.Owner, UserRoles.Member))
            {
                return BadRequest("User does not have permission to the resource.");
            }

186 187 188 189 190 191 192 193 194
            var authHeader = BuildAuthHeader(resource, new string[] { "gitlab" });

            if (authHeader == null)
            {
                return BadRequest($"No provider for: \"{resource.Type.DisplayName}\".");
            }
            else
            {
                // If the path is null, an empty string is added.
195
                string url = $"{_configuration.GetString("coscine/global/waterbutler_url")}{GetResourceTypeName(resource)}/{path}?kind=file";
196 197 198 199 200 201 202

                try
                {
                    var response = await UploadFile(url, authHeader, Request.Body);
                    if (response.IsSuccessStatusCode)
                    {
                        return NoContent();
203 204 205
                    }
                    else
                    {
206
                        return FailedRequest(response, path);
207
                    }
208
                }
209 210 211 212 213
                catch (Exception e)
                {
                    Console.WriteLine(e);
                    return BadRequest(e);
                }
214
            }
215 216
        }

217 218 219 220 221 222 223 224 225 226 227
        private string FormatPath(string path)
        {
            if (!string.IsNullOrWhiteSpace(path))
            {
                path = HttpUtility.UrlDecode(path);
                path = path.Replace(@"\", "/");
            }

            return path;
        }

228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249
        private string GetResourceTypeName(Resource resource)
        {
            if (resource.Type.DisplayName.ToLower().Equals("s3")) {
                return "rds";
            }
            else
            {
                return resource.Type.DisplayName.ToLower();
            }
        }

        private string GetResourceTypeName(JToken resource)
        {
            if (resource["type"]["displayName"].ToString().ToLower().Equals("s3"))
            {
                return "rds";
            }
            else
            {
                return resource["type"]["displayName"].ToString().ToLower();
            }
        }
250

251 252 253 254 255 256 257 258 259 260 261 262

        public async Task<HttpResponseMessage> UploadFile(string url, string authHeader, Stream stream)
        {
            var request = new HttpRequestMessage(HttpMethod.Put, url);
            request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", authHeader);
            request.Content = new StreamContent(stream);
            return await Client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
        }

        [HttpDelete("[controller]/{resourceId}/{path}")]
        public async Task<IActionResult> Delete(string resourceId, string path)
        {
263 264
            var user = _authenticator.GetUser();

265
            path = FormatPath(path);
266 267 268 269 270 271 272

            var check = CheckResourceIdAndPath(resourceId, path, out Resource resource);
            if (check != null)
            {
                return check;
            }

273 274 275 276 277
            if (!_resourceModel.HasAccess(user, resource, UserRoles.Owner, UserRoles.Member))
            {
                return BadRequest("User does not have permission to the resource.");
            }

278 279 280
            var authHeader = BuildAuthHeader(resource, new string[] { "gitlab" });

            if (authHeader == null)
281 282 283
            {
                return BadRequest($"No provider for: \"{resource.Type.DisplayName}\".");
            }
284 285 286
            else
            {
                // If the path is null, an empty string is added.
287
                string url = $"{_configuration.GetString("coscine/global/waterbutler_url")}{GetResourceTypeName(resource)}{path}";
288 289 290 291 292 293 294 295 296 297 298 299 300 301

                var request = new HttpRequestMessage(HttpMethod.Delete, url);
                request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", authHeader);

                // Thread safe according to msdn and HttpCompletionOption sets it to get only headers first.
                try
                {
                    var response = await Client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
                    if (response.IsSuccessStatusCode)
                    {
                        return NoContent();
                    }
                    else
                    {
302
                        return FailedRequest(response, path);
303 304 305 306 307 308 309 310 311 312
                    }
                }
                catch (Exception e)
                {
                    Console.WriteLine(e);
                    return BadRequest(e);
                }
            }
        }

313 314 315 316 317 318 319 320
        [HttpPost("[controller]/validate")]
        public async Task<IActionResult> IsResourceValid()
        {
            var path = "/";

            JToken resource = ObjectFactory<JToken>.DeserializeFromStream(Request.Body);

            string authHeader = null;
321
            if (resource["type"]["displayName"].ToString().ToLower() == "s3")
322
            {
323 324 325 326 327
                S3ResourceType s3ResourceType = new S3ResourceType();
                s3ResourceType.BucketName = resource["resourceTypeOption"]["BucketName"].ToString();
                s3ResourceType.AccessKey = resource["resourceTypeOption"]["AccessKey"].ToString();
                s3ResourceType.SecretKey = resource["resourceTypeOption"]["SecretKey"].ToString();
                authHeader = BuildS3AuthHeader(s3ResourceType);
328 329 330
            }
            else if (resource["type"]["displayName"].ToString().ToLower() == "gitlab")
            {
331 332 333 334 335 336
                GitlabResourceType gitlabResourceType = new GitlabResourceType
                {
                    RepositoryNumber = (int)resource["resourceTypeOption"]["RepositoryNumber"],
                    RepositoryUrl = resource["resourceTypeOption"]["RepositoryUrl"].ToString(),
                    Token = resource["resourceTypeOption"]["Token"].ToString()
                };
337 338 339 340 341 342 343 344 345 346
                authHeader = BuildGitlabAuthHeader(gitlabResourceType);
            }

            if (authHeader == null)
            {
                return BadRequest($"No provider for: \"{resource["type"]["displayName"].ToString()}\".");
            }
            else
            {
                // If the path is null, an empty string is added.
347
                string url = $"{_configuration.GetString("coscine/global/waterbutler_url")}{GetResourceTypeName(resource)}{path}";
348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373

                var request = new HttpRequestMessage(HttpMethod.Get, url);
                request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", authHeader);

                // Thread safe according to msdn and HttpCompletionOption sets it to get only headers first.
                var response = await Client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
                if (response.IsSuccessStatusCode)
                {
                    if (response.Content.Headers.Contains("Content-Disposition"))
                    {
                        return File(await response.Content.ReadAsStreamAsync(),
                            response.Content.Headers.GetValues("Content-Type").First());
                    }
                    else
                    {
                        var data = JObject.Parse(await response.Content.ReadAsStringAsync())["data"];
                        return Ok(new WaterbutlerObject(path, data));
                    }
                }
                else
                {
                    return FailedRequest(response, path);
                }
            }
        }

374
        private IActionResult FailedRequest(HttpResponseMessage response, string path)
375 376 377 378 379
        {
            if (response.StatusCode == System.Net.HttpStatusCode.NotFound)
            {
                return NotFound($"Could not find object for: \"{path}\".");
            }
380
            else if (response.StatusCode == System.Net.HttpStatusCode.Forbidden)
381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398
            {
                return Forbid("Not allowed to access the datasource.");
            }
            else
            {
                return BadRequest($"Error in communication with waterbutler: {response.StatusCode}");
            }
        }

        private IActionResult CheckResourceIdAndPath(string resourceId, string path, out Resource resource)
        {
            resource = null;

            if (string.IsNullOrWhiteSpace(path))
            {
                return BadRequest($"Your path \"{path}\" is empty.");
            }

399 400
            Regex rgx = new Regex(@"[\:?*<>|]+");
            if (rgx.IsMatch(path))
401
            {
402
                return BadRequest($"Your path \"{path}\" contains bad characters. The following characters are not permissible: {@"\/:?*<>|"}.");
403 404
            }

405
            if (!Guid.TryParse(resourceId, out Guid resourceGuid))
406 407 408
            {
                return BadRequest($"{resourceId} is not a guid.");
            }
409

410 411
            try
            {
412
                resource = _resourceModel.GetById(resourceGuid);
413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430
                if (resource == null)
                {
                    return NotFound($"Could not find resource with id: {resourceId}");
                }
            }
            catch (Exception)
            {
                return NotFound($"Could not find resource with id: {resourceId}");
            }

            if (resource.Type == null)
            {
                ResourceTypeModel resourceTypeModel = new ResourceTypeModel();
                resource.Type = resourceTypeModel.GetById(resource.TypeId);
            }

            // All good
            return null;
431 432
        }

433
        private string BuildWaterbutlerPayload(Dictionary<string, object> auth, Dictionary<string, object> credentials, Dictionary<string, object> settings)
434 435
        {
            var data = new Dictionary<string, object>
436 437 438 439 440 441
            {
                { "auth", auth },
                { "credentials", credentials },
                { "settings", settings },
                { "callback_url", "rwth-aachen.de" }
            };
442 443

            var payload = new JwtPayload
444 445 446
            {
                { "data", data }
            };
447 448 449

            return _jwtHandler.GenerateJwtToken(payload);
        }
450

451 452 453 454 455 456 457 458 459 460 461 462 463 464 465
        private string BuildAuthHeader(Resource resource, IEnumerable<string> exclude = null)
        {
            if (exclude != null && exclude.Contains(resource.Type.DisplayName.ToLower()))
            {
                return null;
            }

            string authHeader = null;
            if (resource.Type.DisplayName.ToLower() == "rds")
            {
                RDSResourceTypeModel rdsResourceTypeModel = new RDSResourceTypeModel();
                var rdsResourceType = rdsResourceTypeModel.GetById(resource.ResourceTypeOptionId.Value);

                authHeader = BuildRdsAuthHeader(rdsResourceType);
            }
466 467 468 469 470 471 472
            else if (resource.Type.DisplayName.ToLower() == "s3")
            {
                S3ResourceTypeModel s3ResourceTypeModel = new S3ResourceTypeModel();
                var s3ResourceType = s3ResourceTypeModel.GetById(resource.ResourceTypeOptionId.Value);

                authHeader = BuildS3AuthHeader(s3ResourceType);
            }
473 474 475 476 477 478 479 480 481 482 483
            else if (resource.Type.DisplayName.ToLower() == "gitlab")
            {
                GitlabResourceTypeModel gitlabResourceTypeModel = new GitlabResourceTypeModel();
                var gitlabResourceType = gitlabResourceTypeModel.GetById(resource.ResourceTypeOptionId.Value);

                authHeader = BuildGitlabAuthHeader(gitlabResourceType);
            }

            return authHeader;
        }

484
        private string BuildRdsAuthHeader(RDSResourceType rdsResourceType)
485 486 487 488 489
        {
            var auth = new Dictionary<string, object>();

            var credentials = new Dictionary<string, object>
            {
490 491
                { "access_key", _configuration.GetStringAndWait("coscine/global/buckets/accessKey") },
                { "secret_key", _configuration.GetStringAndWait("coscine/global/buckets/secretKey") }
492 493 494 495
            };

            var settings = new Dictionary<string, object>
            {
496
                { "bucket", rdsResourceType.BucketName }
497 498 499 500 501
            };

            return BuildWaterbutlerPayload(auth, credentials, settings);
        }

502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519
        private string BuildS3AuthHeader(S3ResourceType s3ResourceType)
        {
            var auth = new Dictionary<string, object>();

            var credentials = new Dictionary<string, object>
            {
                { "access_key", s3ResourceType.AccessKey },
                { "secret_key", s3ResourceType.SecretKey }
            };

            var settings = new Dictionary<string, object>
            {
                { "bucket", s3ResourceType.BucketName }
            };

            return BuildWaterbutlerPayload(auth, credentials, settings);
        }

520
        private string BuildGitlabAuthHeader(GitlabResourceType gitlabResourceType)
521
        {
522 523 524 525 526

            var auth = new Dictionary<string, object>();

            var credentials = new Dictionary<string, object>
            {
527
                { "token", gitlabResourceType.Token }
528 529 530 531
            };

            var settings = new Dictionary<string, object>
            {
532
                { "owner", "Tester"},
533
                { "repo", gitlabResourceType.RepositoryUrl},
534
                { "repo_id", gitlabResourceType.RepositoryNumber.ToString()},
535 536 537
                { "host", "https://git.rwth-aachen.de"}
            };

538
            return BuildWaterbutlerPayload(auth, credentials, settings);
539
        }
540 541
    }
}
542
#endregion