using Coscine.WaterbutlerHelper; using Coscine.WaterbutlerHelper.ReturnObjects; using Coscine.ApiCommons; using Coscine.ApiCommons.Factories; using Coscine.Database.DataModel; using Coscine.Database.Models; using Coscine.Database.Util; using Coscine.Logging; using Coscine.Metadata; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; using Newtonsoft.Json.Linq; using System; using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; using System.Threading.Tasks; using System.Web; using VDS.RDF; using VDS.RDF.Parsing; using VDS.RDF.Writing; using Coscine.ResourceLoader; using Coscine.Configuration; using Coscine.ResourceTypeBase; namespace Coscine.Api.Tree.Controllers { /// <summary> /// This controller represents the actions which can be taken with a tree object. /// </summary> [Authorize] public class TreeController : Controller { private readonly Authenticator _authenticator; private readonly ResourceModel _resourceModel; private readonly RdfStoreConnector _rdfStoreConnector; private readonly ProjectRoleModel _projectRoleModel; private readonly ProjectResourceModel _projectResourceModel; private readonly CoscineLogger _coscineLogger; private readonly string _blobApiLink; private readonly string _prefix; private readonly IConfiguration _configuration; private readonly string _shaclPropertyUrl = "http://www.w3.org/ns/shacl#property"; /// <summary> /// Tree controller constructor /// </summary> /// <param name="logger">Logger</param> public TreeController(ILogger<TreeController> logger) { _configuration = Program.Configuration; _authenticator = new Authenticator(this, _configuration); _resourceModel = new ResourceModel(); _rdfStoreConnector = new RdfStoreConnector(_configuration.GetStringAndWait("coscine/local/virtuoso/additional/url")); _projectRoleModel = new ProjectRoleModel(); _projectResourceModel = new ProjectResourceModel(); _coscineLogger = new CoscineLogger(logger); var rule = _configuration.GetStringAndWait("traefik/frontends/Coscine.Api.Blob/routes/Coscine.Api.Blob/rule"); var host = rule["Host:".Length..rule.IndexOf(";")]; var path = rule[rule.IndexOf("/")..]; _blobApiLink = $"https://{host}{path}/blob/"; _prefix = _configuration.GetStringAndWait("coscine/global/epic/prefix"); } /// <summary> /// Generates Id /// </summary> /// <param name="resourceId">Id of the resource</param> /// <param name="path"> Path to file</param> /// <returns> Uri </returns> public Uri GenerateId(string resourceId, string path) { return new Uri($"https://hdl.handle.net/{_prefix}/{resourceId}@path={Uri.EscapeDataString(path)}"); } /// <summary> /// This method retrieves the metadata /// </summary> /// <param name="resourceId"> Id of a resource</param> /// <param name="path">Path to the file</param> /// <returns> JSON Object with the metadata if ok, otherwise Statuscode 400 or 401 or 404</returns> [HttpGet("[controller]/{resourceId}/{*path}")] public async Task<IActionResult> GetMetadata(string resourceId, string path = "") { var rawPath = path; path = $"/{path}"; if (path.Contains("%2F") || path.Contains("%2f")) { return BadRequest("Path can not contain the sequence %2F."); } var user = _authenticator.GetUser(); var check = CheckResourceIdAndPath(resourceId, path, out Resource resource); if (check != null) { return check; } if (resource.ApplicationProfile[^1] != '/') { resource.ApplicationProfile += '/'; } if (user == null || !_resourceModel.HasAccess(user, resource, UserRoles.Owner, UserRoles.Member)) { return BadRequest("User has no Access to this resource."); } try { var resourceTypeOptions = _resourceModel.GetResourceTypeOptions(resource.Id); var resourceTypeDefinition = ResourceTypeFactory.CreateResourceTypeObject(resource.Type.DisplayName, _configuration); if (resourceTypeDefinition == null) { return BadRequest($"No provider for: \"{resource.Type.DisplayName}\"."); } var fileInfos = await resourceTypeDefinition.ListEntries(resourceId, path, resourceTypeOptions); var metadataInfos = new List<ResourceEntry>(fileInfos); if (path.EndsWith("/")) { metadataInfos.Insert(0, new ResourceEntry(path, false, 0, null, null, new DateTime(), new DateTime())); } var graphs = new List<JToken>(); int metadataCount = 0; foreach (var info in metadataInfos) { var id = GenerateId(resourceId, info.Key); if (_rdfStoreConnector.HasGraph(id.AbsoluteUri)) { var graph = _rdfStoreConnector.GetGraph(id); metadataCount = graph.Triples.Count; graphs.Add(JToken.Parse(VDS.RDF.Writing.StringWriter.Write(graph, new RdfJsonWriter()))); } } var jObject = new JObject( new JProperty("data", new JObject( new JProperty("metadataStorage", JToken.FromObject(graphs)), new JProperty("fileStorage", JToken.FromObject(fileInfos.Select(x => { var objectMetaInfo = new ObjectMetaInfo { Name = GetFolderOrFileName(x), Path = x.Key, Size = x.BodyBytes, Kind = x.HasBody ? "file" : "folder", Provider = resource.Type.DisplayName }; var objectMetaInfoReturnObject = new ObjectMetaInfoReturnObject(objectMetaInfo, _blobApiLink, resource.Id.ToString()); var result = new JObject { ["Name"] = objectMetaInfoReturnObject.Name, ["Path"] = objectMetaInfoReturnObject.Path, ["Size"] = objectMetaInfoReturnObject.Size, ["Kind"] = objectMetaInfoReturnObject.Kind, ["Modified"] = objectMetaInfoReturnObject.Modified, ["Created"] = objectMetaInfoReturnObject.Created, ["Provider"] = objectMetaInfoReturnObject.Provider, ["IsFolder"] = !x.HasBody, ["IsFile"] = x.HasBody, ["Action"] = new JObject { ["Delete"] = new JObject { ["Method"] = "DELETE", ["Url"] = objectMetaInfoReturnObject.DeleteLink }, ["Download"] = new JObject { ["Method"] = "GET", ["Url"] = resourceTypeDefinition?.GetEntryDownloadUrl(rawPath, null, resourceTypeOptions).Result?.ToString() }, ["Upload"] = new JObject { ["Method"] = "PUT", ["Url"] = resourceTypeDefinition?.GetEntryStoreUrl(rawPath, null, resourceTypeOptions).Result?.ToString() } } }; return result; }))) )) ); if (CoscineLoggerConfiguration.IsLogLevelActivated(LogType.Analytics) && path != "/") { LogAnalyticsViewMd(_projectResourceModel.GetProjectForResource(resource.Id).Value, resource.Id, path, user, GetMetadataCompleteness(metadataCount, resource)); } return Json(jObject); } catch (Exception e) { return BadRequest($"Error in communication with the resource"); } } /// <summary> /// This method retrieves the folder or file name. /// </summary> /// <param name="x">Resource Entry</param> /// <returns>Name</returns> private static string GetFolderOrFileName(ResourceEntry x) { var lastSlash = x.Key.LastIndexOf("/") + 1; var name = x.Key[lastSlash..]; if (name == "") { var tempPath = x.Key[..(lastSlash - 1)]; if (tempPath.Contains("/")) { tempPath = tempPath[(tempPath.IndexOf("/") + 1)..]; } name = tempPath; } return name; } /// <summary> /// This method stores the metadata of the file /// </summary> /// <param name="resourceId">Id of the resource</param> /// <param name="path">Path to the file</param> /// <returns>If ok Statuscode 204, otherwise Statuscode 400 or 401</returns> [HttpPut("[controller]/{resourceId}/{*path}")] public IActionResult StoreMetadataForFile(string resourceId, string path) { path = $"/{path}"; if (path.Contains("%2F") || path.Contains("%2f")) { return BadRequest("Path can not contain the sequence %2F."); } var innerBlock = ObjectFactory<JToken>.DeserializeFromStream(Request.Body); var graphNameUri = GenerateId(resourceId, path); var json = new JObject { [graphNameUri.AbsoluteUri] = innerBlock }; var user = _authenticator.GetUser(); var resource = _resourceModel.GetById(Guid.Parse(resourceId)); if (user == null || !_resourceModel.HasAccess(user, resource, UserRoles.Owner, UserRoles.Member)) { return BadRequest("User is no project member!"); } if (resource.Archived == "1") { return BadRequest("The resource is readonly!"); } if (resource.ApplicationProfile[^1] != '/') { resource.ApplicationProfile += '/'; } json[graphNameUri.AbsoluteUri]["http://www.w3.org/1999/02/22-rdf-syntax-ns#type"] = new JArray { new JObject { ["value"] = resource.ApplicationProfile, ["type"] = "uri" } }; // throw bad request if empty node value is detected JToken root = json.First.First; foreach (var node in root) { string nodeValue = node.First.First["value"].ToString().ToLower(); if (string.IsNullOrEmpty(nodeValue)) { return BadRequest("Empty values in application profile are not accepted."); } } var graph = new Graph(); graph.LoadFromString(json.ToString(), new RdfJsonParser()); var fixedValuesGraph = new Graph(); fixedValuesGraph.LoadFromString(resource.FixedValues, new RdfJsonParser()); var shapesGraph = (Graph)_rdfStoreConnector.GetGraph(resource.ApplicationProfile); foreach (var triple in fixedValuesGraph.Triples.Where(x => x.Predicate.ToString() == "https://purl.org/coscine/fixedValue")) { var shapeTriples = shapesGraph.Triples.Where((shapeTriple) => shapeTriple.Subject.ToString() == triple.Subject.ToString() && shapeTriple.Predicate.ToString() == "http://www.w3.org/ns/shacl#path"); var entry = shapeTriples.First(); // Remove any existing triples foreach (var triple2 in graph.GetTriplesWithSubjectPredicate(graph.CreateUriNode(graphNameUri), entry.Object).ToList()) { graph.Retract(triple2); } var tripleObject = triple.Object; var tripleObjectString = tripleObject.ToString(); if (tripleObjectString == "{ME}") { tripleObjectString = tripleObjectString.Replace("{ME}", user.DisplayName); tripleObject = graph.CreateLiteralNode(tripleObjectString, new Uri(XmlSpecsHelper.XmlSchemaDataTypeString)); } else if (tripleObjectString == "{TODAY}") { tripleObjectString = tripleObjectString.Replace("{TODAY}", DateTime.Today.ToString("yyyy-MM-dd")); tripleObject = graph.CreateLiteralNode(tripleObjectString, new Uri(XmlSpecsHelper.XmlSchemaDataTypeDate)); } graph.Assert(graph.CreateUriNode(graphNameUri), entry.Object, tripleObject); } // Default values is not checked or added // validate the data if (!_rdfStoreConnector.ValidateShacl(graph, graphNameUri)) { return BadRequest("Data has the wrong format!"); } // store the data if (_rdfStoreConnector.HasGraph(graphNameUri)) { _rdfStoreConnector.ClearGraph(graphNameUri); if (CoscineLoggerConfiguration.IsLogLevelActivated(LogType.Analytics)) { LogAnalyticsUpdateMd(_projectResourceModel.GetProjectForResource(resource.Id).Value, resource.Id, path, user, GetMetadataCompleteness(graph.Triples.Count, resource)); } } else { _rdfStoreConnector.CreateNamedGraph(graphNameUri); if (CoscineLoggerConfiguration.IsLogLevelActivated(LogType.Analytics)) { LogAnalyticsUploadMd(_projectResourceModel.GetProjectForResource(resource.Id).Value, resource.Id, path, user, GetMetadataCompleteness(graph.Triples.Count, resource)); } } // BaseUri must be set for the sparql query graph.BaseUri = graphNameUri; _rdfStoreConnector.AddGraph(graph); return NoContent(); } /// <summary> /// Checks the resource Id and the path /// </summary> /// <param name="resourceId">Id of the resource</param> /// <param name="path">Path to the file</param> /// <param name="resource">Resource</param> /// <returns>null, otherwise Statuscode 400 or 404 </returns> private IActionResult CheckResourceIdAndPath(string resourceId, string path, out Resource resource) { resource = null; if (string.IsNullOrWhiteSpace(path)) { return BadRequest($"Your path \"{path}\" is empty."); } Regex rgx = new Regex(@"[\:?*<>|]+"); if (rgx.IsMatch(path)) { return BadRequest($"Your path \"{path}\" contains bad characters. The following characters are not permissible: {@"\/:?*<>|"}."); } if (!Guid.TryParse(resourceId, out Guid resourceGuid)) { return BadRequest($"{resourceId} is not a guid."); } try { resource = _resourceModel.GetById(resourceGuid); if (resource == null) { return NotFound($"Could not find resource with id: {resourceId}"); } } catch (Exception) { return NotFound($"Could not find resource with id: {resourceId}"); } if (resource.Type == null) { ResourceTypeModel resourceTypeModel = new ResourceTypeModel(); resource.Type = resourceTypeModel.GetById(resource.TypeId); } // All good return null; } private void LogAnalyticsViewMd(Guid projectId, Guid resourceId, string filename, User user, string metadataCompletness) { var analyticsLogObject = new AnalyticsLogObject { Type = "Action", Operation = "View MD", RoleId = _projectRoleModel.GetGetUserRoleForProject(projectId, user.Id).ToString(), ProjectId = projectId.ToString(), ResourceId = resourceId.ToString(), FileId = resourceId.ToString() + "/" + HttpUtility.UrlDecode(filename)[1..], ApplicationsProfile = _resourceModel.CreateReturnObjectFromDatabaseObject(_resourceModel.GetById(resourceId)).ApplicationProfile, MetadataCompleteness = metadataCompletness, }; _coscineLogger.AnalyticsLog(analyticsLogObject); } private void LogAnalyticsUploadMd(Guid projectId, Guid resourceId, string filename, User user, string metadataCompletness) { var analyticsLogObject = new AnalyticsLogObject { Type = "Action", Operation = "Upload MD", RoleId = _projectRoleModel.GetGetUserRoleForProject(projectId, user.Id).ToString(), ProjectId = projectId.ToString(), ResourceId = resourceId.ToString(), FileId = resourceId.ToString() + "/" + HttpUtility.UrlDecode(filename)[1..], ApplicationsProfile = _resourceModel.CreateReturnObjectFromDatabaseObject(_resourceModel.GetById(resourceId)).ApplicationProfile, MetadataCompleteness = metadataCompletness, }; _coscineLogger.AnalyticsLog(analyticsLogObject); } private void LogAnalyticsUpdateMd(Guid projectId, Guid resourceId, string filename, User user, string metadataCompletness) { var analyticsLogObject = new AnalyticsLogObject { Type = "Action", Operation = "Update MD", RoleId = _projectRoleModel.GetGetUserRoleForProject(projectId, user.Id).ToString(), ProjectId = projectId.ToString(), ResourceId = resourceId.ToString(), FileId = resourceId.ToString() + "/" + HttpUtility.UrlDecode(filename)[1..], ApplicationsProfile = _resourceModel.CreateReturnObjectFromDatabaseObject(_resourceModel.GetById(resourceId)).ApplicationProfile, MetadataCompleteness = metadataCompletness, }; _coscineLogger.AnalyticsLog(analyticsLogObject); } private string GetMetadataCompleteness(int metadataCount, Resource resource) { var shapesGraph = _rdfStoreConnector.GetGraph(resource.ApplicationProfile); var nodeFactory = new NodeFactory(); var uriNode = (UriNode)nodeFactory.CreateUriNode(new Uri(_shaclPropertyUrl)); var total = shapesGraph.GetTriplesWithPredicate(uriNode).Count(); var present = metadataCount; return $"{present}/{total}"; } } }