using Coscine.Api.Tree.Util; using Coscine.ApiCommons; using Coscine.ApiCommons.Factories; using Coscine.Configuration; using Coscine.Database.DataModel; using Coscine.Database.Models; using Coscine.Database.Util; using Coscine.Logging; using Coscine.Metadata; using Coscine.ResourceTypes; using Coscine.ResourceTypes.Base; using Coscine.ResourceTypes.Base.Models; using Coscine.WaterbutlerHelper; using Coscine.WaterbutlerHelper.ReturnObjects; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; using Newtonsoft.Json.Linq; using System; using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; using System.Threading.Tasks; using System.Web; using VDS.RDF; using VDS.RDF.Parsing; namespace Coscine.Api.Tree.Controllers { /// <summary> /// This controller represents the actions which can be taken with a tree object. /// </summary> [Authorize] public class TreeController : Controller { private readonly Authenticator _authenticator; private readonly ResourceModel _resourceModel; private readonly RdfStoreConnector _rdfStoreConnector; private readonly ProjectRoleModel _projectRoleModel; private readonly ProjectResourceModel _projectResourceModel; private readonly CoscineLogger _coscineLogger; private readonly string _blobApiLink; private readonly string _prefix; private readonly IConfiguration _configuration; private readonly string _shaclPropertyUrl = "http://www.w3.org/ns/shacl#property"; /// <summary> /// Tree controller constructor /// </summary> /// <param name="logger">Logger</param> public TreeController(ILogger<TreeController> logger) { _configuration = Program.Configuration; _authenticator = new Authenticator(this, _configuration); _resourceModel = new ResourceModel(); _rdfStoreConnector = new RdfStoreConnector(_configuration.GetStringAndWait("coscine/local/virtuoso/additional/url")); _projectRoleModel = new ProjectRoleModel(); _projectResourceModel = new ProjectResourceModel(); _coscineLogger = new CoscineLogger(logger); var rule = _configuration.GetStringAndWait("traefik/frontends/Coscine.Api.Blob/routes/Coscine.Api.Blob/rule"); var host = rule["Host:".Length..rule.IndexOf(";")]; var path = rule[rule.IndexOf("/")..]; _blobApiLink = $"https://{host}{path}/blob/"; _prefix = _configuration.GetStringAndWait("coscine/global/epic/prefix"); } /// <summary> /// Generates Id /// </summary> /// <param name="resourceId">Id of the resource</param> /// <param name="path"> Path to file</param> /// <returns> Uri </returns> public Uri GenerateId(string resourceId, string path) { if (!path.StartsWith("/")) { path = "/" + path; } return new CustomUri($"https://hdl.handle.net/{_prefix}/{resourceId}@path={Uri.EscapeDataString(path)}"); } /// <summary> /// This method retrieves the metadata /// </summary> /// <param name="resourceId"> Id of a resource</param> /// <param name="path">Path to the file</param> /// <returns> JSON Object with the metadata if OK, otherwise status code 400 or 401 or 404</returns> [HttpGet("[controller]/{resourceId}/{*path}")] [ApiExplorerSettings(IgnoreApi = true)] public async Task<IActionResult> GetMetadataWithPath(string resourceId, string path = "") { return await GetMetadata(resourceId, path); } /// <summary> /// This method retrieves the metadata /// </summary> /// <param name="resourceId"> Id of a resource</param> /// <param name="path">Path to the file</param> /// <param name="mimeType">Requested MimeType of the metadata</param> /// <returns> JSON Object with the metadata if OK, otherwise status code 400 or 401 or 404</returns> [HttpGet("[controller]/{resourceId}/")] public async Task<IActionResult> GetMetadataWithParameter(string resourceId, [FromQuery] string path = "", [FromQuery] string mimeType = "application/rdf+json") { // Strip the first slash, to reuse the previous implementation. if (path.StartsWith("/")) { path = path[1..]; } return await GetMetadata(resourceId, path, mimeType); } /// <summary> /// This method retrieves the metadata /// </summary> /// <param name="resourceId"> Id of a resource</param> /// <param name="path">Path to the file</param> /// <param name="mimeType">Requested MimeType of the metadata</param> /// <returns> JSON Object with the metadata if OK, otherwise status code 400 or 401 or 404</returns> public async Task<IActionResult> GetMetadata(string resourceId, string path = "", string mimeType = "application/rdf+json") { if (path.Contains("%2F") || path.Contains("%2f")) { return BadRequest("Path can not contain the sequence %2F."); } var user = _authenticator.GetUser(); var check = CheckResourceIdAndPath(resourceId, $"/{path}", out Resource resource); if (check != null) { return check; } if (resource.ApplicationProfile[^1] != '/') { resource.ApplicationProfile += '/'; } if (user == null || !_resourceModel.HasAccess(user, resource, UserRoles.Owner, UserRoles.Member)) { return BadRequest("User has no Access to this resource."); } try { var resourceTypeDefinition = ResourceTypeFactory.Instance.GetResourceType(resource); if (resourceTypeDefinition == null) { return BadRequest($"No provider for: \"{resource.Type.DisplayName}\"."); } var fileInfos = await resourceTypeDefinition.ListEntries(resourceId, path); var resourceTypeInformation = resourceTypeDefinition.GetResourceTypeInformation().Result; var metadataInfos = new List<ResourceEntry>(fileInfos); // Add to metadata infos, if no "physical" file for it exists if (!metadataInfos.Any((metadataInfo) => metadataInfo.Key == path)) { metadataInfos.Add(new ResourceEntry(path, true, 0, GenerateId(resourceId, path).AbsoluteUri, null, DateTime.Now, DateTime.Now)); } var applicationProfileGraph = _rdfStoreConnector.GetGraph(resource.ApplicationProfile); var propertyTriples = applicationProfileGraph.GetTriplesWithPredicate(applicationProfileGraph.CreateUriNode(new Uri("http://www.w3.org/ns/shacl#property"))); var properties = propertyTriples.Select((propertyTriple) => applicationProfileGraph.GetTriplesWithSubject(propertyTriple.Object)); var graphs = new List<object>(); int metadataCount = 0; foreach (var info in metadataInfos) { var id = GenerateId(resourceId, info.Key); if (_rdfStoreConnector.HasGraph(id.AbsoluteUri)) { var graph = _rdfStoreConnector.GetGraph(id); // Rewrite the datatype since Virtuoso might convert e.g. decimals to integers foreach (var property in properties) { var shPath = property.FirstOrDefault((triple) => triple.Predicate.ToString() == "http://www.w3.org/ns/shacl#path"); if (shPath != null) { var values = graph.GetTriplesWithPredicate(shPath.Object); var shDatatype = property.FirstOrDefault((triple) => triple.Predicate.ToString() == "http://www.w3.org/ns/shacl#datatype"); if (shDatatype != null) { foreach (var value in values) { graph.Retract(value); graph.Assert( value.Subject, value.Predicate, graph.CreateLiteralNode((value.Object as LiteralNode)?.Value, new Uri(shDatatype.Object.ToString())) ); } } } } metadataCount = graph.Triples.Count; var writer = MimeTypesHelper.GetWriter(new List<string>() { mimeType }); var parsedRdf = VDS.RDF.Writing.StringWriter.Write(graph, writer); // Legacy Support if (mimeType == "application/rdf+json") { graphs.Add(JToken.Parse(parsedRdf)); } else { graphs.Add(new JObject { [id.AbsoluteUri] = parsedRdf }); } } } var jObject = new JObject( new JProperty("data", new JObject( new JProperty("metadataStorage", JToken.FromObject(graphs)), new JProperty("fileStorage", JToken.FromObject(fileInfos.Select(x => { var objectMetaInfo = new ObjectMetaInfo { Name = x.Key.Split("/", StringSplitOptions.RemoveEmptyEntries).Last(), Path = x.Key, Size = x.BodyBytes, Kind = x.HasBody ? "file" : "folder", Provider = resource.Type.DisplayName }; var objectMetaInfoReturnObject = new ObjectMetaInfoReturnObject(objectMetaInfo, _blobApiLink, resource.Id.ToString()); var obj = new JObject { ["Name"] = objectMetaInfoReturnObject.Name, ["Path"] = objectMetaInfoReturnObject.Path, ["Size"] = objectMetaInfoReturnObject.Size, ["Kind"] = objectMetaInfoReturnObject.Kind, ["Modified"] = x.HasBody ? x.Modified : null, ["Created"] = x.HasBody ? x.Created : null, ["Provider"] = objectMetaInfoReturnObject.Provider, ["IsFolder"] = !x.HasBody, ["IsFile"] = x.HasBody, ["Action"] = new JObject() }; if (resourceTypeInformation.CanCreateLinks) { obj["Action"] = new JObject { ["Delete"] = new JObject { ["Method"] = "DELETE", ["Url"] = resourceTypeDefinition?.GetPresignedUrl(resource.Id.ToString(), objectMetaInfoReturnObject.Path, CoscineHttpVerb.DELETE).Result?.ToString() }, ["Download"] = new JObject { ["Method"] = "GET", ["Url"] = resourceTypeDefinition?.GetPresignedUrl(resource.Id.ToString(), objectMetaInfoReturnObject.Path, CoscineHttpVerb.GET).Result?.ToString() }, ["Upload"] = new JObject { ["Method"] = "PUT", ["Url"] = resourceTypeDefinition?.GetPresignedUrl(resource.Id.ToString(), objectMetaInfoReturnObject.Path, CoscineHttpVerb.DELETE).Result?.ToString() } }; } return obj; }))) )) ); if (CoscineLoggerConfiguration.IsLogLevelActivated(LogType.Analytics) && path != "/") { LogAnalyticsViewMd(_projectResourceModel.GetProjectForResource(resource.Id).Value, resource.Id, $"/{path}", user, GetMetadataCompleteness(metadataCount, resource)); } return Json(jObject); } catch (Exception) { return BadRequest("Error in communication with the resource"); } } /// <summary> /// This method stores the metadata of the file /// </summary> /// <param name="resourceId">Id of the resource</param> /// <param name="path">Path to the file</param> /// <returns>If OK status code 204, otherwise status code 400 or 401</returns> [HttpPut("[controller]/{resourceId}/{*path}")] [ApiExplorerSettings(IgnoreApi = true)] public IActionResult StoreMetadataForFileWithPath(string resourceId, string path) { return StoreMetadataForFile(resourceId, path); } /// <summary> /// This method stores the metadata of the file /// </summary> /// <param name="resourceId">Id of the resource</param> /// <param name="path">Path to the file</param> /// <param name="mimeType">Requested MimeType of the metadata</param> /// <returns>If OK status code 204, otherwise status code 400 or 401</returns> [HttpPut("[controller]/{resourceId}/")] public IActionResult StoreMetadataForFileWithParameter(string resourceId, [FromQuery] string path = "", [FromQuery] string mimeType = "application/rdf+json") { // Strip the first slash, to reuse the previous implementation. if (path.StartsWith("/")) { path = path[1..]; } return StoreMetadataForFile(resourceId, path, mimeType); } /// <summary> /// This method stores the metadata of the file /// </summary> /// <param name="resourceId">Id of the resource</param> /// <param name="path">Path to the file</param> /// <param name="mimeType">Requested MimeType of the metadata</param> /// <returns>If OK status code 204, otherwise status code 400 or 401</returns> public IActionResult StoreMetadataForFile(string resourceId, string path, string mimeType = "application/rdf+json") { path = $"/{path}"; if (path.Contains("%2F") || path.Contains("%2f")) { return BadRequest("Path can not contain the sequence %2F."); } // Ducktape solution for supporting multiple mimetypes var metadataObject = ObjectFactory<JToken>.DeserializeFromStream(Request.Body); var graphNameUri = GenerateId(resourceId, path); JObject json; // Legacy Support if (mimeType == "application/rdf+json") { json = new JObject { [graphNameUri.AbsoluteUri] = metadataObject }; } else { var tempGraph = new Graph(); StringParser.Parse(tempGraph, metadataObject.Value<string>("metadata"), MimeTypesHelper.GetParser(mimeType)); var triplesList = tempGraph.Triples.ToArray(); var subjectNode = tempGraph.CreateUriNode(graphNameUri); foreach (var triple in triplesList) { tempGraph.Retract(triple); tempGraph.Assert(new Triple(subjectNode, triple.Predicate, triple.Object)); } json = JObject.Parse(VDS.RDF.Writing.StringWriter.Write(tempGraph, MimeTypesHelper.GetWriter("application/rdf+json"))); } var user = _authenticator.GetUser(); var resource = _resourceModel.GetById(Guid.Parse(resourceId)); if (user == null || !_resourceModel.HasAccess(user, resource, UserRoles.Owner, UserRoles.Member)) { return BadRequest("User is no project member!"); } if (resource.Archived == "1") { return BadRequest("The resource is read only!"); } if (resource.ApplicationProfile[^1] != '/') { resource.ApplicationProfile += '/'; } json[graphNameUri.AbsoluteUri]["http://www.w3.org/1999/02/22-rdf-syntax-ns#type"] = new JArray { new JObject { ["value"] = resource.ApplicationProfile, ["type"] = "uri" } }; // throw bad request if empty node value is detected foreach (var node in json.First.First) { string nodeValue = node.First.First["value"].ToString().ToLower(); if (string.IsNullOrEmpty(nodeValue)) { return BadRequest("Empty values in application profile are not accepted."); } } var graph = new Graph(); graph.LoadFromString(json.ToString(), new RdfJsonParser()); var fixedValuesGraph = new Graph(); fixedValuesGraph.LoadFromString(resource.FixedValues, new RdfJsonParser()); var shapesGraph = (Graph)_rdfStoreConnector.GetGraph(resource.ApplicationProfile); foreach (var triple in fixedValuesGraph.Triples.Where(x => x.Predicate.ToString() == "https://purl.org/coscine/fixedValue")) { var shapeTriples = shapesGraph.Triples.Where((shapeTriple) => shapeTriple.Object.ToString() == triple.Subject.ToString() && shapeTriple.Predicate.ToString() == "http://www.w3.org/ns/shacl#path"); var entry = shapeTriples.First(); // Remove any existing triples foreach (var triple2 in graph.GetTriplesWithSubjectPredicate(graph.CreateUriNode(graphNameUri), entry.Object).ToList()) { graph.Retract(triple2); } var tripleObject = triple.Object; if (tripleObject is ILiteralNode) { var tripleObjectString = (tripleObject as ILiteralNode)?.Value; if (tripleObjectString.Equals("{ME}")) { tripleObjectString = tripleObjectString.Replace("{ME}", user.DisplayName); tripleObject = graph.CreateLiteralNode(tripleObjectString, new Uri(XmlSpecsHelper.XmlSchemaDataTypeString)); } else if (tripleObjectString.Equals("{TODAY}")) { tripleObjectString = tripleObjectString.Replace("{TODAY}", DateTime.Today.ToString("yyyy-MM-dd")); tripleObject = graph.CreateLiteralNode(tripleObjectString, new Uri(XmlSpecsHelper.XmlSchemaDataTypeDate)); } } graph.Assert(graph.CreateUriNode(graphNameUri), entry.Object, tripleObject); } // Default values is not checked or added // validate the data if (!_rdfStoreConnector.ValidateShacl(graph, graphNameUri)) { return BadRequest("Data has the wrong format!"); } // store the data if (_rdfStoreConnector.HasGraph(graphNameUri)) { _rdfStoreConnector.ClearGraph(graphNameUri); if (CoscineLoggerConfiguration.IsLogLevelActivated(LogType.Analytics)) { LogAnalyticsUpdateMd(_projectResourceModel.GetProjectForResource(resource.Id).Value, resource.Id, path, user, GetMetadataCompleteness(graph.Triples.Count, resource)); } } else { try { _rdfStoreConnector.CreateNamedGraph(graphNameUri); } #pragma warning disable RCS1075 // Avoid empty catch clause that catches System.Exception. catch (Exception) #pragma warning restore RCS1075 // Avoid empty catch clause that catches System.Exception. { // Graph creation failed because it has been created before, skip this for now } if (CoscineLoggerConfiguration.IsLogLevelActivated(LogType.Analytics)) { LogAnalyticsUploadMd(_projectResourceModel.GetProjectForResource(resource.Id).Value, resource.Id, path, user, GetMetadataCompleteness(graph.Triples.Count, resource)); } } // BaseUri must be set for the sparql query graph.BaseUri = graphNameUri; _rdfStoreConnector.AddGraph(graph); return NoContent(); } /// <summary> /// Checks the resource Id and the path /// </summary> /// <param name="resourceId">Id of the resource</param> /// <param name="path">Path to the file</param> /// <param name="resource">Resource</param> /// <returns>null, otherwise status code 400 or 404 </returns> private IActionResult CheckResourceIdAndPath(string resourceId, string path, out Resource resource) { resource = null; if (string.IsNullOrWhiteSpace(path)) { return BadRequest($"Your path \"{path}\" is empty."); } var rgx = new Regex(@"[\:?*<>|]+"); if (rgx.IsMatch(path)) { return BadRequest($"Your path \"{path}\" contains bad characters. The following characters are not permissible: {@"\/:?*<>|"}."); } if (!Guid.TryParse(resourceId, out Guid resourceGuid)) { return BadRequest($"{resourceId} is not a GUID."); } try { resource = _resourceModel.GetById(resourceGuid); if (resource == null) { return NotFound($"Could not find resource with id: {resourceId}"); } } catch (Exception) { return NotFound($"Could not find resource with id: {resourceId}"); } if (resource.Type == null) { var resourceTypeModel = new ResourceTypeModel(); resource.Type = resourceTypeModel.GetById(resource.TypeId); } // All good return null; } private void LogAnalyticsViewMd(Guid projectId, Guid resourceId, string filename, User user, string metadataCompletness) { var analyticsLogObject = new AnalyticsLogObject { Type = "Action", Operation = "View MD", RoleId = _projectRoleModel.GetGetUserRoleForProject(projectId, user.Id).ToString(), ProjectId = projectId.ToString(), ResourceId = resourceId.ToString(), FileId = resourceId.ToString() + "/" + HttpUtility.UrlDecode(filename)[1..], ApplicationsProfile = _resourceModel.CreateReturnObjectFromDatabaseObject(_resourceModel.GetById(resourceId)).ApplicationProfile, MetadataCompleteness = metadataCompletness, }; _coscineLogger.AnalyticsLog(analyticsLogObject); } private void LogAnalyticsUploadMd(Guid projectId, Guid resourceId, string filename, User user, string metadataCompletness) { var analyticsLogObject = new AnalyticsLogObject { Type = "Action", Operation = "Upload MD", RoleId = _projectRoleModel.GetGetUserRoleForProject(projectId, user.Id).ToString(), ProjectId = projectId.ToString(), ResourceId = resourceId.ToString(), FileId = resourceId.ToString() + "/" + HttpUtility.UrlDecode(filename)[1..], ApplicationsProfile = _resourceModel.CreateReturnObjectFromDatabaseObject(_resourceModel.GetById(resourceId)).ApplicationProfile, MetadataCompleteness = metadataCompletness, }; _coscineLogger.AnalyticsLog(analyticsLogObject); } private void LogAnalyticsUpdateMd(Guid projectId, Guid resourceId, string filename, User user, string metadataCompletness) { var analyticsLogObject = new AnalyticsLogObject { Type = "Action", Operation = "Update MD", RoleId = _projectRoleModel.GetGetUserRoleForProject(projectId, user.Id).ToString(), ProjectId = projectId.ToString(), ResourceId = resourceId.ToString(), FileId = resourceId.ToString() + "/" + HttpUtility.UrlDecode(filename)[1..], ApplicationsProfile = _resourceModel.CreateReturnObjectFromDatabaseObject(_resourceModel.GetById(resourceId)).ApplicationProfile, MetadataCompleteness = metadataCompletness, }; _coscineLogger.AnalyticsLog(analyticsLogObject); } private string GetMetadataCompleteness(int metadataCount, Resource resource) { var shapesGraph = _rdfStoreConnector.GetGraph(resource.ApplicationProfile); var nodeFactory = new NodeFactory(); var uriNode = (UriNode)nodeFactory.CreateUriNode(new Uri(_shaclPropertyUrl)); var total = shapesGraph.GetTriplesWithPredicate(uriNode).Count(); var present = metadataCount; return $"{present}/{total}"; } } }