Merge branch 'Sprint/2020-11' into 'master'

Sprint/2020 11

See merge request coscine/cs/organizationdeployer!5

src/OrganizationDeployer/App.config

@@ -17,6 +17,10 @@
         <assemblyIdentity name="Coscine.Configuration" publicKeyToken="ce3d7a32d7dc1e5a" culture="neutral" />
         <bindingRedirect oldVersion="0.0.0.0-1.5.0.0" newVersion="1.5.0.0" />
       </dependentAssembly>
+      <dependentAssembly>
+        <assemblyIdentity name="Coscine.Database" publicKeyToken="767d77427707b70a" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-1.22.0.0" newVersion="1.22.0.0" />
+      </dependentAssembly>
     </assemblyBinding>
   </runtime>
   <entityFramework>

src/OrganizationDeployer/OrganizationDeployer.csproj

@@ -43,11 +43,11 @@
     <Reference Include="Coscine.Configuration, Version=1.5.0.0, Culture=neutral, PublicKeyToken=ce3d7a32d7dc1e5a, processorArchitecture=MSIL">
       <HintPath>..\packages\Coscine.Configuration.1.5.0\lib\net461\Coscine.Configuration.dll</HintPath>
     </Reference>
-    <Reference Include="Coscine.Database, Version=1.19.0.0, Culture=neutral, PublicKeyToken=767d77427707b70a, processorArchitecture=MSIL">
-      <HintPath>..\packages\Coscine.Database.1.19.0\lib\net461\Coscine.Database.dll</HintPath>
+    <Reference Include="Coscine.Database, Version=1.22.0.0, Culture=neutral, PublicKeyToken=767d77427707b70a, processorArchitecture=MSIL">
+      <HintPath>..\packages\Coscine.Database.1.22.0\lib\net461\Coscine.Database.dll</HintPath>
     </Reference>
-    <Reference Include="Coscine.Metadata, Version=1.1.0.0, Culture=neutral, processorArchitecture=MSIL">
-      <HintPath>..\packages\Coscine.Metadata.1.1.0\lib\net461\Coscine.Metadata.dll</HintPath>
+    <Reference Include="Coscine.Metadata, Version=1.2.0.0, Culture=neutral, processorArchitecture=MSIL">
+      <HintPath>..\packages\Coscine.Metadata.1.2.0\lib\net461\Coscine.Metadata.dll</HintPath>
     </Reference>
     <Reference Include="dotNetRDF, Version=2.5.0.0, Culture=neutral, PublicKeyToken=6055ffe4c97cc780, processorArchitecture=MSIL">
       <HintPath>..\packages\dotNetRDF.2.5.0\lib\net40\dotNetRDF.dll</HintPath>

src/OrganizationDeployer/Program.cs

@@ -14,26 +14,10 @@ namespace Coscine.OrganizationDeployer
     {
         public static void Main(string[] args)
         {
-            ProcessStartInfo startInfo = new ProcessStartInfo
-            {
-                FileName = @"powershell.exe",
-                Arguments = $@"& '{ Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "getOrganizations.ps1") }'",
-                RedirectStandardOutput = true,
-                RedirectStandardError = true,
-                UseShellExecute = false,
-                CreateNoWindow = true
-            };
-            Process process = new Process
-            {
-                StartInfo = startInfo
-            };
-            process.Start();
-
-            string output = process.StandardOutput.ReadToEnd();
-            Console.WriteLine(output);
-
-            string errors = process.StandardError.ReadToEnd();
-            Console.WriteLine(errors);
+            ExecuteCommand(
+                "powershell.exe",
+                $@"& '{ Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "getOrganizations.ps1") }'"
+            );
 
             var configuration = new ConsulConfiguration();
 
@@ -45,28 +29,68 @@ namespace Coscine.OrganizationDeployer
             VirtuosoManager virtuosoManager = new VirtuosoManager($"Server={virtuosoHost};Uid={virtuosoUser};pwd={virtuosoPassword}");
 
             var _util = new Util(virtuosoServer);
-            foreach (var file in GetFiles("organizations").Where((file) => file.Contains(".ttl")))
+            var folder = "/voc";
+            var virtuosoISQLLocation = configuration.GetString(
+                "coscine/local/virtuoso/isql",
+                "C:/Programs/Virtuoso/bin/isql.exe"
+            );
+            var queries = new List<string>();
+            foreach (var file in GetFiles(folder).Where((file) => file.Contains(".ttl")))
             {
+                var fileInfo = new FileInfo(file);
                 var graph = new Graph();
                 graph.LoadFromFile(file);
                 var graphName = graph.BaseUri.ToString();
+
                 if (_util.HasGraph(graphName))
                 {
                     Console.WriteLine($"Clearing {graphName}");
                     _util.ClearGraph(graphName);
                 }
-                else
-                {
-                    Console.WriteLine($"Creating {graphName}");
-                    _util.CreateNamedGraph(graphName);
-                }
-                
-                Console.WriteLine($"Adding to {graphName}");
-                virtuosoManager.SaveGraph(graph);
+
+                queries.Add($"ld_dir('{fileInfo.DirectoryName.Substring(2).Replace("\\", "/")}', '{fileInfo.Name}', '{graphName}');");
+            }
+            queries.Add($"rdf_loader_run ();");
+            queries.Add($"DELETE from DB.DBA.load_list where 1=1;");
+
+            foreach (var query in queries)
+            {
+                ExecuteCommand(
+                    "powershell.exe",
+                    $"\"\\\"{query}\\\" | {virtuosoISQLLocation}\""
+                );
             }
+
             Console.WriteLine("Done");
         }
 
+        private static void ExecuteCommand(string fileName, string arguments)
+        {
+            ProcessStartInfo startInfo = new ProcessStartInfo
+            {
+                FileName = fileName,
+                Arguments = arguments,
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                UseShellExecute = false,
+                CreateNoWindow = true,
+            };
+            using (var process = new Process
+            {
+                StartInfo = startInfo
+            })
+            {
+                process.Start();
+
+                string output = process.StandardOutput.ReadToEnd();
+                Console.WriteLine(output);
+
+                string errors = process.StandardError.ReadToEnd();
+                Console.WriteLine(errors);
+            }
+        }
+
+
         private static IEnumerable<string> GetFiles(string path)
         {
             Queue<string> queue = new Queue<string>();

src/OrganizationDeployer/getOrganizations.ps1

@@ -6,10 +6,32 @@ $gitlab_token = & $fullPath kv get "coscine/global/gitlabtoken"
 
 Push-Location $PSScriptRoot
 
+$organizationsPath = 'organizations'
+# Twice remove, because of: https://stackoverflow.com/questions/7909167/how-to-quietly-remove-a-directory-with-content-in-powershell#comment10316056_7909195
+If(Test-Path $organizationsPath) { Remove-Item -LiteralPath $organizationsPath -Force -Recurse }
+If(Test-Path $organizationsPath) { Remove-Item -LiteralPath $organizationsPath -Force -Recurse }
+
 git clone https://gitlab-ci-token:$gitlab_token@git.rwth-aachen.de/coscine/organizations.git
 
 cd organizations
 
-git checkout Master
+$pagebranch = & $fullPath kv get 'coscine/local/organizationdeployer/branch'
+
+if ($pagebranch -And $pagebranch -ne 'master' ) {
+	git checkout $pagebranch
+	git pull
+}
+
+cd ..
+
+$path = '/voc'
+
+# Twice remove, because of: https://stackoverflow.com/questions/7909167/how-to-quietly-remove-a-directory-with-content-in-powershell#comment10316056_7909195
+If(Test-Path $path) { Remove-Item -LiteralPath $path -Force -Recurse }
+If(Test-Path $path) { Remove-Item -LiteralPath $path -Force -Recurse }
+
+New-Item -ItemType Directory -Force -Path $path
+
+Copy-Item './organizations/*' $path -Recurse;
 
 Pop-Location
\ No newline at end of file

src/OrganizationDeployer/packages.config

@@ -2,8 +2,8 @@
 <packages>
   <package id="Consul" version="0.7.2.6" targetFramework="net461" />
   <package id="Coscine.Configuration" version="1.5.0" targetFramework="net461" />
-  <package id="Coscine.Database" version="1.19.0" targetFramework="net461" />
-  <package id="Coscine.Metadata" version="1.1.0" targetFramework="net461" />
+  <package id="Coscine.Database" version="1.22.0" targetFramework="net461" />
+  <package id="Coscine.Metadata" version="1.2.0" targetFramework="net461" />
   <package id="dotNetRDF" version="2.5.0" targetFramework="net461" />
   <package id="dotNetRDF.Data.Virtuoso" version="2.5.0" targetFramework="net461" />
   <package id="EntityFramework" version="6.2.0" targetFramework="net461" />