Skip to content
Snippets Groups Projects
Commit 3133112b authored by Petar Hristov's avatar Petar Hristov :speech_balloon:
Browse files

WIP

parent e82bb316
No related branches found
No related tags found
1 merge request!7Breaking: Refactored the GraphDeployer
Show whitespace changes
Inline Side-by-side
Showing
with 247 additions and 128 deletions
src/GraphDeployer/GraphDeployer.csproj
+ 9
− 2
View file @ 3133112b
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
......@@ -23,6 +23,13 @@
<ItemGroup>
<PackageReference Include="Coscine.Metadata" Version="*-*" />
<PackageReference Include="dotNetRDF.Data.Virtuoso" Version="2.7.5" />
<PackageReference Include="GitLabApiClient" Version="1.8.1-beta.5" />
<PackageReference Include="LibGit2Sharp" Version="0.26.2" />
<PackageReference Include="NLog" Version="5.1.0" />
<PackageReference Include="NLog.Extensions.Logging" Version="5.2.0" />
</ItemGroup>
<ItemGroup>
<None Update="nlog.config" CopyToOutputDirectory="Always" />
</ItemGroup>
</Project>
src/GraphDeployer/GraphRepo.cs 0 → 100644
+ 8
− 0
View file @ 3133112b
namespace Coscine.GraphDeployer;
public class GraphRepo
{
public string Name { get; set; } = null!;
public string Branch { get; set; } = null!;
public string Url { get; set; } = null!;
}
src/GraphDeployer/Logging/AssemblyNameLayoutRenderer.cs 0 → 100644
+ 23
− 0
View file @ 3133112b
using NLog;
using NLog.LayoutRenderers;
using System.Text;
namespace Coscine.GraphDeployer.Logging
{
[LayoutRenderer("assembly-name")]
public class AssemblyNameLayoutRenderer : LayoutRenderer
{
protected override void Append(StringBuilder builder, LogEventInfo logEvent)
{
var assembly = typeof(Program).Assembly.GetName();
if (assembly is not null)
{
builder.Append(assembly.Name);
}
else
{
builder.Append(new Guid().ToString().Take(8));
}
}
}
}
src/GraphDeployer/Logging/AssemblyVersionLayoutRenderer.cs 0 → 100644
+ 23
− 0
View file @ 3133112b
using NLog;
using NLog.LayoutRenderers;
using System.Text;
namespace Coscine.GraphDeployer.Logging
{
[LayoutRenderer("assembly-version")]
public class AssemblyVersionLayoutRenderer : LayoutRenderer
{
protected override void Append(StringBuilder builder, LogEventInfo logEvent)
{
var assembly = typeof(Program).Assembly.GetName();
if (assembly is not null && assembly.Version is not null)
{
builder.Append(assembly.Version.ToString(3));
}
else
{
builder.Append(new Version().ToString(3));
}
}
}
}
src/GraphDeployer/Program.cs
+ 133
− 79
View file @ 3133112b
using Coscine.Configuration;
using Coscine.GraphDeployer.Logging;
using Coscine.Metadata;
using GitLabApiClient;
using GitLabApiClient.Models.Projects.Responses;
using LibGit2Sharp;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using NLog.Config;
using NLog.Extensions.Logging;
using System.Diagnostics;
using VDS.RDF;
using VDS.RDF.Storage;
......@@ -8,50 +16,85 @@ namespace Coscine.GraphDeployer;
public class Program
{
public static ConsulConfiguration Configuration { get; } = new ConsulConfiguration();
public static string WorkingFolder { get; } = Path.GetFullPath("C:/voc");
public static string GraphsConsulLocation { get; } = "coscine/local/graphs";
private static string Token { get; set; } = string.Empty;
private static ILogger _logger = null!;
public static void Main(string[] args)
{
var configuration = new ConsulConfiguration();
ConfigurationItemFactory.Default.LayoutRenderers.RegisterDefinition("assembly-name", typeof(AssemblyNameLayoutRenderer));
ConfigurationItemFactory.Default.LayoutRenderers.RegisterDefinition("assembly-version", typeof(AssemblyVersionLayoutRenderer));
_logger = LoggerFactory.Create(builder => builder.AddNLog()).CreateLogger<Program>();
Token = Configuration.GetStringAndWait("coscine/global/gitlabtoken");
try
{
Run();
}
catch (Exception ex)
{
// Handle exceptions by logging them in
_logger.LogError(ex, ex.Message);
LogInnerException(ex);
throw;
}
}
var virtuosoServer = configuration.GetString("coscine/local/virtuoso/additional/url");
private static void Run()
{
var virtuosoServer = Configuration.GetString("coscine/local/virtuoso/additional/url");
var virtuosoHost = new Uri(virtuosoServer).Host;
var virtuosoUser = configuration.GetString("coscine/global/virtuoso_db_user");
var virtuosoPassword = configuration.GetString("coscine/global/virtuoso_db_password");
var virtuosoUser = Configuration.GetString("coscine/global/virtuoso_db_user");
var virtuosoPassword = Configuration.GetString("coscine/global/virtuoso_db_password");
var virtuosoManager = new VirtuosoManager($"Server={virtuosoHost};Uid={virtuosoUser};pwd={virtuosoPassword}");
var _rdfStoreConnector = new RdfStoreConnector(virtuosoServer);
var folder = "C:/voc";
var virtuosoISQLLocation = configuration.GetString(
_logger.LogInformation("Connecting to Virtuoso Server - {serverUrl}", virtuosoServer);
var virtuosoISQLLocation = Configuration.GetString(
"coscine/local/virtuoso/isql",
"C:/Programs/Virtuoso/bin/isql.exe"
);
var graphKeys = configuration.Keys("coscine/local/graphs/");
graphKeys = graphKeys.Where((entry) => entry.Contains("repositoryurl"))
.Select((entry) => entry.Substring(0, entry.LastIndexOf("/") + 1))
.ToArray();
foreach (var graphKey in graphKeys)
// Collect Graphs Types to deploy from Consul
var graphRepos = new List<GraphRepo>();
var graphRepoKeys = Configuration.Keys(GraphsConsulLocation).Select(x =>
{
// Example: {gitHost}/{groupName}/{repository}.git
var repositoryUrl = configuration.GetString(graphKey + "repositoryurl");
Console.WriteLine("Working with: " + repositoryUrl);
x = x.Replace($"{GraphsConsulLocation}/", "");
return x[..x.IndexOf('/')];
}).Distinct().ToList();
graphRepoKeys.ForEach(e =>
{
var graphRepo = new GraphRepo()
{
Name = e,
Branch = Configuration.GetStringAndWait(Path.Combine(GraphsConsulLocation, e, "branch")),
Url = Configuration.GetStringAndWait(Path.Combine(GraphsConsulLocation, e, "repositoryurl"))
};
_logger.LogInformation("Found Graph Repository Url - {@graphRepo}", graphRepo);
graphRepos.Add(graphRepo);
});
var branch = configuration.GetString(graphKey + "branch", "master");
foreach (var graphRepo in graphRepos)
{
_logger.LogInformation("Working with {repoName}", graphRepo.Name.ToUpper());
var repositoryFolder = repositoryUrl.Substring(0, repositoryUrl.LastIndexOf("."));
repositoryFolder = repositoryFolder.Substring(repositoryFolder.LastIndexOf("/") + 1);
// Clear contents inside Working Folder
EmptyWorkingFolder();
ExecuteCommand(
"powershell.exe",
$@"& '{Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "getRepository.ps1")}' "
+ $"'{repositoryUrl}' '{branch}' '{repositoryFolder}'"
);
// Clone the repository inside the Working Folder
CloneGraphRepo(graphRepo, out var success);
if (success)
{
// Graph Insertion
var queries = new List<string>();
foreach (var file in GetFiles(folder).Where((file) => file.Contains(".ttl")))
var turtleFiles = Directory.GetFiles(WorkingFolder, "*.ttl", SearchOption.AllDirectories);
foreach (var file in turtleFiles)
{
var fileInfo = new FileInfo(file);
var graph = new Graph();
......@@ -60,8 +103,9 @@ public class Program
if (_rdfStoreConnector.HasGraph(graphName))
{
Console.WriteLine($"Clearing {graphName}");
_logger.LogDebug("Clearing Graph {graphName}", graphName);
_rdfStoreConnector.ClearGraph(graphName);
_logger.LogInformation("Cleared Graph {graphName}", graphName);
}
queries.Add($"ld_dir('{fileInfo.DirectoryName[2..].Replace("\\", "/")}', '{fileInfo.Name}', '{graphName}');");
......@@ -71,6 +115,7 @@ public class Program
foreach (var query in queries)
{
_logger.LogDebug("Executing queries {@queries}", queries);
ExecuteCommand(
"powershell.exe",
$"\"\\\"{query}\\\" | {virtuosoISQLLocation}\""
......@@ -78,6 +123,10 @@ public class Program
}
}
// Clear contents inside Working Folder
EmptyWorkingFolder();
}
Console.WriteLine("Done");
}
......@@ -105,41 +154,46 @@ public class Program
Console.WriteLine(errors);
}
private static IEnumerable<string> GetFiles(string path)
public static void CloneGraphRepo(GraphRepo graphRepo, out bool success)
{
var queue = new Queue<string>();
queue.Enqueue(path);
while (queue.Count > 0)
string url = $"https://gitlab-ci-token:{Token}@{graphRepo.Url}";
var cloneOptions = new CloneOptions()
{
path = queue.Dequeue();
BranchName = graphRepo.Branch
};
try
{
foreach (string subDir in Directory.GetDirectories(path))
{
queue.Enqueue(subDir);
}
var cloneResult = Repository.Clone(url, WorkingFolder, cloneOptions);
_logger.LogInformation("Repository successfully cloned and switched to branch {branchName}", graphRepo.Branch);
success = true;
}
catch (Exception ex)
{
Console.Error.WriteLine(ex);
_logger.LogError(ex, "Cloning Repository {repoUrl} on branch {branchName}", graphRepo.Url, graphRepo.Branch);
success = false;
}
string[] files = null;
try
{
files = Directory.GetFiles(path);
}
catch (Exception ex)
public static void EmptyWorkingFolder()
{
Console.Error.WriteLine(ex);
}
if (files != null)
var directory = new DirectoryInfo(WorkingFolder);
_logger.LogInformation("Deleting {fileCount} files and {directoryCount} directories from {workingFolder}.", directory.EnumerateFiles().Count(), directory.EnumerateDirectories().Count(), WorkingFolder);
foreach (FileInfo file in directory.EnumerateFiles())
{
for (int i = 0; i < files.Length; i++)
file.Delete();
}
foreach (DirectoryInfo dir in directory.EnumerateDirectories())
{
yield return files[i];
dir.Delete(true);
}
}
private static void LogInnerException(Exception ex)
{
if (ex.InnerException is not null)
{
_logger.LogError(ex.InnerException, "InnerException: {innerException}", ex.InnerException.Message);
LogInnerException(ex.InnerException);
}
}
}
\ No newline at end of file
src/GraphDeployer/getRepository.ps1 deleted 100644 → 0
+ 0
− 47
View file @ e82bb316
param (
[Parameter(Mandatory=$true, Position=0)]
[string] $repositoryUrl,
[Parameter(Mandatory=$true, Position=1)]
[string] $branch,
[Parameter(Mandatory=$true, Position=2)]
[string] $repositoryFolder
)
$extractPath = "C:\Programs\Consul\"
$fileName = "Consul.exe"
$fullPath = ($extractPath + $fileName)
$gitlab_token = & $fullPath kv get "coscine/global/gitlabtoken"
Push-Location $PSScriptRoot
# Twice remove, because of: https://stackoverflow.com/questions/7909167/how-to-quietly-remove-a-directory-with-content-in-powershell#comment10316056_7909195
If(Test-Path $repositoryFolder) { Remove-Item -LiteralPath $repositoryFolder -Force -Recurse }
If(Test-Path $repositoryFolder) { Remove-Item -LiteralPath $repositoryFolder -Force -Recurse }
git clone https://gitlab-ci-token:$gitlab_token@$repositoryUrl
cd $repositoryFolder
if ($branch -And $branch -ne 'master' ) {
git checkout $branch
git pull
}
cd ..
$path = 'C:/voc'
# Twice remove, because of: https://stackoverflow.com/questions/7909167/how-to-quietly-remove-a-directory-with-content-in-powershell#comment10316056_7909195
If(Test-Path $path) { Remove-Item -LiteralPath $path -Force -Recurse }
If(Test-Path $path) { Remove-Item -LiteralPath $path -Force -Recurse }
New-Item -ItemType Directory -Force -Path $path
Copy-Item "./$repositoryFolder/*" $path -Recurse;
# Twice remove, because of: https://stackoverflow.com/questions/7909167/how-to-quietly-remove-a-directory-with-content-in-powershell#comment10316056_7909195
If(Test-Path $repositoryFolder) { Remove-Item -LiteralPath $repositoryFolder -Force -Recurse }
If(Test-Path $repositoryFolder) { Remove-Item -LiteralPath $repositoryFolder -Force -Recurse }
Pop-Location
src/GraphDeployer/nlog.config 0 → 100644
+ 51
− 0
View file @ 3133112b
<?xml version="1.0" encoding="utf-8" ?>
<nlog xmlns="http://www.nlog-project.org/schemas/NLog.xsd"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
throwExceptions="true"
internalLogFile="Logs/internal-nlog.txt"
internalLogLevel="Trace">
<targets>
<!-- Write logs to File -->
<target xsi:type="File" name="fileLog" fileName="C:/coscine/logs/${assembly-name}/${assembly-version}/log-${shortdate}.log" >
<layout xsi:type="CompoundLayout">
<layout xsi:type="JsonLayout" EscapeForwardSlash="true">
<attribute layout="${longdate}" name="Timestamp"/>
<attribute layout="${level:upperCase=true}" name="Level"/>
<attribute layout="${message}" name="Message"/>
<attribute layout="${exception:format=tostring,StackTrace}" name="Exception"/>
<attribute layout="${ndlc}" name="Context"/>
<attribute layout="${event-properties:item=Metric}" name="Alarm" encode="false"/>
<attribute name="EventProperties" encode="false" >
<layout xsi:type='JsonLayout' includeAllProperties="true" maxRecursionLimit="2"/>
</attribute>
</layout>
<layout xsi:type='SimpleLayout' text="," />
</layout>
</target>
<!-- Write colored logs to Console -->
<target name="consoleLog" xsi:type="ColoredConsole" layout="[${uppercase:${level}}]: ${message}">
<highlight-row condition="level == LogLevel.Debug" foregroundColor="DarkGray" />
<highlight-row condition="level == LogLevel.Info" foregroundColor="White" />
<highlight-row condition="level == LogLevel.Warn" foregroundColor="Yellow" />
<highlight-row condition="level == LogLevel.Error" foregroundColor="DarkRed" />
<highlight-row condition="level == LogLevel.Fatal" foregroundColor="Red" backgroundColor="White" />
</target>
</targets>
<rules>
<!--All logs, including from Microsoft, Level Trace-->
<logger name="*" minlevel="Trace" writeTo="fileLog">
</logger>
<!--All logs, including from Microsoft, Level Info-->
<logger name="*" minlevel="Info" writeTo="consoleLog">
<filters defaultAction="Log">
<when condition="contains('${ndlc}','/api/heartbeat')" action="Ignore"/>
</filters>
</logger>
</rules>
</nlog>
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment