Skip to content
Snippets Groups Projects
Commit 6eb65c7e authored by L. Ellenbeck's avatar L. Ellenbeck Committed by Marcel Nellesen
Browse files

New: Creating the parser (coscine/issues#888)

parent 3d93101f
Branches
Tags
2 merge requests!3Product/754 automated import,!2Sprint/2020 13
Showing
with 1649 additions and 0 deletions
## Ignore Visual Studio temporary files, build results, and
## files generated by popular Visual Studio add-ons.
# User-specific files
*.suo
*.user
*.userosscache
*.sln.docstates
# User-specific files (MonoDevelop/Xamarin Studio)
*.userprefs
# Build results
[Dd]ebug/
[Dd]ebugPublic/
[Rr]elease/
[Rr]eleases/
x64/
x86/
bld/
[Bb]in/
[Oo]bj/
[Ll]og/
# Visual Studio 2015 cache/options directory
.vs/
# Uncomment if you have tasks that create the project's static files in wwwroot
#wwwroot/
# MSTest test Results
[Tt]est[Rr]esult*/
[Bb]uild[Ll]og.*
# NUNIT
*.VisualState.xml
TestResult.xml
# Build Results of an ATL Project
[Dd]ebugPS/
[Rr]eleasePS/
dlldata.c
# DNX
project.lock.json
project.fragment.lock.json
artifacts/
*_i.c
*_p.c
*_i.h
*.ilk
*.meta
*.obj
*.pch
*.pdb
*.pgc
*.pgd
*.rsp
*.sbr
*.tlb
*.tli
*.tlh
*.tmp
*.tmp_proj
*.log
*.vspscc
*.vssscc
.builds
*.pidb
*.svclog
*.scc
# Chutzpah Test files
_Chutzpah*
# Visual C++ cache files
ipch/
*.aps
*.ncb
*.opendb
*.opensdf
*.sdf
*.cachefile
*.VC.db
*.VC.VC.opendb
# Visual Studio profiler
*.psess
*.vsp
*.vspx
*.sap
# TFS 2012 Local Workspace
$tf/
# Guidance Automation Toolkit
*.gpState
# ReSharper is a .NET coding add-in
_ReSharper*/
*.[Rr]e[Ss]harper
*.DotSettings.user
Resharper
# JustCode is a .NET coding add-in
.JustCode
# TeamCity is a build add-in
_TeamCity*
# DotCover is a Code Coverage Tool
*.dotCover
# NCrunch
_NCrunch_*
.*crunch*.local.xml
nCrunchTemp_*
# MightyMoose
*.mm.*
AutoTest.Net/
# Web workbench (sass)
.sass-cache/
# Installshield output folder
[Ee]xpress/
# DocProject is a documentation generator add-in
DocProject/buildhelp/
DocProject/Help/*.HxT
DocProject/Help/*.HxC
DocProject/Help/*.hhc
DocProject/Help/*.hhk
DocProject/Help/*.hhp
DocProject/Help/Html2
DocProject/Help/html
# Click-Once directory
publish/
# Publish Web Output
*.[Pp]ublish.xml
*.azurePubxml
# TODO: Comment the next line if you want to checkin your web deploy settings
# but database connection strings (with potential passwords) will be unencrypted
#*.pubxml
*.publishproj
# Microsoft Azure Web App publish settings. Comment the next line if you want to
# checkin your Azure Web App publish settings, but sensitive information contained
# in these scripts will be unencrypted
PublishScripts/
# NuGet Packages
*.nupkg
# The packages folder can be ignored because of Package Restore
**/packages/*
# except build/, which is used as an MSBuild target.
!**/packages/build/
# Uncomment if necessary however generally it will be regenerated when needed
#!**/packages/repositories.config
# NuGet v3's project.json files produces more ignoreable files
*.nuget.props
*.nuget.targets
# Microsoft Azure Build Output
csx/
*.build.csdef
# Microsoft Azure Emulator
ecf/
rcf/
# Windows Store app package directories and files
AppPackages/
BundleArtifacts/
Package.StoreAssociation.xml
_pkginfo.txt
# Visual Studio cache files
# files ending in .cache can be ignored
*.[Cc]ache
# but keep track of directories ending in .cache
!*.[Cc]ache/
# Others
ClientBin/
~$*
*~
*.dbmdl
*.dbproj.schemaview
*.jfm
*.pfx
*.publishsettings
node_modules/
orleans.codegen.cs
# Since there are multiple workflows, uncomment next line to ignore bower_components
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
#bower_components/
# RIA/Silverlight projects
Generated_Code/
# Backup & report files from converting an old project file
# to a newer Visual Studio version. Backup files are not needed,
# because we have git ;-)
_UpgradeReport_Files/
Backup*/
UpgradeLog*.XML
UpgradeLog*.htm
# SQL Server files
*.mdf
*.ldf
# Business Intelligence projects
*.rdl.data
*.bim.layout
*.bim_*.settings
# Microsoft Fakes
FakesAssemblies/
# GhostDoc plugin setting file
*.GhostDoc.xml
# Node.js Tools for Visual Studio
.ntvs_analysis.dat
# Visual Studio 6 build log
*.plg
# Visual Studio 6 workspace options file
*.opt
# Visual Studio LightSwitch build output
**/*.HTMLClient/GeneratedArtifacts
**/*.DesktopClient/GeneratedArtifacts
**/*.DesktopClient/ModelManifest.xml
**/*.Server/GeneratedArtifacts
**/*.Server/ModelManifest.xml
_Pvt_Extensions
# Paket dependency manager
.paket/paket.exe
paket-files/
# FAKE - F# Make
.fake/
# JetBrains Rider
.idea/
*.sln.iml
# CodeRush
.cr/
# Python Tools for Visual Studio (PTVS)
__pycache__/
*.pyc
#cake
tools/
dist/
\ No newline at end of file
stages:
- test
- docs
- release
- releasetrigger
- schedule
cake:Test:
stage: test
script:
- PowerShell .\build.ps1 -Target Test -Configuration Debug
variables:
GIT_STRATEGY: clone
artifacts:
reports:
junit: "./Artifacts/TestResults.xml"
paths:
- "./Artifacts/*"
except:
- master
- tags
cake:Release:
stage: release
script:
- PowerShell .\build.ps1 -Target Release -Configuration Release --nugetApiKey="${NUGET_API_KEY}"
variables:
GIT_STRATEGY: clone
dependencies:
- cake:Test
artifacts:
paths:
- "./Artifacts/*"
only:
- tags
cake:Prerelease:
stage: release
script:
- PowerShell .\build.ps1 -Target Prerelease -Configuration Release
variables:
GIT_STRATEGY: clone
dependencies:
- cake:Test
artifacts:
paths:
- "./Artifacts/*"
except:
- tags
- master
cake:GitlabRelease:
stage: releasetrigger
script:
- PowerShell .\build.ps1 -Target GitlabRelease --GitlabProjectPath="${CI_PROJECT_PATH}" --gitlabProjectId="${CI_PROJECT_ID}" --gitlabToken="${GITLAB_TOKEN}"
only:
- master
cake:PushToGit:
stage: schedule
script:
- Powershell .\Artifacts\OrganizationLoader.exe --output=Artifacts/index.ttl --force="${04XFQ0F34_FORCE}" --rorId="${04XFQ0F34_ROR_ID}" --username="${04XFQ0F34_USERNAME}" --password="${04XFQ0F34_PASSWORD}" --organizationsLink="${04XFQ0F34_ORGANIZATIONS_LINK}" --employeesLink="${04XFQ0F34_EMPLOYEES_LINK}"
- Powershell .\build.ps1 -Target PushToGit --gitlabToken="${GITLAB_TOKEN}" --gitAuthorEmail="${GIT_AUTHOR_EMAIL}" --gitAuthorName="${GIT_AUTHOR_NAME}" --branch="${04XFQ0F34_BRANCH}"
dependencies:
- cake:Prerelease
artifacts:
paths:
- "./Artifacts/index.ttl"
only:
- schedules
\ No newline at end of file
mode: ContinuousDeployment
next-version: 1.0.0
major-version-bump-message: 'Breaking:'
minor-version-bump-message: '(Update|New):'
patch-version-bump-message: 'Fix:'
no-bump-message: '.*'
commit-message-incrementing: Enabled
\ No newline at end of file
LICENSE 0 → 100644
MIT License
Copyright (c) 2019 RWTH Aachen University
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
\ No newline at end of file
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\NUnit3TestAdapter.3.17.0\build\net35\NUnit3TestAdapter.props" Condition="Exists('..\packages\NUnit3TestAdapter.3.17.0\build\net35\NUnit3TestAdapter.props')" />
<Import Project="..\packages\NUnit.3.12.0\build\NUnit.props" Condition="Exists('..\packages\NUnit.3.12.0\build\NUnit.props')" />
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{03640D83-D935-41BE-80A7-7C723FBF58B4}</ProjectGuid>
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>Coscine.OrganizationLoader.Tests</RootNamespace>
<AssemblyName>Coscine.OrganizationLoader.Tests</AssemblyName>
<TargetFrameworkVersion>v4.6.1</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<Deterministic>true</Deterministic>
<NuGetPackageImportStamp>
</NuGetPackageImportStamp>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="Consul, Version=0.7.2.6, Culture=neutral, PublicKeyToken=20a6ad9a81df1d95, processorArchitecture=MSIL">
<HintPath>..\packages\Consul.0.7.2.6\lib\net45\Consul.dll</HintPath>
</Reference>
<Reference Include="Coscine.Configuration, Version=1.5.0.0, Culture=neutral, PublicKeyToken=ce3d7a32d7dc1e5a, processorArchitecture=MSIL">
<HintPath>..\packages\Coscine.Configuration.1.5.0\lib\net461\Coscine.Configuration.dll</HintPath>
</Reference>
<Reference Include="dotNetRDF, Version=2.5.1.0, Culture=neutral, PublicKeyToken=6055ffe4c97cc780, processorArchitecture=MSIL">
<HintPath>..\packages\dotNetRDF.2.5.1\lib\net40\dotNetRDF.dll</HintPath>
</Reference>
<Reference Include="HtmlAgilityPack, Version=1.11.17.0, Culture=neutral, PublicKeyToken=bd319b19eaf3b43a, processorArchitecture=MSIL">
<HintPath>..\packages\HtmlAgilityPack.1.11.17\lib\Net45\HtmlAgilityPack.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json, Version=12.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<HintPath>..\packages\Newtonsoft.Json.12.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="nunit.framework, Version=3.12.0.0, Culture=neutral, PublicKeyToken=2638cd05610744eb, processorArchitecture=MSIL">
<HintPath>..\packages\NUnit.3.12.0\lib\net45\nunit.framework.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Configuration" />
<Reference Include="System.Core" />
<Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Security" />
<Reference Include="System.Web" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Xml" />
<Reference Include="VDS.Common, Version=1.10.0.0, Culture=neutral, PublicKeyToken=ab5f4eb908061bf0, processorArchitecture=MSIL">
<HintPath>..\packages\VDS.Common.1.10.0\lib\net40-client\VDS.Common.dll</HintPath>
</Reference>
</ItemGroup>
<ItemGroup>
<Compile Include="OrganizationLoaderTests.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<ItemGroup>
<None Include="app.config" />
<None Include="packages.config" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\OrganizationLoader\OrganizationLoader.csproj">
<Project>{74141db8-e6ae-49f7-8e99-33ceafb92729}</Project>
<Name>OrganizationLoader</Name>
</ProjectReference>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\NUnit.3.12.0\build\NUnit.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\NUnit.3.12.0\build\NUnit.props'))" />
<Error Condition="!Exists('..\packages\NUnit3TestAdapter.3.17.0\build\net35\NUnit3TestAdapter.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\NUnit3TestAdapter.3.17.0\build\net35\NUnit3TestAdapter.props'))" />
</Target>
</Project>
\ No newline at end of file
using Coscine.Configuration;
using NUnit.Framework;
using System;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using VDS.RDF;
using VDS.RDF.Parsing;
namespace Coscine.OrganizationLoader.Tests
{
[TestFixture]
public class OrganizationLoaderTests
{
readonly IConfiguration configuration = new ConsulConfiguration();
[Test]
public void GraphParsingTest()
{
var httpClient = new HttpClient();
// Create a basic Authentication header
var username = configuration.GetString("coscine/global/organizations/rwth/connection_data/username");
var password = configuration.GetString("coscine/global/organizations/rwth/connection_data/password");
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes($"{username}:{password}")));
string organizationsString;
string employeesString;
using (var response = httpClient.GetStringAsync(configuration.GetString("coscine/global/organizations/rwth/files/organizations/link")))
{
organizationsString = response.Result;
}
using (var response = httpClient.GetStringAsync(configuration.GetString("coscine/global/organizations/rwth/files/employees/link")))
{
employeesString = response.Result;
}
var organizations = Program.ParseOrganizations(organizationsString);
var employees = Program.LoadEmployees(employeesString, organizations);
var ttl = Program.GenerateTurtle(organizations, employees, configuration.GetString("coscine/global/organizations/rwth/ror_url"));
try
{
var graph = new Graph();
graph.LoadFromString(ttl, new TurtleParser());
}
catch (Exception)
{
Assert.True(false);
}
}
}
}
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("OrganizationLoader.Tests")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("OrganizationLoader.Tests")]
[assembly: AssemblyCopyright("Copyright © 2020")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("03640d83-d935-41be-80a7-7c723fbf58b4")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<runtime>
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="linq2db" publicKeyToken="e41013125f9e410a" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-3.0.1.0" newVersion="3.0.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Coscine.Configuration" publicKeyToken="ce3d7a32d7dc1e5a" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.5.0.0" newVersion="1.5.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="dotNetRDF" publicKeyToken="6055ffe4c97cc780" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-2.5.1.0" newVersion="2.5.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Coscine.Database" publicKeyToken="767d77427707b70a" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.22.1.0" newVersion="1.22.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="HtmlAgilityPack" publicKeyToken="bd319b19eaf3b43a" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.11.24.0" newVersion="1.11.24.0" />
</dependentAssembly>
</assemblyBinding>
</runtime>
</configuration>
\ No newline at end of file
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Consul" version="0.7.2.6" targetFramework="net461" />
<package id="Coscine.Configuration" version="1.5.0" targetFramework="net461" />
<package id="dotNetRDF" version="2.5.1" targetFramework="net461" />
<package id="HtmlAgilityPack" version="1.11.17" targetFramework="net461" />
<package id="Newtonsoft.Json" version="12.0.3" targetFramework="net461" />
<package id="NUnit" version="3.12.0" targetFramework="net461" />
<package id="NUnit3TestAdapter" version="3.17.0" targetFramework="net461" />
<package id="VDS.Common" version="1.10.0" targetFramework="net461" />
</packages>
\ No newline at end of file

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16
VisualStudioVersion = 16.0.28803.156
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OrganizationLoader", "OrganizationLoader\OrganizationLoader.csproj", "{74141DB8-E6AE-49F7-8E99-33CEAFB92729}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OrganizationLoader.Tests", "OrganizationLoader.Tests\OrganizationLoader.Tests.csproj", "{03640D83-D935-41BE-80A7-7C723FBF58B4}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{74141DB8-E6AE-49F7-8E99-33CEAFB92729}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{74141DB8-E6AE-49F7-8E99-33CEAFB92729}.Debug|Any CPU.Build.0 = Debug|Any CPU
{74141DB8-E6AE-49F7-8E99-33CEAFB92729}.Release|Any CPU.ActiveCfg = Release|Any CPU
{74141DB8-E6AE-49F7-8E99-33CEAFB92729}.Release|Any CPU.Build.0 = Release|Any CPU
{03640D83-D935-41BE-80A7-7C723FBF58B4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{03640D83-D935-41BE-80A7-7C723FBF58B4}.Debug|Any CPU.Build.0 = Debug|Any CPU
{03640D83-D935-41BE-80A7-7C723FBF58B4}.Release|Any CPU.ActiveCfg = Release|Any CPU
{03640D83-D935-41BE-80A7-7C723FBF58B4}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {9C09B46C-8D03-4E62-B112-0191A5F42D4A}
EndGlobalSection
EndGlobal
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<configSections>
<!-- For more information on Entity Framework configuration, visit http://go.microsoft.com/fwlink/?LinkID=237468 -->
<section name="entityFramework" type="System.Data.Entity.Internal.ConfigFile.EntityFrameworkSection, EntityFramework, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089" requirePermission="false" />
</configSections>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.6.1" />
</startup>
<entityFramework>
<defaultConnectionFactory type="System.Data.Entity.Infrastructure.LocalDbConnectionFactory, EntityFramework">
<parameters>
<parameter value="mssqllocaldb" />
</parameters>
</defaultConnectionFactory>
<providers>
<provider invariantName="System.Data.SqlClient" type="System.Data.Entity.SqlServer.SqlProviderServices, EntityFramework.SqlServer" />
</providers>
</entityFramework>
<runtime>
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="Coscine.Configuration" publicKeyToken="ce3d7a32d7dc1e5a" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.5.0.0" newVersion="1.5.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="dotNetRDF" publicKeyToken="6055ffe4c97cc780" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-2.5.1.0" newVersion="2.5.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Coscine.Database" publicKeyToken="767d77427707b70a" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.22.1.0" newVersion="1.22.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="HtmlAgilityPack" publicKeyToken="bd319b19eaf3b43a" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.11.24.0" newVersion="1.11.24.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="linq2db" publicKeyToken="e41013125f9e410a" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-3.0.1.0" newVersion="3.0.1.0" />
</dependentAssembly>
</assemblyBinding>
</runtime>
</configuration>
\ No newline at end of file
using System.Collections.Generic;
namespace Coscine.OrganizationLoader
{
public class Employee
{
public string Surname { get; set; }
public string PreferredName { get; set; }
public string ID { get; set; }
public string RONID { get; set; }
public List<string> Memberships { get; set; }
public string DegreePre { get; set; }
public string DegreePost { get; set; }
public string Ubpublications { get; set; }
public List<Role> RoleAssignments { get; set; }
}
}
using System.Collections.Generic;
namespace Coscine.OrganizationLoader
{
public class Organization
{
public string OrgId { get; set; }
public string Name { get; set; }
public string IKZ { get; set; }
public string Abbreviation { get; set; }
public string CMSLink { get; set; }
public List<Employee> Members { get; set; } = new List<Employee>();
}
}
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\LibGit2Sharp.NativeBinaries.2.0.306\build\net46\LibGit2Sharp.NativeBinaries.props" Condition="Exists('..\packages\LibGit2Sharp.NativeBinaries.2.0.306\build\net46\LibGit2Sharp.NativeBinaries.props')" />
<Import Project="..\packages\linq2db.t4models.3.0.1\build\linq2db.t4models.props" Condition="Exists('..\packages\linq2db.t4models.3.0.1\build\linq2db.t4models.props')" />
<Import Project="..\packages\linq2db.SqlServer.3.0.1\build\linq2db.SqlServer.props" Condition="Exists('..\packages\linq2db.SqlServer.3.0.1\build\linq2db.SqlServer.props')" />
<Import Project="..\packages\EntityFramework.6.4.4\build\EntityFramework.props" Condition="Exists('..\packages\EntityFramework.6.4.4\build\EntityFramework.props')" />
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{74141DB8-E6AE-49F7-8E99-33CEAFB92729}</ProjectGuid>
<OutputType>Exe</OutputType>
<RootNamespace>Coscine.OrganizationLoader</RootNamespace>
<AssemblyName>OrganizationLoader</AssemblyName>
<TargetFrameworkVersion>v4.6.1</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<Deterministic>true</Deterministic>
<NuGetPackageImportStamp>
</NuGetPackageImportStamp>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PlatformTarget>AnyCPU</PlatformTarget>
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<PlatformTarget>AnyCPU</PlatformTarget>
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="Consul, Version=0.7.2.6, Culture=neutral, PublicKeyToken=20a6ad9a81df1d95, processorArchitecture=MSIL">
<HintPath>..\packages\Consul.0.7.2.6\lib\net45\Consul.dll</HintPath>
</Reference>
<Reference Include="Coscine.Configuration, Version=1.5.0.0, Culture=neutral, PublicKeyToken=ce3d7a32d7dc1e5a, processorArchitecture=MSIL">
<HintPath>..\packages\Coscine.Configuration.1.5.0\lib\net461\Coscine.Configuration.dll</HintPath>
</Reference>
<Reference Include="Coscine.Database, Version=1.22.1.0, Culture=neutral, PublicKeyToken=767d77427707b70a, processorArchitecture=MSIL">
<HintPath>..\packages\Coscine.Database.1.22.1\lib\net461\Coscine.Database.dll</HintPath>
</Reference>
<Reference Include="Coscine.Metadata, Version=1.2.0.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\Coscine.Metadata.1.2.0\lib\net461\Coscine.Metadata.dll</HintPath>
</Reference>
<Reference Include="dotNetRDF, Version=2.5.1.0, Culture=neutral, PublicKeyToken=6055ffe4c97cc780, processorArchitecture=MSIL">
<HintPath>..\packages\dotNetRDF.2.5.1\lib\net40\dotNetRDF.dll</HintPath>
</Reference>
<Reference Include="dotNetRDF.Data.Virtuoso, Version=2.5.1.0, Culture=neutral, PublicKeyToken=6055ffe4c97cc780, processorArchitecture=MSIL">
<HintPath>..\packages\dotNetRDF.Data.Virtuoso.2.5.1\lib\net40\dotNetRDF.Data.Virtuoso.dll</HintPath>
</Reference>
<Reference Include="EntityFramework, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089, processorArchitecture=MSIL">
<HintPath>..\packages\EntityFramework.6.4.4\lib\net45\EntityFramework.dll</HintPath>
</Reference>
<Reference Include="EntityFramework.SqlServer, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089, processorArchitecture=MSIL">
<HintPath>..\packages\EntityFramework.6.4.4\lib\net45\EntityFramework.SqlServer.dll</HintPath>
</Reference>
<Reference Include="HtmlAgilityPack, Version=1.11.24.0, Culture=neutral, PublicKeyToken=bd319b19eaf3b43a, processorArchitecture=MSIL">
<HintPath>..\packages\HtmlAgilityPack.1.11.24\lib\Net45\HtmlAgilityPack.dll</HintPath>
</Reference>
<Reference Include="LibGit2Sharp, Version=0.26.0.0, Culture=neutral, PublicKeyToken=7cbde695407f0333, processorArchitecture=MSIL">
<HintPath>..\packages\LibGit2Sharp.0.26.2\lib\net46\LibGit2Sharp.dll</HintPath>
</Reference>
<Reference Include="linq2db, Version=3.0.1.0, Culture=neutral, PublicKeyToken=e41013125f9e410a, processorArchitecture=MSIL">
<HintPath>..\packages\linq2db.3.0.1\lib\net46\linq2db.dll</HintPath>
</Reference>
<Reference Include="LinqKit, Version=1.1.17.0, Culture=neutral, PublicKeyToken=bc217f8844052a91, processorArchitecture=MSIL">
<HintPath>..\packages\LinqKit.1.1.17\lib\net45\LinqKit.dll</HintPath>
</Reference>
<Reference Include="NDesk.Options, Version=0.2.1.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\NDesk.Options.0.2.1\lib\NDesk.Options.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json, Version=12.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<HintPath>..\packages\Newtonsoft.Json.12.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.ComponentModel.DataAnnotations" />
<Reference Include="System.Configuration" />
<Reference Include="System.Core" />
<Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Security" />
<Reference Include="System.Web" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Xml" />
<Reference Include="VDS.Common, Version=1.10.0.0, Culture=neutral, PublicKeyToken=ab5f4eb908061bf0, processorArchitecture=MSIL">
<HintPath>..\packages\VDS.Common.1.10.0\lib\net40-client\VDS.Common.dll</HintPath>
</Reference>
<Reference Include="virtado4, Version=4.0.0.0, Culture=neutral, PublicKeyToken=391bf132017ae989, processorArchitecture=MSIL">
<HintPath>..\packages\OpenLink.Data.Virtuoso.7.20.3214.1\lib\net40\virtado4.dll</HintPath>
</Reference>
</ItemGroup>
<ItemGroup>
<Compile Include="Employee.cs" />
<Compile Include="Organization.cs" />
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="Role.cs" />
</ItemGroup>
<ItemGroup>
<None Include="App.config" />
<None Include="packages.config" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\EntityFramework.6.4.4\build\EntityFramework.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\EntityFramework.6.4.4\build\EntityFramework.props'))" />
<Error Condition="!Exists('..\packages\EntityFramework.6.4.4\build\EntityFramework.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\EntityFramework.6.4.4\build\EntityFramework.targets'))" />
<Error Condition="!Exists('..\packages\linq2db.SqlServer.3.0.1\build\linq2db.SqlServer.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\linq2db.SqlServer.3.0.1\build\linq2db.SqlServer.props'))" />
<Error Condition="!Exists('..\packages\linq2db.t4models.3.0.1\build\linq2db.t4models.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\linq2db.t4models.3.0.1\build\linq2db.t4models.props'))" />
<Error Condition="!Exists('..\packages\LibGit2Sharp.NativeBinaries.2.0.306\build\net46\LibGit2Sharp.NativeBinaries.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\LibGit2Sharp.NativeBinaries.2.0.306\build\net46\LibGit2Sharp.NativeBinaries.props'))" />
</Target>
<Import Project="..\packages\EntityFramework.6.4.4\build\EntityFramework.targets" Condition="Exists('..\packages\EntityFramework.6.4.4\build\EntityFramework.targets')" />
</Project>
\ No newline at end of file
using Coscine.Configuration;
using NDesk.Options;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Security.Cryptography;
using System.Text;
using System.Xml;
namespace Coscine.OrganizationLoader
{
public class Program
{
public static HttpClient httpClient;
static void Main(string[] args)
{
bool showHelp = false;
bool force = false;
string username = null;
string password = null;
string organizationsLink = null;
string employeesLink = null;
string rorId = null;
string output = null;
var optionSet = new OptionSet() {
{ "force=", "Skip checks for to many or no changes in the exports.",
x => force = x == null ? false : x == "true" },
{ "username=", "Username for the export. If none is provided, the consul value is used.",
x => username = x },
{ "password=", "Password for the export. If none is provided, the consul value is used.",
x => password = x },
{ "organizationsLink=", "Link for the organizations export. If none is provided, the consul value is used.",
x => organizationsLink = x },
{ "employeesLink=", "Link for the employee export. If none is provided, the consul value is used.",
x => employeesLink = x },
{ "rorId=", "The RWTH ror Id. If none is provided, the consul value is used.",
x => rorId = x },
{ "output=", "Output for the generated file (Mandatory).",
x => output = x },
{ "h|help", "show help and exit",
x => showHelp = x != null },
};
List<string> extra;
try
{
extra = optionSet.Parse(args);
}
catch (OptionException e)
{
Console.WriteLine(e.Message);
Console.WriteLine("Try '--help' for more information.");
return;
}
if (showHelp)
{
foreach (var option in optionSet)
{
Console.WriteLine($"{option}: {option.Description}");
}
return;
}
if (force)
{
Console.WriteLine("Forced execution! Checks will be skipped.");
}
if(string.IsNullOrWhiteSpace(output))
{
Console.WriteLine("No value for output provided.");
return;
}
var configuration = new ConsulConfiguration();
rorId = string.IsNullOrWhiteSpace(rorId) ? configuration.GetString("coscine/global/organizations/rwth/ror_url") : rorId;
if(string.IsNullOrWhiteSpace(rorId))
{
Console.WriteLine("No value for rorId provided.");
}
organizationsLink = string.IsNullOrWhiteSpace(organizationsLink) ? configuration.GetString("coscine/global/organizations/rwth/files/organizations/link") : organizationsLink;
if (string.IsNullOrWhiteSpace(organizationsLink))
{
Console.WriteLine("No value for organizationsLink provided.");
}
employeesLink = string.IsNullOrWhiteSpace(employeesLink) ? configuration.GetString("coscine/global/organizations/rwth/files/employees/link") : employeesLink;
if (string.IsNullOrWhiteSpace(employeesLink))
{
Console.WriteLine("No value for employeesLink provided.");
}
username = string.IsNullOrWhiteSpace(username) ? configuration.GetString("coscine/global/organizations/rwth/connection_data/username") : username;
if (string.IsNullOrWhiteSpace(username))
{
Console.WriteLine("No value for username provided.");
}
password = string.IsNullOrWhiteSpace(password) ? configuration.GetString("coscine/global/organizations/rwth/connection_data/password") : password;
if (string.IsNullOrWhiteSpace(password))
{
Console.WriteLine("No value for password provided.");
}
httpClient = new HttpClient();
// Create a basic Authentication header
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes($"{username}:{password}")));
var previousEmployeeCountString = configuration.GetString("coscine/global/organizations/rwth/files/employees/number_of_entries");
var previousEmployeeCount = previousEmployeeCountString != null ? int.Parse(previousEmployeeCountString) : -1;
var previousOrganizationsCountString = configuration.GetString("coscine/global/organizations/rwth/files/organizations/number_of_entries");
var previousOrganizationsCount = previousOrganizationsCountString != null ? int.Parse(previousOrganizationsCountString) : -1;
var previousEmployeeSha = configuration.GetString("coscine/global/organizations/rwth/files/employees/sha");
var previousOrganizationsSha = configuration.GetString("coscine/global/organizations/rwth/files/organizations/sha");
string organizationsString;
string employeesString;
try
{
Console.WriteLine("Downloading...");
organizationsString = DownloadExport(organizationsLink);
employeesString = DownloadExport(employeesLink);
}
catch (Exception e)
{
Console.WriteLine($"Failed to download exports: {e.Message}");
return;
}
Console.WriteLine("Parsing...");
var organizations = ParseOrganizations(organizationsString);
var employees = LoadEmployees(employeesString, organizations);
if (!force)
{
Console.WriteLine("Checking...");
if (previousEmployeeCount > 0 && Math.Abs(previousEmployeeCount - employees.Count()) > previousEmployeeCount * 0.1f)
{
Console.WriteLine($"To many employees changed! Previously: {previousEmployeeCount}, Now: {employees.Count()}, Difference: {Math.Abs(previousEmployeeCount - employees.Count())} or {((float)Math.Abs(previousEmployeeCount - employees.Count()) / previousEmployeeCount) * 100} %");
return;
}
if (previousOrganizationsCount > 0 && Math.Abs(previousOrganizationsCount - organizations.Count()) > previousOrganizationsCount * 0.1f)
{
Console.WriteLine($"To many organizations changed! Previously: {previousOrganizationsCount}, Now: {organizations.Count()}, Difference: {Math.Abs(previousOrganizationsCount - organizations.Count())} or {((float)Math.Abs(previousOrganizationsCount - organizations.Count()) / previousOrganizationsCount) * 100} %");
return;
}
}
var organizationsSha = Sha256Hash(organizationsString);
var employeesSha = Sha256Hash(employeesString);
if(!force && previousEmployeeSha != null && previousEmployeeSha == employeesSha && previousOrganizationsSha != null && previousOrganizationsSha == organizationsSha)
{
Console.WriteLine("The sha of the exports has not changed.");
return;
}
Console.WriteLine("Generating ttl file...");
var ttl = GenerateTurtle(organizations, employees, rorId);
var file = new FileInfo(output);
file.Directory.Create();
File.WriteAllText(output, ttl);
Console.WriteLine($"File written to {output}");
Console.WriteLine("Saving to Consul...");
configuration.PutAndWait("coscine/global/organizations/rwth/files/organizations/number_of_entries", organizations.Count().ToString());
configuration.PutAndWait("coscine/global/organizations/rwth/files/employees/number_of_entries", employees.Count().ToString());
configuration.PutAndWait("coscine/global/organizations/rwth/files/organizations/sha", organizationsSha);
configuration.PutAndWait("coscine/global/organizations/rwth/files/employees/sha", employeesSha);
Console.WriteLine("Finished");
}
public static string Sha256Hash(string value)
{
using (var hash = SHA256.Create())
{
return string.Concat(hash
.ComputeHash(Encoding.UTF8.GetBytes(value))
.Select(item => item.ToString("x2")));
}
}
public static List<Employee> LoadEmployees(string xml, Dictionary<string, Organization> organizations)
{
var employeeDocument = new XmlDocument();
employeeDocument.LoadXml(xml);
var employees = new List<Employee>();
foreach (XmlNode node in employeeDocument.SelectNodes("/IDM_Exports/Employees/Employee"))
{
var employee = new Employee
{
Surname = node["Surname"].InnerText?.Trim(),
PreferredName = node["PreferredName"].InnerText?.Trim(),
ID = node["ID"].InnerText?.Trim(),
RONID = node["RONID"].InnerText?.Trim(),
Memberships = node["OrganizationMemberships"]?.ChildNodes?.Cast<XmlNode>().Select(x => x["OrgId"].InnerText?.Trim()).ToList(),
DegreePre = node["DegreePre"]?.InnerText,
DegreePost = node["DegreePost"]?.InnerText,
Ubpublications = node["Ubpublications"]?.InnerText?.Trim(),
RoleAssignments = node["RoleAssignments"]?.ChildNodes?.Cast<XmlNode>().Select(x => new Role { RID = x["RID"].InnerText?.Trim(), OrgId = x["OrgId"].InnerText?.Trim() }).ToList(),
};
employees.Add(employee);
if (employee.Memberships != null)
{
foreach (var orgId in employee.Memberships)
{
organizations[orgId].Members.Add(employee);
}
}
}
return employees;
}
public static string DownloadExport(string link)
{
using (var response = httpClient.GetStringAsync(link))
{
return response.Result;
}
}
public static Dictionary<string, Organization> ParseOrganizations(string xml)
{
var organizationDocument = new XmlDocument();
organizationDocument.LoadXml(xml);
var organizations = new Dictionary<string, Organization>();
foreach (XmlNode node in organizationDocument.SelectNodes("/IDM_Exports/Organizations/Organization"))
{
organizations.Add(node["OrgId"].InnerText,
new Organization
{
OrgId = node["OrgId"].InnerText?.Trim(),
Name = node["Name"].InnerText?.Trim(),
IKZ = node["IKZ"].InnerText?.Trim(),
Abbreviation = node["Abbreviation"]?.InnerText?.Trim(),
CMSLink = node["CMSLink"].InnerText?.Trim(),
}
);
}
return organizations;
}
public static string GenerateTurtle(Dictionary<string, Organization> organizations, List<Employee> employees, string rorId)
{
// Disposed by streamWriter
var memoryStream = new MemoryStream();
using (var streamWriter = new StreamWriter(memoryStream))
{
// Header
streamWriter.WriteLine($"@base <{rorId}> .");
streamWriter.WriteLine();
streamWriter.WriteLine(@"@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .");
streamWriter.WriteLine(@"@prefix org: <http://www.w3.org/ns/org#> .");
streamWriter.WriteLine(@"@prefix dcterms: <http://purl.org/dc/terms/> .");
streamWriter.WriteLine(@"@prefix foaf: <http://xmlns.com/foaf/0.1/> .");
streamWriter.WriteLine();
streamWriter.WriteLine($"<{rorId}>");
streamWriter.WriteLine(@" dcterms:publisher <https://itc.rwth-aachen.de/> ;");
streamWriter.WriteLine(@" dcterms:rights ""Copyright © 2020 IT Center, RWTH Aachen University"" ;");
streamWriter.WriteLine(@" dcterms:title ""RWTH Aachen University""@en ;");
streamWriter.WriteLine();
streamWriter.WriteLine(@" a org:FormalOrganization ;");
streamWriter.WriteLine(@" rdfs:label ""RWTH Aachen University"" ;");
foreach (var organization in organizations.Values)
{
streamWriter.WriteLine($" org:hasUnit <{rorId}#{organization.OrgId}> ;");
}
streamWriter.WriteLine(".");
// Organizations
foreach (var organization in organizations.Values)
{
streamWriter.WriteLine($"<{rorId}#{organization.OrgId}>");
streamWriter.WriteLine($" rdfs:label \"{organization.Name}\" ;");
streamWriter.WriteLine($" org:identifier \"{organization.OrgId}\" ;");
streamWriter.WriteLine($" a org:OrganizationalUnit ;");
streamWriter.WriteLine(".");
}
// Memberships
foreach (var employee in employees)
{
if (employee.Memberships != null)
{
foreach (var membership in employee.Memberships)
{
streamWriter.WriteLine("[] a org:Membership ;");
streamWriter.WriteLine($" org:member <{rorId}#{employee.ID}> ;");
streamWriter.WriteLine($" org:organization <{rorId}#{membership}> ;");
streamWriter.WriteLine(".");
}
}
}
// Persons
foreach (var employee in employees)
{
streamWriter.WriteLine($"<{rorId}#{employee.ID}>");
streamWriter.WriteLine($" foaf:name \"{employee.PreferredName} {employee.Surname}\" ;");
streamWriter.WriteLine($" foaf:givenName \"{employee.PreferredName}\" ;");
streamWriter.WriteLine($" foaf:familyName \"{employee.Surname}\" ;");
streamWriter.WriteLine($" foaf:openId \"{employee.ID}\" ;");
streamWriter.WriteLine($" a foaf:Person ;");
streamWriter.WriteLine(".");
}
streamWriter.Flush();
return Encoding.UTF8.GetString(memoryStream.ToArray());
}
}
}
}
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("OrganizationLoader")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("OrganizationLoader")]
[assembly: AssemblyCopyright("Copyright © 2020")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("74141db8-e6ae-49f7-8e99-33ceafb92729")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]
namespace Coscine.OrganizationLoader
{
public class Role
{
public string RID { get; set; }
public string OrgId { get; set; }
}
}
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Consul" version="0.7.2.6" targetFramework="net461" />
<package id="Coscine.Configuration" version="1.5.0" targetFramework="net461" />
<package id="Coscine.Database" version="1.22.1" targetFramework="net461" />
<package id="Coscine.Metadata" version="1.2.0" targetFramework="net461" />
<package id="dotNetRDF" version="2.5.1" targetFramework="net461" />
<package id="dotNetRDF.Data.Virtuoso" version="2.5.1" targetFramework="net461" />
<package id="EntityFramework" version="6.4.4" targetFramework="net461" />
<package id="HtmlAgilityPack" version="1.11.24" targetFramework="net461" />
<package id="LibGit2Sharp" version="0.26.2" targetFramework="net461" />
<package id="LibGit2Sharp.NativeBinaries" version="2.0.306" targetFramework="net461" />
<package id="linq2db" version="3.0.1" targetFramework="net461" />
<package id="linq2db.SqlServer" version="3.0.1" targetFramework="net461" />
<package id="linq2db.t4models" version="3.0.1" targetFramework="net461" />
<package id="LinqKit" version="1.1.17" targetFramework="net461" />
<package id="NDesk.Options" version="0.2.1" targetFramework="net461" />
<package id="Newtonsoft.Json" version="12.0.3" targetFramework="net461" />
<package id="OpenLink.Data.Virtuoso" version="7.20.3214.1" targetFramework="net461" />
<package id="VDS.Common" version="1.10.0" targetFramework="net461" />
</packages>
\ No newline at end of file
# OrganizationLoader
Commandline application which downloads the IDM exports and parses it to a .ttl file.
It has the following parameters
- force=: Skip checks for to many or no changes in the exports.
- username=: Username for the export. If none is provided, the consul value is used.
- password=: Password for the export. If none is provided, the consul value is used.
- organizationsLink=: Link for the organizations export. If none is provided, the consul value is used.
- employeesLink=: Link for the employee export. If none is provided, the consul value is used.
- rorId=: The RWTH ror Id. If none is provided, the consul value is used.
- output=: Output for the generated file (Mandatory).
- h|help: show help and exit
To show the above output, run: `.\OrganizationLoader.exe --h`
Create a .ttl file with: `.\OrganizationLoader.exe --output="index.ttl"`
Should a parameter be empty or not provided, the program will look it up in Consul.
Only the `output` parameter has to be supplied via the commandline.
The other configuration parametes must be provided by the commandline or by Consul.
Commandline parameters have a higher priority and are choosen instead of Consul values.
# Gitlab CI
Most commandline parameters can be set over the variables section of the Gitlab project.
The help and output parameter are unused or fixed.
The scheduled CI task runs every morning at 1am.
The task builds the executable and runs it automatically.
\ No newline at end of file
#tool nuget:?package=NUnit.ConsoleRunner&version=3.10.0
#tool nuget:?package=vswhere&version=2.8.4
#tool nuget:?package=GitVersion.CommandLine&version=5.1.3
#addin nuget:https://api.nuget.org/v3/index.json?package=Cake.Json&version=4.0.0
#addin nuget:https://api.nuget.org/v3/index.json?package=Newtonsoft.Json&version=11.0.2
#addin nuget:https://api.nuget.org/v3/index.json?package=Cake.FileHelpers&version=3.2.1
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
// Commandline arguments
var target = Argument("target", "Default");
var configuration = Argument("configuration", "Release");
var nugetApiKey = Argument<string>("nugetApiKey", null);
var version = Argument("nugetVersion", "");
var gitlabProjectPath = Argument("gitlabProjectPath", "");
var gitlabProjectId = Argument("gitlabProjectId", "");
var gitlabToken = Argument("gitlabToken", "");
var branch = Argument("branch", "");
var gitAuthorEmail = Argument("gitAuthorEmail", "");
var gitAuthorName = Argument("gitAuthorName", "");
// Define directories
var projects = GetFiles("./**/*.csproj");
var artifactsDir = Directory("./Artifacts");
string nupkgDir;
var solutionFile = GetFiles("./**/*.sln").First();
var projectName = solutionFile.GetFilenameWithoutExtension().ToString();
var nugetSource = "https://api.nuget.org/v3/index.json";
var assemblyInfoSubPath = "Properties/AssemblyInfo.cs";
var semanticVersion = "";
string localNugetFeed;
// get latest MSBuild version
var vsLatest = VSWhereLatest();
var msBuildPathX64 = (vsLatest == null) ? null : vsLatest.CombineWithFilePath("./MSBuild/Current/Bin/MSBuild.exe");
Setup(context =>{
nupkgDir = $"{artifactsDir.ToString()}/nupkg";
var branch = GitVersion(new GitVersionSettings {
UpdateAssemblyInfo = false
}).BranchName.Replace("/", "-");
localNugetFeed = $"C:\\coscine\\LocalNugetFeeds\\{branch}";
Information("{0}", branch);
Information("Started at {0}", DateTime.Now);
});
Teardown(context =>{
Information("Finished at {0}", DateTime.Now);
});
Task("Clean")
.Description("Cleans all build and artifacts directories")
.Does(() =>{
var settings = new DeleteDirectorySettings {
Recursive = true,
Force = true
};
var directoriesToClean = new List<DirectoryPath>();
foreach(var project in projects) {
directoriesToClean.Add(Directory($"{project.GetDirectory()}/obj"));
directoriesToClean.Add(Directory($"{project.GetDirectory()}/bin"));
}
directoriesToClean.Add(artifactsDir);
foreach(var dir in directoriesToClean) {
Information("Cleaning {0}", dir.ToString());
if (DirectoryExists(dir)) {
DeleteDirectory(dir, settings);
CreateDirectory(dir);
} else {
CreateDirectory(dir);
}
}
});
Task("Restore")
.Does(() =>{
NuGetRestore(solutionFile, new NuGetRestoreSettings {
NoCache = true,
FallbackSource = new List<string>{ localNugetFeed },
});
});
Task("Test")
.IsDependentOn("Build")
.Does(() =>{
NUnit3($"./src/**/bin/{configuration}/*.Tests.dll", new NUnit3Settings {
// generate the xml file
NoResults = false,
Results = new NUnit3Result[] {
new NUnit3Result() {
FileName = $"{artifactsDir}/TestResults.xml",
Transform = $"{Context.Environment.WorkingDirectory}/nunit3-junit.xslt"
}
}
});
});
Task("GitVersion")
.Does(() => {
if(string.IsNullOrWhiteSpace(version)) {
version = GitVersion(new GitVersionSettings {
UpdateAssemblyInfo = false
}).NuGetVersionV2;
}
var index = version.IndexOf("-");
semanticVersion = index > 0 ? version.Substring(0, index) : version;
Information("Version: {0}, SemanticVersion: {1}", version, semanticVersion);
});
Task("UpdateAssemblyInfo")
.Does(() =>{
var index = version.IndexOf("-");
var semanticVersion = index > 0 ? version.Substring(0, index) : version;
foreach(var project in projects) {
CreateAssemblyInfo($"{project.GetDirectory()}/{assemblyInfoSubPath}", new AssemblyInfoSettings {
Product = project.GetFilenameWithoutExtension().ToString(),
Title = project.GetFilenameWithoutExtension().ToString(),
Company = "IT Center, RWTH Aachen University",
Version = semanticVersion,
FileVersion = semanticVersion,
InformationalVersion = version,
Copyright = $"{DateTime.Now.Year} IT Center, RWTH Aachen University",
Description = $"{project.GetFilenameWithoutExtension().ToString()} is a part of the CoScInE group."
});
}
});
Task("GitlabRelease")
.IsDependentOn("GitVersion")
.Does(() => {
var client = new HttpClient();
client.DefaultRequestHeaders.Add("PRIVATE-TOKEN", gitlabToken);
// get the latest tag
var result = client.GetAsync($"https://git.rwth-aachen.de/api/v4/projects/{gitlabProjectId}/repository/tags").Result;
if(!result.IsSuccessStatusCode) {
throw new Exception("Tag query failed.");
}
var tagList = result.Content.ReadAsStringAsync().Result;
var jArray = JArray.Parse(tagList);
// null if not tags exists yet
var lastTag = jArray.Select(x => x["name"]).FirstOrDefault();
var url = $"https://git.rwth-aachen.de/{gitlabProjectPath}";
if(url.EndsWith(".git")) {
url = url.Substring(0, url.Length - ".git".Length);
}
if(url.EndsWith("/")) {
url = url.Substring(0, url.Length - 1);
}
var description = "";
// First line of description
// Gitlab compare url, if something can be compared
if(lastTag == null) {
description = $"# {semanticVersion} ({DateTime.Now.Year}-{DateTime.Now.Month}-{DateTime.Now.Day})\n\n\n";
} else {
description = $"# [{semanticVersion}]({url}/compare/{lastTag}...v{semanticVersion}) ({DateTime.Now.Year}-{DateTime.Now.Month}-{DateTime.Now.Day})\n\n\n";
}
// From when will messages be parsed, null results in all messages
var logParam = "";
if(lastTag != null) {
logParam = $"{lastTag}..Head";
}
Information(lastTag);
IEnumerable<string> redirectedStandardOutput;
var exitCodeWithArgument =
StartProcess(
"git",
new ProcessSettings {
Arguments = $"log {logParam} --pretty=format:HASH%h:%B",
RedirectStandardOutput = true
},
out redirectedStandardOutput
);
var prefixList = new Dictionary<string, List<string>>{
{"Fix", new List<string>()},
{"Update", new List<string>()},
{"New", new List<string>()},
{"Breaking", new List<string>()},
{"Docs", new List<string>()},
{"Build", new List<string>()},
{"Upgrade", new List<string>()},
{"Chore", new List<string>()},
};
var currentHash = "";
// Output last line of process output.
foreach(var line in redirectedStandardOutput) {
var commitMessage = "";
if(line.StartsWith("HASH")) {
currentHash = line.Substring("HASH".Length);
currentHash = currentHash.Substring(0, currentHash.IndexOf(":"));
commitMessage = line.Substring(currentHash.Length + line.IndexOf(currentHash) + 1);
} else {
commitMessage = line;
}
foreach(var kv in prefixList) {
if(commitMessage.StartsWith($"{kv.Key}:")) {
kv.Value.Add($"* {commitMessage.Substring(kv.Key.Length + 1).Trim()} {currentHash}");
break;
}
};
}
foreach(var kv in prefixList) {
if(kv.Value.Any()) {
description += $" ### {kv.Key}\n\n";
foreach(var line in kv.Value) {
description += $"{line}\n";
}
description += "\n";
}
}
// correctly escape the json newlines
description = description.Replace("\n", "\\n");
Information("Description: {0}", description);
// create tag
result = client.PostAsync($"https://git.rwth-aachen.de/api/v4/projects/{gitlabProjectId}/repository/tags?tag_name=v{semanticVersion}&ref=master", null).Result;
Information("Create tag: {0}", result.Content.ReadAsStringAsync().Result);
if(!result.IsSuccessStatusCode) {
throw new Exception("Tag creation failed.");
}
// create release
var json = $"{{\"name\": \"v{semanticVersion}\", \"tag_name\": \"v{semanticVersion}\", \"description\": \"{description}\"}}";
var content = new StringContent(json, Encoding.UTF8, "application/json");
result = client.PostAsync($"https://git.rwth-aachen.de/api/v4/projects/{gitlabProjectId}/releases", content).Result;
Information("Create release: {0}", result.Content.ReadAsStringAsync().Result);
if(!result.IsSuccessStatusCode) {
throw new Exception("Release creation failed.");
}
});
Task("Build")
.IsDependentOn("Clean")
.IsDependentOn("GitVersion")
.IsDependentOn("UpdateAssemblyInfo")
.IsDependentOn("Restore")
.Does(() =>{
var frameworkSettingsWindows = new MSBuildSettings {
Configuration = configuration
};
frameworkSettingsWindows.ToolPath = msBuildPathX64;
frameworkSettingsWindows.WorkingDirectory = Context.Environment.WorkingDirectory;
if (configuration.Equals("Release")) {
frameworkSettingsWindows.WithProperty("DebugSymbols", "false");
frameworkSettingsWindows.WithProperty("DebugType", "None");
}
// Use MSBuild
Information("Building {0}", solutionFile);
MSBuild(solutionFile, frameworkSettingsWindows);
});
Task("NugetPack")
.IsDependentOn("Build")
.Does(() =>{
foreach(var project in projects) {
var nuspec = $"{project.GetDirectory()}/{project.GetFilenameWithoutExtension()}.nuspec";
if(!project.ToString().EndsWith(".Tests") && FileExists(nuspec))
{
var settings = new NuGetPackSettings
{
OutputDirectory = nupkgDir,
Version = version,
Properties = new Dictionary<string, string>
{
{ "Configuration", configuration}
}
};
NuGetPack(project.ToString(), settings);
}
}
});
Task("NugetPush")
.IsDependentOn("NugetPack")
.Does(() =>{
var nupkgs = GetFiles($"{nupkgDir}/*.nupkg");
Information("Need to push {0} packages", nupkgs.Count);
if(!String.IsNullOrWhiteSpace(nugetApiKey)) {
foreach(var nupkg in nupkgs) {
Information("Pushing {0}", nupkg);
NuGetPush(nupkg, new NuGetPushSettings {
Source = nugetSource,
ApiKey = nugetApiKey
});
}
} else {
Information("NugetApiKey is not set. Can't push.");
throw new Exception("NugetApiKey is not set. Can't push.");
}
});
Task("CopyToArtifacts")
.Does(() =>{
foreach(var project in projects) {
if(!project.GetDirectory().ToString().EndsWith(".Tests")
&& !FileExists($"{project.GetDirectory()}/{project.GetFilenameWithoutExtension()}.nuspec")
&& DirectoryExists(project.GetDirectory()))
{
Information("Copying {0}/* to {1}", $"{project.GetDirectory()}/bin/{configuration}", artifactsDir);
CopyDirectory($"{project.GetDirectory()}/bin/{configuration}/", artifactsDir);
}
}
});
Task("NugetPushLocal")
.IsDependentOn("NugetPack")
.Does(() =>{
var nupkgs = GetFiles($"{nupkgDir}/*.nupkg");
foreach(var nupkg in nupkgs) {
if(!DirectoryExists(localNugetFeed)) {
CreateDirectory(localNugetFeed);
}
CopyFile(nupkg.ToString(), $"{localNugetFeed}\\{nupkg.GetFilename()}");
}
});
Task("PushToGit")
.Does(() =>{
if(FileExists($"{artifactsDir}/index.ttl"))
{
var client = new HttpClient();
var content = FileReadText($"{artifactsDir}/index.ttl");
var request = new HttpRequestMessage
{
Method = HttpMethod.Put,
RequestUri = new Uri("https://git.rwth-aachen.de/api/v4/projects/47542/repository/files/index.ttl"),
Headers =
{
{ "private-token", gitlabToken },
},
Content = new StringContent($"{{\"branch\": \"{branch}\", \"author_email\":\"{gitAuthorEmail}\", \"author_name\": \"{gitAuthorName}\", \"content\": \"{content.Replace("\r", "\\r").Replace("\n", "\\n").Replace("\"", "\\\"")}\", \"commit_message\": \"update file\"}}")
{
Headers =
{
ContentType = new MediaTypeHeaderValue("application/json")
}
}
};
using (var response = client.SendAsync(request).Result)
{
var body = response.Content.ReadAsStringAsync().Result;
Information("IsSuccess: {0}", response.IsSuccessStatusCode);
Information("StatusCode: {0}", response.StatusCode);
Information("Response: {0}", response.Content.ReadAsStringAsync().Result);
response.EnsureSuccessStatusCode();
}
} else {
Information("No index ttl found. Nothing to do.");
}
});
Task("Prerelease")
.IsDependentOn("Build")
.IsDependentOn("CopyToArtifacts")
.IsDependentOn("NugetPushLocal");
Task("Release")
.IsDependentOn("NugetPack")
.IsDependentOn("CopyToArtifacts")
.IsDependentOn("NugetPushLocal")
.IsDependentOn("NugetPush");
Task("Default")
.IsDependentOn("Test");
RunTarget(target);
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment