.gitlab-ci.yml 11.5 KB
Newer Older
1
variables:
2
  RSYNC_OPTS: --recursive --ignore-missing-args --chown ${DEPLOY_USER}:${DEPLOY_USER}
3
4
5
  DOCKER_TAG: ${CI_COMMIT_REF_NAME}
  DOCKER_IMAGE: rwthacs/dpsim
  DOCKER_IMAGE_DEV: rwthacs/dpsim-dev
6
7

before_script:
Steffen Vogel's avatar
fix CI    
Steffen Vogel committed
8
  - git config --local core.longpaths true
9
  - git submodule sync --recursive
10
  - git submodule update --init --recursive
11

12
13
14
stages:
  - prepare
  - build
Markus Mirz's avatar
Markus Mirz committed
15
  - compile
Steffen Vogel's avatar
Steffen Vogel committed
16
  - test
Markus Mirz's avatar
fix ci    
Markus Mirz committed
17
  - generate
Markus Mirz's avatar
Markus Mirz committed
18
  - deploy
19

Markus Mirz's avatar
Markus Mirz committed
20
.docker-fedora-template: &docker-fedora-template
21
  stage: prepare
22
  needs: []
23
  script:
24
    - docker build
25
        --file Packaging/Docker/Dockerfile.dev
Markus Mirz's avatar
Markus Mirz committed
26
27
        ${TAGS}
        ${BUILD_ARGS} .
28
29
30
31
  tags:
    - shell
    - linux

Markus Mirz's avatar
Markus Mirz committed
32
33
34
35
36
37
38
39
40
41
42
docker:fedora:
  <<: *docker-fedora-template
  variables:
    TAGS: --tag ${DOCKER_IMAGE_DEV}:${CI_COMMIT_REF_NAME} --tag ${DOCKER_IMAGE_DEV}:latest

docker:fedora-cgmes:
  <<: *docker-fedora-template
  variables:
    TAGS:        --tag ${DOCKER_IMAGE_DEV}:${CI_COMMIT_REF_NAME}-CGMES_2.4.15_16FEB2016
    BUILD_ARGS:  --build-arg CIM_VERSION=CGMES_2.4.15_16FEB2016

43
44
45
46
47
48
49
50
51
52
53
54
55
docker:centos:
  stage: prepare
  needs: []
  script:
    - docker build
        --file Packaging/Docker/Dockerfile.dev-centos
        --tag ${DOCKER_IMAGE_DEV}-centos:${DOCKER_TAG}
        --tag ${DOCKER_IMAGE_DEV}-centos:latest .
  tags:
    - shell
    - linux

build:linux:
56
  stage: build
57
  needs: ["docker:fedora"]
58
59
60
  script:
    - mkdir -p build
    - cd build
61
    - cmake ..
62
63
64
65
66
67
68
69
70
71
72
73
    - make -j 32
  image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}
  cache:
    paths:
      - build
    key: build-linux
  artifacts:
    paths:
      - build
  tags:
    - docker

Markus Mirz's avatar
Markus Mirz committed
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
build:linux-cgmes:
  stage: build
  needs: ["docker:fedora-cgmes"]
  script:
    - mkdir -p build
    - cd build
    - cmake -DCIM_VERSION=CGMES_2.4.15_16FEB2016 ..
    - make -j 32
  image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}-CGMES_2.4.15_16FEB2016
  cache:
    paths:
      - build
    key: build-linux-cgmes
  artifacts:
    paths:
      - build
  tags:
    - docker

Niklas Eiling's avatar
Niklas Eiling committed
93
94
95
96
97
98
build:linux-profiling:
  stage: build
  needs: ["docker:centos"]
  script:
    - mkdir -p build
    - cd build
Markus Mirz's avatar
Markus Mirz committed
99
    - cmake -DWITH_PROFILING=ON -DWITH_ASAN=ON -DWITH_CUDA=OFF -DWITH_SPDLOG_SUBMODULE=ON ..
Niklas Eiling's avatar
Niklas Eiling committed
100
101
102
103
104
105
106
107
108
109
110
111
    - make -j 32
  image: ${DOCKER_IMAGE_DEV}-centos:${DOCKER_TAG}
  cache:
    paths:
      - build
    key: build-linux-profiling
  artifacts:
    paths:
      - build
  tags:
    - docker

112
build:linux-cuda:
Markus Mirz's avatar
Markus Mirz committed
113
  stage: build
114
  needs: ["docker:centos"]
115
  script:
116
    - mkdir -p build
Markus Mirz's avatar
Markus Mirz committed
117
    - cd build
118
    - cmake -DWITH_SPDLOG_SUBMODULE=ON ..
Steffen Vogel's avatar
Steffen Vogel committed
119
    - make -j 32
120
121
122
123
124
  image: ${DOCKER_IMAGE_DEV}-centos:${DOCKER_TAG}
  cache:
    paths:
      - build
    key: build-linux-cuda
125
126
  artifacts:
    paths:
127
     - build
128
  tags:
Georg Martin Reinke's avatar
Georg Martin Reinke committed
129
130
    - docker

131
build:windows:
Steffen Vogel's avatar
Steffen Vogel committed
132
  stage: build
133
  needs: []
Steffen Vogel's avatar
Steffen Vogel committed
134
  script:
Steffen Vogel's avatar
Steffen Vogel committed
135
136
    - New-Item -ItemType Directory -Force -Path build
    - Set-Location -Path build
137
138
139
    # 64 Bit Graphviz Builds for Windows:
    # https://ci.appveyor.com/project/ellson/graphviz-pl238/
    - cmake .. -DGraphviz_ROOT="C:/Program Files/Graphviz 2.41.20190716.1839"
140
    - cmake --build . -j 24
Steffen Vogel's avatar
Steffen Vogel committed
141
142
143
144
145
146
147
  tags:
    - win10
  cache:
    paths:
      - build
    key: build-win10

Markus Mirz's avatar
Markus Mirz committed
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
build:docker:
  stage: build
  needs: []
  script:
    - docker build
        --file Packaging/Docker/Dockerfile
        --build-arg DOCKER_TAG=${DOCKER_TAG}
        --build-arg CI=1
        --build-arg CI_COMMIT_SHA=${CI_COMMIT_SHA}
        --build-arg CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME}
        --build-arg CI_COMMIT_TAG=${CI_COMMIT_TAG}
        --tag ${DOCKER_IMAGE}:${DOCKER_TAG}
        --tag ${DOCKER_IMAGE}:latest .
  tags:
    - shell
    - linux
  only:
    - tags

Markus Mirz's avatar
Markus Mirz committed
167
168
169
170
171
172
173
#build:macos:
#  stage: build
#  script:
#    - mkdir -p build
#    - cd build
#    - cmake -DCMAKE_C_COMPILER=/usr/local/bin/gcc-9
#            -DCMAKE_CXX_COMPILER=/usr/local/bin/g++-9
Markus Mirz's avatar
Markus Mirz committed
174
#            -DWITH_CIM_SUBMODULE=ON
Markus Mirz's avatar
Markus Mirz committed
175
176
177
178
179
180
181
182
183
184
185
#            -DWITH_SUNDIALS=OFF ..
#    - make -j $(sysctl -n hw.ncpu)
#  cache:
#    paths:
#      - build
#    key: build-macos
#  artifacts:
#    paths:
#      - build
#  tags:
#    - mac
Steffen Vogel's avatar
Steffen Vogel committed
186

Markus Mirz's avatar
Markus Mirz committed
187
188
189
190
191
192
193
194
195
196
197
198
199
#test:pytest:
#  stage: test
#  variables:
#    PYTHONPATH: "build/Source/Python:Source/Python"
#  script:
#    - pytest -v Examples/CIM/test_CIM.yml
#    - pytest -v Examples/Cxx/test_Circuits.yml
#    - pytest -v Examples/Cxx/test_RealTime.yml
#    #- pytest -v Examples/Cxx/test_Shmem.yml
#    - pytest -v Examples/Python/Base
#    - pytest -v Examples/Python/Circuits
#    - pytest -v Examples/Python/RealTime
#    #- pytest -v Examples/Python/Shmem
Markus Mirz's avatar
Markus Mirz committed
200
#  image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}
Markus Mirz's avatar
Markus Mirz committed
201
202
203
204
#  dependencies:
#    - build:linux
#  tags:
#    - docker
Steffen Vogel's avatar
Steffen Vogel committed
205

206
generate:docs:
Markus Mirz's avatar
fix ci    
Markus Mirz committed
207
  stage: generate
208
  needs: ["build:linux"]
Georg Martin Reinke's avatar
Georg Martin Reinke committed
209
  script:
Steffen Vogel's avatar
Steffen Vogel committed
210
211
    - make -j$(nproc) -C build docs
    - make -j$(nproc) -C build docs_cxx
Markus Mirz's avatar
Markus Mirz committed
212
  image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}
Steffen Vogel's avatar
Steffen Vogel committed
213
  dependencies:
Steffen Vogel's avatar
fix CI    
Steffen Vogel committed
214
    - build:linux
Georg Martin Reinke's avatar
Georg Martin Reinke committed
215
216
  artifacts:
    paths:
Markus Mirz's avatar
Markus Mirz committed
217
218
      - build/docs/sphinx/html
      - build/docs/doxygen/html
Georg Martin Reinke's avatar
Georg Martin Reinke committed
219
220
  tags:
    - docker
221

222
generate:packages:
Markus Mirz's avatar
fix ci    
Markus Mirz committed
223
  stage: generate
224
  needs: ["build:linux"]
Steffen Vogel's avatar
Steffen Vogel committed
225
  script:
Steffen Vogel's avatar
Steffen Vogel committed
226
    - make -j$(nproc) -C build package
Steffen Vogel's avatar
Steffen Vogel committed
227
228
229
230
  only:
    - tags
  tags:
    - docker
Markus Mirz's avatar
Markus Mirz committed
231
  image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}
Steffen Vogel's avatar
Steffen Vogel committed
232
  dependencies:
Steffen Vogel's avatar
fix CI    
Steffen Vogel committed
233
    - build:linux
Steffen Vogel's avatar
Steffen Vogel committed
234
235
236
237
238
  artifacts:
    paths:
      - build/*.rpm
      - build/*.tar.gz

239
test:jupyter:
240
  stage: test
241
  needs: ["build:linux"]
242
243
244
245
246
247
  variables:
    PYTHONPATH: "build/Source/Python:Source/Python"
  script:
    - pytest -v Examples/Notebooks
  dependencies:
    - build:linux
Markus Mirz's avatar
Markus Mirz committed
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
  image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}
  tags:
    - docker
  artifacts:
    paths:
      - outputs/Examples/Notebooks/

test:jupyter-cgmes:
  stage: test
  needs: ["build:linux-cgmes"]
  variables:
    PYTHONPATH: "build/Source/Python:Source/Python"
  script:
    - pytest -v Examples/Notebooks
  dependencies:
    - build:linux-cgmes
  image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}-CGMES_2.4.15_16FEB2016
265
266
267
268
269
  tags:
    - docker
  artifacts:
    paths:
      - outputs/Examples/Notebooks/
Steffen Vogel's avatar
Steffen Vogel committed
270

Niklas Eiling's avatar
Niklas Eiling committed
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
test:cppcheck 1/2:
  stage: test
  needs: ["docker:centos"]
  script:
    - cppcheck --max-configs=32 -j 32 --error-exitcode=1 -q --std=c++11 -I Include/ -I models/Include/ Source/ Examples/ models/Source/ | tee cppcheck-error.log
  image: ${DOCKER_IMAGE_DEV}-centos:${DOCKER_TAG}
  dependencies:
    - docker:centos
  tags:
    - docker
  artifacts:
    when: on_failure
    paths:
      - cppcheck-error.log
    expose_as: 'cppcheck-error'

test:cppcheck 2/2:
  stage: test
  needs: ["docker:centos"]
  allow_failure: true
  script:
    - cppcheck --max-configs=32 -j 32 --error-exitcode=1 -q --enable=warning,performance,portability,information,missingInclude --std=c++11 -I Include/ -I models/Include/ Source/ Examples/ models/Source/ | tee cppcheck-warn.log
  image: ${DOCKER_IMAGE_DEV}-centos:${DOCKER_TAG}
  dependencies:
    - docker:centos
  tags:
    - docker
  artifacts:
    when: on_failure
    paths:
      - cppcheck-warn.log
    expose_as: 'cppcheck-warn'

.test-examples:
  stage: test
  needs: ["build:linux-profiling"]
  script:
    - $TEST_BINARY_PATH/$TEST_BINARY $TEST_PARAM 2>&1 | tee output.log
    - cat output.log |
      sed -n -E -e 's/^.*Average step time. ([0-9]+\.[0-9]+)$/step-time \1/p' |
      tee -a metrics.txt
    - gprof $TEST_BINARY_PATH/$TEST_BINARY | gprof2dot -s | dot -Tpng -o profiling.png || true
  image: ${DOCKER_IMAGE_DEV}-centos:${DOCKER_TAG}
  dependencies:
    - build:linux-profiling
  tags:
    - docker
  artifacts:
    reports:
        metrics: metrics.txt
    paths:
      - profiling.png
    expose_as: 'test-examples'

test:examples 1/1:
    extends: .test-examples
    variables:
# Name of the test binary
      TEST_BINARY: 'WSCC_9bus_mult_decoupled'
# Path where the test binary is located
      TEST_BINARY_PATH: 'build/Examples/Cxx'
# Command line parameters for the test binary
      TEST_PARAM: ''

test:examples 2/2:
    extends: .test-examples
    variables:
# Name of the test binary
      TEST_BINARY: 'WSCC_9bus_mult_coupled'
# Path where the test binary is located
      TEST_BINARY_PATH: 'build/Examples/Cxx'
# Command line parameters for the test binary
      TEST_PARAM: ''

345
346
.remote-gpu:
  stage: test
347
  needs: ["build:linux-cuda"]
348
349
350
351
352
353
  variables:
    GPU_TARGET: 'gitlab-runner@ghost'
    RDIR: '/tmp/gitlab-jobs/$CI_PROJECT_NAME/$CI_JOB_ID'
    CI_DIR: '$CI_BUILDS_DIR/$CI_PROJECT_PATH'
    LDIR: '/cricket/bin'
    TEST_FILES: ''
354
355
356
357
358
359
360
361
362
363
364
365
366
    TEST_PARAM: ''
    REMOTE_LIBS: 'linux-vdso.so.1
                  librt.so.1
                  libdl.so.2
                  libcudart.so.10.2
                  libcusolver.so.10
                  libstdc\+\+.so.6
                  libm.so.6
                  libgcc_s.so.1
                  libc.so.6
                  /lib64/ld-linux-x86-64.so.2
                  libutil.so.1
                  libpthread.so.0'
367
  script:
368
369
370
371
372
373
    - mkdir ~/.ssh &&
      echo "-----BEGIN OPENSSH PRIVATE KEY-----" > ~/.ssh/id_rsa &&
      echo $SSH_PRIVATE | sed "s/\S\{64\}/&\n/g" >> ~/.ssh/id_rsa &&
      echo "-----END OPENSSH PRIVATE KEY-----" >> ~/.ssh/id_rsa &&
      echo $SSH_PUBLIC > ~/.ssh/id_rsa.pub &&
      echo $KNOWN_HOSTS > ~/.ssh/known_hosts && chmod 600 ~/.ssh/id_rsa
374
    - echo "$TEST_BINARY_PATH/$TEST_BINARY $TEST_FILES" | tr -d '\n' | xargs -r -d " " -I % cp % $LDIR
375
376
377
378
    - ldd $LDIR/$TEST_BINARY |
      sed -n -E -e 's/^\s+\S+ => (\S+) \S+$/\1/p' |
      grep -Ev "$(echo $REMOTE_LIBS | tr ' ' '|')|not" |
      xargs -I % cp % $LDIR/
379
380
    - ssh $GPU_TARGET mkdir -p $RDIR
    - scp -r $LDIR/* $GPU_TARGET:$RDIR/
381
    - ssh $GPU_TARGET "LD_LIBRARY_PATH=$RDIR:${LD_LIBRARY_PATH} LD_PRELOAD=$RDIR/cricket-server.so $RDIR/$TEST_BINARY" &
382
    - sleep 2
383
    - PATH=$LDIR:${PATH} LD_PRELOAD=$LDIR/cricket-client.so $TEST_BINARY_PATH/$TEST_BINARY $TEST_PARAM
384
  after_script:
385
    - ssh $GPU_TARGET rm -rf $RDIR
386
    - ssh $GPU_TARGET pkill -fe -2 $RDIR/$TEST_BINARY
387
  image: ${DOCKER_IMAGE_DEV}-centos:${DOCKER_TAG}
388
389
390
391
392
  dependencies:
    - build:linux-cuda
  tags:
    - docker

Niklas Eiling's avatar
Niklas Eiling committed
393
test:cuda 1/2:
394
    extends: .remote-gpu
395
    allow_failure: true
396
    variables:
397
# Name of the test binary
398
      TEST_BINARY: 'WSCC_9bus_mult_decoupled'
399
# Path where the test binary is located
400
      TEST_BINARY_PATH: 'build/Examples/Cxx'
401
402
403
404
# Additional files that are necessary to run the application on the GPU node
      TEST_FILE: ''
# Command line parameters for the test binary
      TEST_PARAM: ''
405

Niklas Eiling's avatar
Niklas Eiling committed
406
407
408
409
410
411
412
413
414
415
416
417
418
test:cuda 2/2:
    extends: .remote-gpu
    allow_failure: true
    variables:
# Name of the test binary
      TEST_BINARY: 'WSCC_9bus_mult_coupled'
# Path where the test binary is located
      TEST_BINARY_PATH: 'build/Examples/Cxx'
# Additional files that are necessary to run the application on the GPU node
      TEST_FILE: ''
# Command line parameters for the test binary
      TEST_PARAM: ''

Markus Mirz's avatar
Markus Mirz committed
419
420
# Stage deploy
##############################################################################
421
pages:
Markus Mirz's avatar
Markus Mirz committed
422
  stage: deploy
423
  needs: ["generate:docs"]
Markus Mirz's avatar
Markus Mirz committed
424
  script:
Markus Mirz's avatar
Markus Mirz committed
425
426
    - cp -r build/docs/sphinx/html/. public/sphinx
    - cp -r build/docs/doxygen/html/. public/doxygen
Markus Mirz's avatar
Markus Mirz committed
427
428
429
430
431
432
  artifacts:
    paths:
      - public
  dependencies:
    - generate:docs
  only:
433
    - deploy
Markus Mirz's avatar
Markus Mirz committed
434
435
436
437
  tags:
    - shell
    - linux

438
deploy:docker:
439
440
  stage: deploy
  script:
Markus Mirz's avatar
Markus Mirz committed
441
    - docker push ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}
442
    - docker push ${DOCKER_IMAGE_DEV}:latest
443
    - docker push ${DOCKER_IMAGE}:${DOCKER_TAG}
444
    - docker push ${DOCKER_IMAGE}:latest
445
446
447
448
  dependencies:
    - build:docker
  tags:
    - shell
Steffen Vogel's avatar
Steffen Vogel committed
449
    - linux
Steffen Vogel's avatar
Steffen Vogel committed
450
451
  only:
    - tags
452

453
deploy:packages:
Steffen Vogel's avatar
Steffen Vogel committed
454
455
  stage: deploy
  script:
Markus Mirz's avatar
Markus Mirz committed
456
    - rsync ${RSYNC_OPTS} build/*.rpm ${DEPLOY_USER}@${DEPLOY_HOST}:/var/www/packages/fedora/29/x86_64
Steffen Vogel's avatar
Steffen Vogel committed
457
    - rsync ${RSYNC_OPTS} build/*.tar.gz ${DEPLOY_USER}@${DEPLOY_HOST}:/var/www/dpsim/src
Markus Mirz's avatar
Markus Mirz committed
458
    - ssh ${DEPLOY_USER}@${DEPLOY_HOST} createrepo /var/www/packages/fedora/29
Steffen Vogel's avatar
Steffen Vogel committed
459
  dependencies:
Markus Mirz's avatar
fix ci    
Markus Mirz committed
460
    - generate:packages
Steffen Vogel's avatar
Steffen Vogel committed
461
462
463
464
465
  only:
    - tags
  tags:
    - fein-deploy
    - shell
466
    - linux