.gitlab-ci.yml 10.8 KB
Newer Older
1
variables:
2
  RSYNC_OPTS: --recursive --ignore-missing-args --chown ${DEPLOY_USER}:${DEPLOY_USER}
3
4
5
  DOCKER_TAG: ${CI_COMMIT_REF_NAME}
  DOCKER_IMAGE: rwthacs/dpsim
  DOCKER_IMAGE_DEV: rwthacs/dpsim-dev
6
7

before_script:
Steffen Vogel's avatar
fix CI    
Steffen Vogel committed
8
  - git config --local core.longpaths true
9
  - git submodule sync --recursive
10
  - git submodule update --init --recursive
11

12
13
14
stages:
  - prepare
  - build
Steffen Vogel's avatar
Steffen Vogel committed
15
  - test
Markus Mirz's avatar
fix ci    
Markus Mirz committed
16
  - generate
Markus Mirz's avatar
Markus Mirz committed
17
  - deploy
18

19
docker:fedora:
20
  stage: prepare
21
  needs: []
22
  script:
23
    - docker build
24
        --file Packaging/Docker/Dockerfile.dev
Markus Mirz's avatar
Markus Mirz committed
25
        --tag ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}
26
        --tag ${DOCKER_IMAGE_DEV}:latest .
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
  tags:
    - shell
    - linux

docker:centos:
  stage: prepare
  needs: []
  script:
    - docker build
        --file Packaging/Docker/Dockerfile.dev-centos
        --tag ${DOCKER_IMAGE_DEV}-centos:${DOCKER_TAG}
        --tag ${DOCKER_IMAGE_DEV}-centos:latest .
  tags:
    - shell
    - linux

43
44
45
46
47
48
49
50
51
52
53
54
55
#docker:jupyter:
#  stage: prepare
#  needs: []
#  script:
#    - docker build
#       --file Packaging/Docker/Dockerfile.jupyter
#       --build-arg DOCKER_IMAGE=${DOCKER_IMAGE_DEV}
#       --build-arg DOCKER_TAG=${DOCKER_TAG}
#       --tag ${DOCKER_IMAGE_DEV}-jupyter:${DOCKER_TAG}
#       --tag ${DOCKER_IMAGE_DEV}-jupyter:latest .
#  tags:
#    - shell
#    - linux
56

57
build:linux:
58
  stage: build
59
  needs: ["docker:fedora"]
60
61
62
  script:
    - mkdir -p build
    - cd build
63
    - cmake ..
64
65
66
67
68
69
70
71
72
73
74
75
    - make -j 32
  image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}
  cache:
    paths:
      - build
    key: build-linux
  artifacts:
    paths:
      - build
  tags:
    - docker

Niklas Eiling's avatar
Niklas Eiling committed
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
build:linux-profiling:
  stage: build
  needs: ["docker:centos"]
  script:
    - mkdir -p build
    - cd build
    - cmake  -DWITH_PROFILING=ON -DWITH_ASAN=ON -DWITH_CUDA=OFF -DWITH_SPDLOG_SUBMODULE=ON ..
    - make -j 32
  image: ${DOCKER_IMAGE_DEV}-centos:${DOCKER_TAG}
  cache:
    paths:
      - build
    key: build-linux-profiling
  artifacts:
    paths:
      - build
  tags:
    - docker

95
build:linux-cuda:
Markus Mirz's avatar
Markus Mirz committed
96
  stage: build
97
  needs: ["docker:centos"]
98
  script:
99
    - mkdir -p build
Markus Mirz's avatar
Markus Mirz committed
100
    - cd build
101
    - cmake -DWITH_SPDLOG_SUBMODULE=ON ..
Steffen Vogel's avatar
Steffen Vogel committed
102
    - make -j 32
103
104
105
106
107
  image: ${DOCKER_IMAGE_DEV}-centos:${DOCKER_TAG}
  cache:
    paths:
      - build
    key: build-linux-cuda
108
109
  artifacts:
    paths:
110
     - build
111
  tags:
Georg Martin Reinke's avatar
Georg Martin Reinke committed
112
113
    - docker

114
build:windows:
Steffen Vogel's avatar
Steffen Vogel committed
115
  stage: build
116
  needs: []
Steffen Vogel's avatar
Steffen Vogel committed
117
  script:
Steffen Vogel's avatar
Steffen Vogel committed
118
119
    - New-Item -ItemType Directory -Force -Path build
    - Set-Location -Path build
120
121
122
    # 64 Bit Graphviz Builds for Windows:
    # https://ci.appveyor.com/project/ellson/graphviz-pl238/
    - cmake .. -DGraphviz_ROOT="C:/Program Files/Graphviz 2.41.20190716.1839"
123
    - cmake --build . -j 24
Steffen Vogel's avatar
Steffen Vogel committed
124
125
126
127
128
129
130
  tags:
    - win10
  cache:
    paths:
      - build
    key: build-win10

Markus Mirz's avatar
Markus Mirz committed
131
132
133
134
135
136
137
#build:macos:
#  stage: build
#  script:
#    - mkdir -p build
#    - cd build
#    - cmake -DCMAKE_C_COMPILER=/usr/local/bin/gcc-9
#            -DCMAKE_CXX_COMPILER=/usr/local/bin/g++-9
Markus Mirz's avatar
Markus Mirz committed
138
#            -DWITH_CIM_SUBMODULE=ON
Markus Mirz's avatar
Markus Mirz committed
139
140
141
142
143
144
145
146
147
148
149
#            -DWITH_SUNDIALS=OFF ..
#    - make -j $(sysctl -n hw.ncpu)
#  cache:
#    paths:
#      - build
#    key: build-macos
#  artifacts:
#    paths:
#      - build
#  tags:
#    - mac
Steffen Vogel's avatar
Steffen Vogel committed
150

Markus Mirz's avatar
Markus Mirz committed
151
152
153
154
155
156
157
158
159
160
161
162
163
#test:pytest:
#  stage: test
#  variables:
#    PYTHONPATH: "build/Source/Python:Source/Python"
#  script:
#    - pytest -v Examples/CIM/test_CIM.yml
#    - pytest -v Examples/Cxx/test_Circuits.yml
#    - pytest -v Examples/Cxx/test_RealTime.yml
#    #- pytest -v Examples/Cxx/test_Shmem.yml
#    - pytest -v Examples/Python/Base
#    - pytest -v Examples/Python/Circuits
#    - pytest -v Examples/Python/RealTime
#    #- pytest -v Examples/Python/Shmem
Markus Mirz's avatar
Markus Mirz committed
164
#  image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}
Markus Mirz's avatar
Markus Mirz committed
165
166
167
168
#  dependencies:
#    - build:linux
#  tags:
#    - docker
Steffen Vogel's avatar
Steffen Vogel committed
169

170
generate:docs:
Markus Mirz's avatar
fix ci    
Markus Mirz committed
171
  stage: generate
172
  needs: ["build:linux"]
Georg Martin Reinke's avatar
Georg Martin Reinke committed
173
  script:
Steffen Vogel's avatar
Steffen Vogel committed
174
175
    - make -j$(nproc) -C build docs
    - make -j$(nproc) -C build docs_cxx
Markus Mirz's avatar
Markus Mirz committed
176
  image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}
Steffen Vogel's avatar
Steffen Vogel committed
177
  dependencies:
Steffen Vogel's avatar
fix CI    
Steffen Vogel committed
178
    - build:linux
Georg Martin Reinke's avatar
Georg Martin Reinke committed
179
180
  artifacts:
    paths:
Markus Mirz's avatar
Markus Mirz committed
181
182
      - build/docs/sphinx/html
      - build/docs/doxygen/html
Georg Martin Reinke's avatar
Georg Martin Reinke committed
183
184
  tags:
    - docker
185

186
generate:packages:
Markus Mirz's avatar
fix ci    
Markus Mirz committed
187
  stage: generate
188
  needs: ["build:linux"]
Steffen Vogel's avatar
Steffen Vogel committed
189
  script:
Steffen Vogel's avatar
Steffen Vogel committed
190
    - make -j$(nproc) -C build package
Steffen Vogel's avatar
Steffen Vogel committed
191
192
193
194
  only:
    - tags
  tags:
    - docker
Markus Mirz's avatar
Markus Mirz committed
195
  image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}
Steffen Vogel's avatar
Steffen Vogel committed
196
  dependencies:
Steffen Vogel's avatar
fix CI    
Steffen Vogel committed
197
    - build:linux
Steffen Vogel's avatar
Steffen Vogel committed
198
199
200
201
202
  artifacts:
    paths:
      - build/*.rpm
      - build/*.tar.gz

203
204
build:docker:
  stage: build
205
  needs: []
206
  script:
Markus Mirz's avatar
Markus Mirz committed
207
    - docker build
208
209
        --file Packaging/Docker/Dockerfile
        --build-arg DOCKER_TAG=${DOCKER_TAG}
210
211
212
213
        --build-arg CI=1
        --build-arg CI_COMMIT_SHA=${CI_COMMIT_SHA}
        --build-arg CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME}
        --build-arg CI_COMMIT_TAG=${CI_COMMIT_TAG}
214
215
        --tag ${DOCKER_IMAGE}:${DOCKER_TAG}
        --tag ${DOCKER_IMAGE}:latest .
216
217
  tags:
    - shell
Steffen Vogel's avatar
Steffen Vogel committed
218
    - linux
Markus Mirz's avatar
Markus Mirz committed
219
220
  only:
    - tags
221

222
test:jupyter:
223
  stage: test
224
  needs: ["build:linux"]
225
226
227
228
  variables:
    PYTHONPATH: "build/Source/Python:Source/Python"
  script:
    - pytest -v Examples/Notebooks
Markus Mirz's avatar
Markus Mirz committed
229
  image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}
230
231
232
233
234
235
236
  dependencies:
    - build:linux
  tags:
    - docker
  artifacts:
    paths:
      - outputs/Examples/Notebooks/
Steffen Vogel's avatar
Steffen Vogel committed
237

Niklas Eiling's avatar
Niklas Eiling committed
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
test:cppcheck 1/2:
  stage: test
  needs: ["docker:centos"]
  script:
    - cppcheck --max-configs=32 -j 32 --error-exitcode=1 -q --std=c++11 -I Include/ -I models/Include/ Source/ Examples/ models/Source/ | tee cppcheck-error.log
  image: ${DOCKER_IMAGE_DEV}-centos:${DOCKER_TAG}
  dependencies:
    - docker:centos
  tags:
    - docker
  artifacts:
    when: on_failure
    paths:
      - cppcheck-error.log
    expose_as: 'cppcheck-error'

test:cppcheck 2/2:
  stage: test
  needs: ["docker:centos"]
  allow_failure: true
  script:
    - cppcheck --max-configs=32 -j 32 --error-exitcode=1 -q --enable=warning,performance,portability,information,missingInclude --std=c++11 -I Include/ -I models/Include/ Source/ Examples/ models/Source/ | tee cppcheck-warn.log
  image: ${DOCKER_IMAGE_DEV}-centos:${DOCKER_TAG}
  dependencies:
    - docker:centos
  tags:
    - docker
  artifacts:
    when: on_failure
    paths:
      - cppcheck-warn.log
    expose_as: 'cppcheck-warn'

.test-examples:
  stage: test
  needs: ["build:linux-profiling"]
  script:
    - $TEST_BINARY_PATH/$TEST_BINARY $TEST_PARAM 2>&1 | tee output.log
    - cat output.log |
      sed -n -E -e 's/^.*Average step time. ([0-9]+\.[0-9]+)$/step-time \1/p' |
      tee -a metrics.txt
    - gprof $TEST_BINARY_PATH/$TEST_BINARY | gprof2dot -s | dot -Tpng -o profiling.png || true
  image: ${DOCKER_IMAGE_DEV}-centos:${DOCKER_TAG}
  dependencies:
    - build:linux-profiling
  tags:
    - docker
  artifacts:
    reports:
        metrics: metrics.txt
    paths:
      - profiling.png
    expose_as: 'test-examples'

test:examples 1/1:
    extends: .test-examples
    variables:
# Name of the test binary
      TEST_BINARY: 'WSCC_9bus_mult_decoupled'
# Path where the test binary is located
      TEST_BINARY_PATH: 'build/Examples/Cxx'
# Command line parameters for the test binary
      TEST_PARAM: ''

test:examples 2/2:
    extends: .test-examples
    variables:
# Name of the test binary
      TEST_BINARY: 'WSCC_9bus_mult_coupled'
# Path where the test binary is located
      TEST_BINARY_PATH: 'build/Examples/Cxx'
# Command line parameters for the test binary
      TEST_PARAM: ''

312
313
.remote-gpu:
  stage: test
314
  needs: ["build:linux-cuda"]
315
316
317
318
319
320
  variables:
    GPU_TARGET: 'gitlab-runner@ghost'
    RDIR: '/tmp/gitlab-jobs/$CI_PROJECT_NAME/$CI_JOB_ID'
    CI_DIR: '$CI_BUILDS_DIR/$CI_PROJECT_PATH'
    LDIR: '/cricket/bin'
    TEST_FILES: ''
321
322
323
324
325
326
327
328
329
330
331
332
333
    TEST_PARAM: ''
    REMOTE_LIBS: 'linux-vdso.so.1
                  librt.so.1
                  libdl.so.2
                  libcudart.so.10.2
                  libcusolver.so.10
                  libstdc\+\+.so.6
                  libm.so.6
                  libgcc_s.so.1
                  libc.so.6
                  /lib64/ld-linux-x86-64.so.2
                  libutil.so.1
                  libpthread.so.0'
334
  script:
335
336
337
338
339
340
    - mkdir ~/.ssh &&
      echo "-----BEGIN OPENSSH PRIVATE KEY-----" > ~/.ssh/id_rsa &&
      echo $SSH_PRIVATE | sed "s/\S\{64\}/&\n/g" >> ~/.ssh/id_rsa &&
      echo "-----END OPENSSH PRIVATE KEY-----" >> ~/.ssh/id_rsa &&
      echo $SSH_PUBLIC > ~/.ssh/id_rsa.pub &&
      echo $KNOWN_HOSTS > ~/.ssh/known_hosts && chmod 600 ~/.ssh/id_rsa
341
    - echo "$TEST_BINARY_PATH/$TEST_BINARY $TEST_FILES" | tr -d '\n' | xargs -r -d " " -I % cp % $LDIR
342
343
344
345
    - ldd $LDIR/$TEST_BINARY |
      sed -n -E -e 's/^\s+\S+ => (\S+) \S+$/\1/p' |
      grep -Ev "$(echo $REMOTE_LIBS | tr ' ' '|')|not" |
      xargs -I % cp % $LDIR/
346
347
    - ssh $GPU_TARGET mkdir -p $RDIR
    - scp -r $LDIR/* $GPU_TARGET:$RDIR/
348
    - ssh $GPU_TARGET "LD_LIBRARY_PATH=$RDIR:${LD_LIBRARY_PATH} LD_PRELOAD=$RDIR/cricket-server.so $RDIR/$TEST_BINARY" &
349
    - sleep 2
350
    - PATH=$LDIR:${PATH} LD_PRELOAD=$LDIR/cricket-client.so $TEST_BINARY_PATH/$TEST_BINARY $TEST_PARAM
351
  after_script:
352
    - ssh $GPU_TARGET rm -rf $RDIR
353
    - ssh $GPU_TARGET pkill -fe -2 $RDIR/$TEST_BINARY
354
  image: ${DOCKER_IMAGE_DEV}-centos:${DOCKER_TAG}
355
356
357
358
359
  dependencies:
    - build:linux-cuda
  tags:
    - docker

Niklas Eiling's avatar
Niklas Eiling committed
360
test:cuda 1/2:
361
    extends: .remote-gpu
362
    allow_failure: true
363
    variables:
364
# Name of the test binary
365
      TEST_BINARY: 'WSCC_9bus_mult_decoupled'
366
# Path where the test binary is located
367
      TEST_BINARY_PATH: 'build/Examples/Cxx'
368
369
370
371
# Additional files that are necessary to run the application on the GPU node
      TEST_FILE: ''
# Command line parameters for the test binary
      TEST_PARAM: ''
372

Niklas Eiling's avatar
Niklas Eiling committed
373
374
375
376
377
378
379
380
381
382
383
384
385
test:cuda 2/2:
    extends: .remote-gpu
    allow_failure: true
    variables:
# Name of the test binary
      TEST_BINARY: 'WSCC_9bus_mult_coupled'
# Path where the test binary is located
      TEST_BINARY_PATH: 'build/Examples/Cxx'
# Additional files that are necessary to run the application on the GPU node
      TEST_FILE: ''
# Command line parameters for the test binary
      TEST_PARAM: ''

Markus Mirz's avatar
Markus Mirz committed
386
387
# Stage deploy
##############################################################################
388
pages:
Markus Mirz's avatar
Markus Mirz committed
389
  stage: deploy
390
  needs: ["generate:docs"]
Markus Mirz's avatar
Markus Mirz committed
391
  script:
Markus Mirz's avatar
Markus Mirz committed
392
393
    - cp -r build/docs/sphinx/html/. public/sphinx
    - cp -r build/docs/doxygen/html/. public/doxygen
Markus Mirz's avatar
Markus Mirz committed
394
395
396
397
398
399
  artifacts:
    paths:
      - public
  dependencies:
    - generate:docs
  only:
400
    - deploy
Markus Mirz's avatar
Markus Mirz committed
401
402
403
404
  tags:
    - shell
    - linux

405
deploy:docker:
406
407
  stage: deploy
  script:
Markus Mirz's avatar
Markus Mirz committed
408
    - docker push ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}
409
    - docker push ${DOCKER_IMAGE_DEV}:latest
410
    - docker push ${DOCKER_IMAGE}:${DOCKER_TAG}
411
    - docker push ${DOCKER_IMAGE}:latest
412
413
414
415
  dependencies:
    - build:docker
  tags:
    - shell
Steffen Vogel's avatar
Steffen Vogel committed
416
    - linux
Steffen Vogel's avatar
Steffen Vogel committed
417
418
  only:
    - tags
419

420
deploy:packages:
Steffen Vogel's avatar
Steffen Vogel committed
421
422
  stage: deploy
  script:
Markus Mirz's avatar
Markus Mirz committed
423
    - rsync ${RSYNC_OPTS} build/*.rpm ${DEPLOY_USER}@${DEPLOY_HOST}:/var/www/packages/fedora/29/x86_64
Steffen Vogel's avatar
Steffen Vogel committed
424
    - rsync ${RSYNC_OPTS} build/*.tar.gz ${DEPLOY_USER}@${DEPLOY_HOST}:/var/www/dpsim/src
Markus Mirz's avatar
Markus Mirz committed
425
    - ssh ${DEPLOY_USER}@${DEPLOY_HOST} createrepo /var/www/packages/fedora/29
Steffen Vogel's avatar
Steffen Vogel committed
426
  dependencies:
Markus Mirz's avatar
fix ci    
Markus Mirz committed
427
    - generate:packages
Steffen Vogel's avatar
Steffen Vogel committed
428
429
430
431
432
  only:
    - tags
  tags:
    - fein-deploy
    - shell
433
    - linux