.gitlab-ci.yml 7.9 KB
Newer Older
1
variables:
2
  RSYNC_OPTS: --recursive --ignore-missing-args --chown ${DEPLOY_USER}:${DEPLOY_USER}
3
4
5
  DOCKER_TAG: ${CI_COMMIT_REF_NAME}
  DOCKER_IMAGE: rwthacs/dpsim
  DOCKER_IMAGE_DEV: rwthacs/dpsim-dev
6
7

before_script:
Steffen Vogel's avatar
fix CI    
Steffen Vogel committed
8
  - git config --local core.longpaths true
9
  - git submodule sync --recursive
10
  - git submodule update --init --recursive
11

12
13
14
stages:
  - prepare
  - build
Steffen Vogel's avatar
Steffen Vogel committed
15
  - test
Markus Mirz's avatar
fix ci    
Markus Mirz committed
16
  - generate
Markus Mirz's avatar
Markus Mirz committed
17
  - deploy
18

19
docker:fedora:
20
  stage: prepare
21
  needs: []
22
  script:
23
    - docker build
24
        --file Packaging/Docker/Dockerfile.dev
Markus Mirz's avatar
Markus Mirz committed
25
        --tag ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}
26
        --tag ${DOCKER_IMAGE_DEV}:latest .
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
  tags:
    - shell
    - linux

docker:centos:
  stage: prepare
  needs: []
  script:
    - docker build
        --file Packaging/Docker/Dockerfile.dev-centos
        --tag ${DOCKER_IMAGE_DEV}-centos:${DOCKER_TAG}
        --tag ${DOCKER_IMAGE_DEV}-centos:latest .
  tags:
    - shell
    - linux

43
44
45
46
47
48
49
50
51
52
53
54
55
#docker:jupyter:
#  stage: prepare
#  needs: []
#  script:
#    - docker build
#       --file Packaging/Docker/Dockerfile.jupyter
#       --build-arg DOCKER_IMAGE=${DOCKER_IMAGE_DEV}
#       --build-arg DOCKER_TAG=${DOCKER_TAG}
#       --tag ${DOCKER_IMAGE_DEV}-jupyter:${DOCKER_TAG}
#       --tag ${DOCKER_IMAGE_DEV}-jupyter:latest .
#  tags:
#    - shell
#    - linux
56

57
build:linux:
58
  stage: build
59
  needs: ["docker:fedora"]
60
61
62
  script:
    - mkdir -p build
    - cd build
63
    - cmake ..
64
65
66
67
68
69
70
71
72
73
74
75
76
    - make -j 32
  image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}
  cache:
    paths:
      - build
    key: build-linux
  artifacts:
    paths:
      - build
  tags:
    - docker

build:linux-cuda:
Markus Mirz's avatar
Markus Mirz committed
77
  stage: build
78
  needs: ["docker:centos"]
79
  script:
80
    - mkdir -p build
Markus Mirz's avatar
Markus Mirz committed
81
    - cd build
82
    - cmake -DWITH_SPDLOG_SUBMODULE=ON ..
Steffen Vogel's avatar
Steffen Vogel committed
83
    - make -j 32
84
85
86
87
88
  image: ${DOCKER_IMAGE_DEV}-centos:${DOCKER_TAG}
  cache:
    paths:
      - build
    key: build-linux-cuda
89
90
  artifacts:
    paths:
91
     - build
92
  tags:
Georg Martin Reinke's avatar
Georg Martin Reinke committed
93
94
    - docker

95
build:windows:
Steffen Vogel's avatar
Steffen Vogel committed
96
  stage: build
97
  needs: []
Steffen Vogel's avatar
Steffen Vogel committed
98
  script:
Steffen Vogel's avatar
Steffen Vogel committed
99
100
    - New-Item -ItemType Directory -Force -Path build
    - Set-Location -Path build
101
102
103
    # 64 Bit Graphviz Builds for Windows:
    # https://ci.appveyor.com/project/ellson/graphviz-pl238/
    - cmake .. -DGraphviz_ROOT="C:/Program Files/Graphviz 2.41.20190716.1839"
104
    - cmake --build . -j 24
Steffen Vogel's avatar
Steffen Vogel committed
105
106
107
108
109
110
111
  tags:
    - win10
  cache:
    paths:
      - build
    key: build-win10

Markus Mirz's avatar
Markus Mirz committed
112
113
114
115
116
117
118
#build:macos:
#  stage: build
#  script:
#    - mkdir -p build
#    - cd build
#    - cmake -DCMAKE_C_COMPILER=/usr/local/bin/gcc-9
#            -DCMAKE_CXX_COMPILER=/usr/local/bin/g++-9
Markus Mirz's avatar
Markus Mirz committed
119
#            -DWITH_CIM_SUBMODULE=ON
Markus Mirz's avatar
Markus Mirz committed
120
121
122
123
124
125
126
127
128
129
130
#            -DWITH_SUNDIALS=OFF ..
#    - make -j $(sysctl -n hw.ncpu)
#  cache:
#    paths:
#      - build
#    key: build-macos
#  artifacts:
#    paths:
#      - build
#  tags:
#    - mac
Steffen Vogel's avatar
Steffen Vogel committed
131

Markus Mirz's avatar
Markus Mirz committed
132
133
134
135
136
137
138
139
140
141
142
143
144
#test:pytest:
#  stage: test
#  variables:
#    PYTHONPATH: "build/Source/Python:Source/Python"
#  script:
#    - pytest -v Examples/CIM/test_CIM.yml
#    - pytest -v Examples/Cxx/test_Circuits.yml
#    - pytest -v Examples/Cxx/test_RealTime.yml
#    #- pytest -v Examples/Cxx/test_Shmem.yml
#    - pytest -v Examples/Python/Base
#    - pytest -v Examples/Python/Circuits
#    - pytest -v Examples/Python/RealTime
#    #- pytest -v Examples/Python/Shmem
Markus Mirz's avatar
Markus Mirz committed
145
#  image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}
Markus Mirz's avatar
Markus Mirz committed
146
147
148
149
#  dependencies:
#    - build:linux
#  tags:
#    - docker
Steffen Vogel's avatar
Steffen Vogel committed
150

151
generate:docs:
Markus Mirz's avatar
fix ci    
Markus Mirz committed
152
  stage: generate
153
  needs: ["build:linux"]
Georg Martin Reinke's avatar
Georg Martin Reinke committed
154
  script:
Steffen Vogel's avatar
Steffen Vogel committed
155
156
    - make -j$(nproc) -C build docs
    - make -j$(nproc) -C build docs_cxx
Markus Mirz's avatar
Markus Mirz committed
157
  image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}
Steffen Vogel's avatar
Steffen Vogel committed
158
  dependencies:
Steffen Vogel's avatar
fix CI    
Steffen Vogel committed
159
    - build:linux
Georg Martin Reinke's avatar
Georg Martin Reinke committed
160
161
  artifacts:
    paths:
Markus Mirz's avatar
Markus Mirz committed
162
163
      - build/docs/sphinx/html
      - build/docs/doxygen/html
Georg Martin Reinke's avatar
Georg Martin Reinke committed
164
165
  tags:
    - docker
166

167
generate:packages:
Markus Mirz's avatar
fix ci    
Markus Mirz committed
168
  stage: generate
169
  needs: ["build:linux"]
Steffen Vogel's avatar
Steffen Vogel committed
170
  script:
Steffen Vogel's avatar
Steffen Vogel committed
171
    - make -j$(nproc) -C build package
Steffen Vogel's avatar
Steffen Vogel committed
172
173
174
175
  only:
    - tags
  tags:
    - docker
Markus Mirz's avatar
Markus Mirz committed
176
  image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}
Steffen Vogel's avatar
Steffen Vogel committed
177
  dependencies:
Steffen Vogel's avatar
fix CI    
Steffen Vogel committed
178
    - build:linux
Steffen Vogel's avatar
Steffen Vogel committed
179
180
181
182
183
  artifacts:
    paths:
      - build/*.rpm
      - build/*.tar.gz

184
185
build:docker:
  stage: build
186
  needs: []
187
  script:
Markus Mirz's avatar
Markus Mirz committed
188
    - docker build
189
190
        --file Packaging/Docker/Dockerfile
        --build-arg DOCKER_TAG=${DOCKER_TAG}
191
192
193
194
        --build-arg CI=1
        --build-arg CI_COMMIT_SHA=${CI_COMMIT_SHA}
        --build-arg CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME}
        --build-arg CI_COMMIT_TAG=${CI_COMMIT_TAG}
195
196
        --tag ${DOCKER_IMAGE}:${DOCKER_TAG}
        --tag ${DOCKER_IMAGE}:latest .
197
198
  tags:
    - shell
Steffen Vogel's avatar
Steffen Vogel committed
199
    - linux
Markus Mirz's avatar
Markus Mirz committed
200
201
  only:
    - tags
202

203
test:jupyter:
204
  stage: test
205
  needs: ["build:linux"]
206
207
208
209
  variables:
    PYTHONPATH: "build/Source/Python:Source/Python"
  script:
    - pytest -v Examples/Notebooks
Markus Mirz's avatar
Markus Mirz committed
210
  image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}
211
212
213
214
215
216
217
  dependencies:
    - build:linux
  tags:
    - docker
  artifacts:
    paths:
      - outputs/Examples/Notebooks/
Steffen Vogel's avatar
Steffen Vogel committed
218

219
220
.remote-gpu:
  stage: test
221
  needs: ["build:linux-cuda"]
222
223
224
225
226
227
  variables:
    GPU_TARGET: 'gitlab-runner@ghost'
    RDIR: '/tmp/gitlab-jobs/$CI_PROJECT_NAME/$CI_JOB_ID'
    CI_DIR: '$CI_BUILDS_DIR/$CI_PROJECT_PATH'
    LDIR: '/cricket/bin'
    TEST_FILES: ''
228
229
230
231
232
233
234
235
236
237
238
239
240
    TEST_PARAM: ''
    REMOTE_LIBS: 'linux-vdso.so.1
                  librt.so.1
                  libdl.so.2
                  libcudart.so.10.2
                  libcusolver.so.10
                  libstdc\+\+.so.6
                  libm.so.6
                  libgcc_s.so.1
                  libc.so.6
                  /lib64/ld-linux-x86-64.so.2
                  libutil.so.1
                  libpthread.so.0'
241
  script:
242
243
244
245
246
247
    - mkdir ~/.ssh &&
      echo "-----BEGIN OPENSSH PRIVATE KEY-----" > ~/.ssh/id_rsa &&
      echo $SSH_PRIVATE | sed "s/\S\{64\}/&\n/g" >> ~/.ssh/id_rsa &&
      echo "-----END OPENSSH PRIVATE KEY-----" >> ~/.ssh/id_rsa &&
      echo $SSH_PUBLIC > ~/.ssh/id_rsa.pub &&
      echo $KNOWN_HOSTS > ~/.ssh/known_hosts && chmod 600 ~/.ssh/id_rsa
248
    - echo "$TEST_BINARY_PATH/$TEST_BINARY $TEST_FILES" | tr -d '\n' | xargs -r -d " " -I % cp % $LDIR
249
250
251
252
    - ldd $LDIR/$TEST_BINARY |
      sed -n -E -e 's/^\s+\S+ => (\S+) \S+$/\1/p' |
      grep -Ev "$(echo $REMOTE_LIBS | tr ' ' '|')|not" |
      xargs -I % cp % $LDIR/
253
254
    - ssh $GPU_TARGET mkdir -p $RDIR
    - scp -r $LDIR/* $GPU_TARGET:$RDIR/
255
    - ssh $GPU_TARGET "LD_LIBRARY_PATH=$RDIR:${LD_LIBRARY_PATH} LD_PRELOAD=$RDIR/cricket-server.so $RDIR/$TEST_BINARY" &
256
    - sleep 2
257
    - PATH=$LDIR:${PATH} LD_PRELOAD=$LDIR/cricket-client.so $TEST_BINARY_PATH/$TEST_BINARY $TEST_PARAM
258
  after_script:
259
    - ssh $GPU_TARGET rm -rf $RDIR
260
    - ssh $GPU_TARGET pkill -fe -2 $RDIR/test_kernel
261
  image: ${DOCKER_IMAGE_DEV}-centos:${DOCKER_TAG}
262
263
264
265
266
267
268
  dependencies:
    - build:linux-cuda
  tags:
    - docker

test:cuda:
    extends: .remote-gpu
269
    allow_failure: true
270
    variables:
271
# Name of the test binary
272
      TEST_BINARY: 'WSCC_9bus_mult_decoupled'
273
# Path where the test binary is located
274
      TEST_BINARY_PATH: 'build/Examples/Cxx'
275
276
277
278
# Additional files that are necessary to run the application on the GPU node
      TEST_FILE: ''
# Command line parameters for the test binary
      TEST_PARAM: ''
279

Markus Mirz's avatar
Markus Mirz committed
280
281
# Stage deploy
##############################################################################
282
pages:
Markus Mirz's avatar
Markus Mirz committed
283
  stage: deploy
284
  needs: ["generate:docs"]
Markus Mirz's avatar
Markus Mirz committed
285
  script:
Markus Mirz's avatar
Markus Mirz committed
286
287
    - cp -r build/docs/sphinx/html/. public/sphinx
    - cp -r build/docs/doxygen/html/. public/doxygen
Markus Mirz's avatar
Markus Mirz committed
288
289
290
291
292
293
  artifacts:
    paths:
      - public
  dependencies:
    - generate:docs
  only:
294
    - deploy
Markus Mirz's avatar
Markus Mirz committed
295
296
297
298
  tags:
    - shell
    - linux

299
deploy:docker:
300
301
  stage: deploy
  script:
Markus Mirz's avatar
Markus Mirz committed
302
    - docker push ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}
303
    - docker push ${DOCKER_IMAGE_DEV}:latest
304
    - docker push ${DOCKER_IMAGE}:${DOCKER_TAG}
305
    - docker push ${DOCKER_IMAGE}:latest
306
307
308
309
  dependencies:
    - build:docker
  tags:
    - shell
Steffen Vogel's avatar
Steffen Vogel committed
310
    - linux
Steffen Vogel's avatar
Steffen Vogel committed
311
312
  only:
    - tags
313

314
deploy:packages:
Steffen Vogel's avatar
Steffen Vogel committed
315
316
  stage: deploy
  script:
Markus Mirz's avatar
Markus Mirz committed
317
    - rsync ${RSYNC_OPTS} build/*.rpm ${DEPLOY_USER}@${DEPLOY_HOST}:/var/www/packages/fedora/29/x86_64
Steffen Vogel's avatar
Steffen Vogel committed
318
    - rsync ${RSYNC_OPTS} build/*.tar.gz ${DEPLOY_USER}@${DEPLOY_HOST}:/var/www/dpsim/src
Markus Mirz's avatar
Markus Mirz committed
319
    - ssh ${DEPLOY_USER}@${DEPLOY_HOST} createrepo /var/www/packages/fedora/29
Steffen Vogel's avatar
Steffen Vogel committed
320
  dependencies:
Markus Mirz's avatar
fix ci    
Markus Mirz committed
321
    - generate:packages
Steffen Vogel's avatar
Steffen Vogel committed
322
323
324
325
326
  only:
    - tags
  tags:
    - fein-deploy
    - shell
327
    - linux