.gitlab-ci.yml 16 KB
Newer Older
1
  image: frolvlad/alpine-glibc
2

3
  variables:
4
    DOCKERHUB_USERNAME: deichman
Mattias Lundmark's avatar
Mattias Lundmark committed
5

6 7 8 9 10 11
  .before_script: &before
    - echo -n "START :" && date -u +'%Y-%m-%dT%H:%M:%SZ'
    - apk add --update --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing make docker curl bash git
    - curl -L "https://github.com/docker/compose/releases/download/1.25.4/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
    - chmod +x /usr/local/bin/docker-compose
    - echo $DOCKERHUB_PASSWORD|docker login -u $DOCKERHUB_USERNAME --password-stdin
12 13 14
    - export DOCKER_BUILDKIT=1
    - export COMPOSE_DOCKER_CLI_BUILD=1
    - export TAG=${CI_COMMIT_TAG:-latest}
15 16
  .after_script: &after
    - echo -n "END :" && date -u +'%Y-%m-%dT%H:%M:%SZ'
17

18
  before_script:
19
    - *before
20

21
  after_script:
22
    - *after
23

24
  stages:
25
    - docker-cleanup
26 27 28 29 30 31
    - build
    - test
    - push
    - deploy-test
    - sanity
    - deploy-stage
32
    - deploy-prod
33 34


35 36 37 38 39 40
  #Building
  .build-application:
    script:
      - echo -n "Building ${APPLICATION_NAME}"
      - make build-${APPLICATION_NAME}
    stage: build
41 42 43
    except:
      refs:
        - tags
44

45 46 47 48 49 50 51 52 53
  build-all:
    stage: build
    script:
      - echo -n "Building all"
      - make build-all
    only:
      refs:
        - tags

54 55 56 57 58 59 60 61
#  build-all-scheduled:
#    stage: build
#    script:
#      - echo -n "Building all Scheduled"
#      - make build-all
#    only:
#      variables:
#        - $BUILD_TYPE == "scheduled"
62

63 64 65 66 67 68 69
  build-authorization-server:
    variables:
      APPLICATION_NAME: authorization-server
    extends:
      - .build-application
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "authorization-server/**/*" ]
70 71 72
    except:
      refs:
        - tags
73

74 75 76 77 78 79 80
  build-deichman.no:
    variables:
      APPLICATION_NAME: deichman.no
    extends:
      - .build-application
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "deichman.no/**/*"]
81

82 83 84 85 86 87 88
  build-elasticsearch:
    variables:
      APPLICATION_NAME: elasticsearch
    extends:
      - .build-application
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "elasticsearch/**/*"]
89

90 91 92 93 94 95 96
  build-euler:
    variables:
      APPLICATION_NAME: euler
    extends:
      - .build-application
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "euler/**/*"]
97

98 99 100 101 102 103 104
  build-f-users:
    variables:
      APPLICATION_NAME: f-users
    extends:
      - .build-application
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "f-users/**/*"]
105

106 107 108 109 110 111 112
  build-image-api:
    variables:
      APPLICATION_NAME: image-api
    extends:
      - .build-application
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "image-api/**/*"]
113

114 115 116 117 118 119 120 121
  build-image-cache:
    variables:
      APPLICATION_NAME: image-cache
    extends:
      - .build-application
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "image-cache/**/*"]

122 123 124 125 126 127 128
  build-katalog:
    variables:
      APPLICATION_NAME: katalog
    extends:
      - .build-application
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "katalog/**/*"]
129

130 131 132 133 134 135 136
  build-kibana:
    variables:
      APPLICATION_NAME: kibana
    extends:
      - .build-application
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "kibana/**/*"]
Magnus Westergaard's avatar
Magnus Westergaard committed
137

138 139 140 141 142 143 144
  build-koha:
    variables:
      APPLICATION_NAME: koha
    extends:
      - .build-application
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "koha/**/*"]
Magnus Westergaard's avatar
Magnus Westergaard committed
145

146 147 148 149 150 151 152
  build-mimir:
    variables:
      APPLICATION_NAME: mimir
    extends:
      - .build-application
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "mimir/**/*"]
Mattias Lundmark's avatar
Mattias Lundmark committed
153 154


155 156 157 158 159 160 161
  build-redia-api:
    variables:
      APPLICATION_NAME: redia-api
    extends:
      - .build-application
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "redia-api/**/*"]
Mattias Lundmark's avatar
Mattias Lundmark committed
162

163 164 165 166 167 168 169
  build-sibyl:
    variables:
      APPLICATION_NAME: sibyl
    extends:
      - .build-application
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "sibyl/**/*"]
170

171 172 173 174 175 176 177 178
  build-skranke:
    variables:
      APPLICATION_NAME: skranke
    extends:
      - .build-application
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "skranke/**/*"]

179 180 181 182 183 184 185
  build-test-ldap:
    variables:
      APPLICATION_NAME: test-ldap
    extends:
      - .build-application
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "test-ldap/**/*"]
186

187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208
  build-tjenestekatalog:
    variables:
      APPLICATION_NAME: tjenestekatalog
    extends:
      - .build-application
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "tjenestekatalog/**/*"]

  #Deployment
  .deploy-scp:

    before_script:
      - *before
      - apk add --update --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing make curl bash git
      - 'which ssh-agent || ( apk add --update --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing openssh-client git )'
      - eval $(ssh-agent -s)
      - echo "$SSH_KEY" | tr -d '\r' | ssh-add - > /dev/null
      - mkdir -p ~/.ssh
      - chmod 700 ~/.ssh
      - echo "$SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
      - chmod 644 ~/.ssh/known_hosts
      - export APPLICATION_NAME=${APPLICATION_NAME}
209
    script:
210
      - make remote-deploy-cold ssh_user=${SSH_USER} cold_node="${SSH_COLD_SERVER}"
211 212
      - make remote-deploy ssh_user=${SSH_USER} ssh_servers="${SSH_SERVERS}" CONTAINER=${APPLICATION_NAME}
      - make remote-deploy-voting ssh_user=${SSH_USER} voting_node="${SSH_VOTING_SERVER}"
213 214 215 216 217
    when: manual

  #Deploy test
  .devtest:
    stage: deploy-test
218 219 220
    environment:
      name: test
      url: https://deichman.test.obib.no
221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301
    variables:
      SSH_USER: $DEPLOY_USER
      SSH_SERVERS: $DEVTEST_SERVER
      SSH_KNOWN_HOSTS: $DEVTEST_KNOWN_HOSTS
      SSH_KEY: $DEPLOY_KEY

  deploy-scheduled-test:
    extends:
      - .devtest
      - .deploy-scp
    after_script:
      - ./transitionIssues.sh test
      - *after
    only:
      variables:
        - $BUILD_TYPE == "scheduled"
    when: on_success

  deploy-all-test:
    extends:
      - .devtest
      - .deploy-scp
    after_script:
      - ./transitionIssues.sh test
      - *after
    when: manual

  deploy-authorization-server-test:
    extends:
      - .devtest
      - .deploy-scp
    variables:
      APPLICATION_NAME: authorization-server
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "authorization-server/**/*"]

  deploy-deichman_no-test:
    extends:
      - .devtest
      - .deploy-scp
    variables:
      APPLICATION_NAME: deichman.no
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "deichman.no/**/*"]

  deploy-elasticsearch-test:
    extends:
      - .devtest
      - .deploy-scp
    variables:
      APPLICATION_NAME: elasticsearch
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "elasticsearch/**/*"]

  deploy-euler-test:
    extends:
      - .devtest
      - .deploy-scp
    variables:
      APPLICATION_NAME: euler
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "euler/**/*"]

  deploy-f-users-test:
    extends:
      - .devtest
      - .deploy-scp
    variables:
      APPLICATION_NAME: f-users
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "f-users/**/*"]

  deploy-image-api-test:
    extends:
      - .devtest
      - .deploy-scp
    variables:
      APPLICATION_NAME: image-api
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "image-api/**/*"]

302 303 304 305 306 307 308 309 310
  deploy-image-cache-test:
    extends:
      - .devtest
      - .deploy-scp
    variables:
      APPLICATION_NAME: image-cache
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "image-cache/**/*"]

311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334
  deploy-katalog-test:
    extends:
      - .devtest
      - .deploy-scp
    variables:
      APPLICATION_NAME: katalog
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "katalog/**/*"]

  deploy-kibana-test:
    extends:
      - .devtest
      - .deploy-scp
    variables:
      APPLICATION_NAME: kibana
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "kibana/**/*"]

  deploy-koha-test:
    extends:
      - .devtest
      - .deploy-scp
    variables:
      APPLICATION_NAME: koha
335 336
    after_script:
      - *after
337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "koha/**/*"]

  deploy-mimir-test:
    extends:
      - .devtest
      - .deploy-scp
    variables:
      APPLICATION_NAME: mimir
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "mimir/**/*"]

  deploy-redia-api-test:
    extends:
      - .devtest
      - .deploy-scp
    variables:
      APPLICATION_NAME: redia-api
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "redia-api/**/*"]

  deploy-sibyl-test:
    extends:
      - .devtest
      - .deploy-scp
    variables:
      APPLICATION_NAME: sibyl
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "sibyl/**/*"]

367 368 369 370 371 372 373 374 375
  deploy-skranke-test:
    extends:
      - .devtest
      - .deploy-scp
    variables:
      APPLICATION_NAME: skranke
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "skranke/**/*"]

376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392
  deploy-test-ldap-test:
    extends:
      - .devtest
      - .deploy-scp
    variables:
      APPLICATION_NAME: test-ldap
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "test-ldap/**/*"]

  deploy-tjenestekatalog-test:
    extends:
      - .devtest
      - .deploy-scp
    variables:
      APPLICATION_NAME: tjenestekatalog
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "tjenestekatalog/**/*"]
Mattias Lundmark's avatar
Mattias Lundmark committed
393

394
  #Deploy staging
Magnus Westergaard's avatar
Magnus Westergaard committed
395

396 397
  .staging:
    stage: deploy-stage
398 399
    environment:
      name: stage
400
      url: https://staging.obib.no
401 402 403 404 405
    variables:
      SSH_USER: $DEPLOY_USER
      SSH_KNOWN_HOSTS: $STAGING_HOSTS
      SSH_SERVERS: "${STAGING_SERVERS}"
      SSH_KEY: $DEPLOY_KEY
406
      SSH_VOTING_SERVER: ${STAGING_VOTING_SERVER}
407
      SSH_COLD_SERVER: ${STAGING_COLD_SERVER}
408 409 410 411 412

  deploy-all-staging:
    extends:
      - .staging
      - .deploy-scp
413 414 415
#    only:
#      refs:
#        - tags
416 417
    when: manual
    after_script:
418
#      - ./transitionIssues.sh stage
419 420
      - *after

421 422 423 424
  .production:
    stage: deploy-prod
    environment:
      name: prod
425
      url: https://beta.deichman.no
426 427 428 429 430 431 432 433 434
    variables:
      SSH_USER: $DEPLOY_USER
      SSH_KNOWN_HOSTS: $PRODUCTION_HOSTS
      SSH_SERVERS: "${PRODUCTION_SERVERS}"
      SSH_KEY: $DEPLOY_KEY
      SSH_VOTING_SERVER: ${PRODUCTION_VOTING_SERVER}

  deploy-production:
    extends:
435
      - .production
436 437 438 439 440 441 442 443 444
      - .deploy-scp
    #    only:
    #      refs:
    #        - tags
    when: manual
    after_script:
      #      - ./transitionIssues.sh stage
      - *after

445 446
  #Automated tests
  koha-api-tests:
447 448 449
    stage: sanity
    extends:
      - .devtest
450
    variables:
451 452 453
      DEVTEST_KOHA_DB: "koha-db"
      DEVTEST_KOHA_API: "${DEVTEST_KOHA_API}"
      #CMD: "'mysql -u $MYSQL_USER -p$MYSQL_PASSWORD koha_deichman' < /opt/deichman/deployment-config/koha/fixtures/cleanup.sql"
454 455 456 457 458
      APPLICATION_NAME: koha
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "koha/**/*"]
    tags:
      - koha
459 460
    #before_script: # ensure clean slate
    #  - make remote-docker-command ssh_user=${SSH_USER} ssh_server="${SSH_SERVERS}" CONTAINER=${DEVTEST_KOHA_DB} cmd="${cmd}"
461 462
    script:
      - cd koha/tests
463
      #- ./run_tests.sh
464 465
    when: on_success

466
  e2e-tests:
467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482
    tags:
      - e2e
    script:
      - cd e2e-tests
      - docker-compose -p $CI_PIPELINE_IID -f docker-compose-common.yml -f docker-compose-ci.yml up --build --exit-code-from tests
    after_script:
      - cd e2e-tests
      - docker-compose -p $CI_PIPELINE_IID -f docker-compose-common.yml -f docker-compose-ci.yml down -v --rmi local
      - echo -n "END :" && date -u +'%Y-%m-%dT%H:%M:%SZ'
    artifacts:
      when: always
      expire_in: 1 week
      paths:
        - e2e-tests/cypress/videos/
        - e2e-tests/cypress/screenshots/
    stage: test
483 484 485 486
    rules:
      - if: '$CI_PIPELINE_SOURCE == "schedule"'
        when: always
      - when: manual
487
    allow_failure: true
488

489 490 491 492 493
  zork-cleanup:
    stage: docker-cleanup
    script:
      - df -h /
      - docker system df
494
      - docker image prune -a -f # TODO: prune cache, volumes if necessary
495 496 497 498
      - docker system df
      - df -h /
    rules:
      - if: '$CI_PIPELINE_SOURCE == "schedule"'
499
        when: always
500 501 502
      - when: manual
    allow_failure: true

503 504 505 506
  #
  ## Push stages here
  .push:
    script:
507
      - make push-${APPLICATION_NAME}
508
    stage: push
509 510 511
    except:
      variables:
        - $BUILD_TYPE == "scheduled"
512 513
      refs:
        - tags
514 515
    when: manual

516 517 518 519 520 521 522
  push-tag:
    script:
      - make push-all
    stage: push
    only:
      refs:
        - tags
523

524
  push-authorization-server-latest:
525 526 527 528 529 530
    extends:
      - .push
    when: on_success
    variables:
      APPLICATION_NAME: authorization-server
    only:
531 532
      refs:
        - master
533
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "authorization-server/**/*"]
534

535

536 537 538 539 540 541 542
  push-deichman.no-latest:
    extends:
      - .push
    when: on_success
    variables:
      APPLICATION_NAME: deichman.no
    only:
543 544
      refs:
        - master
545 546 547 548 549 550 551 552 553
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "deichman.no/**/*"]

  push-elasticsearch-latest:
    extends:
      - .push
    when: on_success
    variables:
      APPLICATION_NAME: elasticsearch
    only:
554 555
      refs:
        - master
556 557 558 559 560 561 562 563 564
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "elasticsearch/**/*"]

  push-euler-latest:
    extends:
      - .push
    when: on_success
    variables:
      APPLICATION_NAME: euler
    only:
565 566
      refs:
        - master
567 568 569 570 571 572 573 574 575
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "euler/**/*"]

  push-f-users-latest:
    extends:
      - .push
    when: on_success
    variables:
      APPLICATION_NAME: f-users
    only:
576 577
      refs:
        - master
578 579 580 581 582 583 584 585 586
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "f-users/**/*"]

  push-image-api-latest:
    extends:
      - .push
    when: on_success
    variables:
      APPLICATION_NAME: image-api
    only:
587 588
      refs:
        - master
589 590
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "image-api/**/*"]

591 592 593 594 595 596 597 598 599 600 601
  push-image-cache-latest:
    extends:
      - .push
    when: on_success
    variables:
      APPLICATION_NAME: image-cache
    only:
      refs:
        - master
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "image-cache/**/*"]

602 603 604 605 606 607 608 609
  push-katalog-latest:
    extends:
      - .push
    when: on_success
    variables:
      APPLICATION_NAME: katalog
    only:
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "katalog/**/*"]
610 611
      refs:
        - master
612 613 614 615 616 617 618 619

  push-kibana-latest:
    extends:
      - .push
    when: on_success
    variables:
      APPLICATION_NAME: kibana
    only:
620 621
      refs:
        - master
622 623 624 625 626 627 628 629 630
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "kibana/**/*"]

  push-koha-latest:
    extends:
      - .push
    when: on_success
    variables:
      APPLICATION_NAME: koha
    only:
631 632
      refs:
        - master
633 634 635 636 637 638 639 640
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "koha/**/*"]
  push-mimir-latest:
    extends:
      - .push
    when: on_success
    variables:
      APPLICATION_NAME: mimir
    only:
641 642
      refs:
        - master
643 644 645 646 647 648 649 650
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "mimir/**/*"]
  push-redia-api-latest:
    extends:
      - .push
    when: on_success
    variables:
      APPLICATION_NAME: redia-api
    only:
651 652
      refs:
        - master
653 654 655 656 657 658 659 660
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "redia-api/**/*"]
  push-sibyl-latest:
    extends:
      - .push
    when: on_success
    variables:
      APPLICATION_NAME: sibyl
    only:
661 662
      refs:
        - master
663 664
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "sibyl/**/*"]

665 666 667 668 669 670 671
  push-skranke-latest:
    extends:
      - .push
    when: on_success
    variables:
      APPLICATION_NAME: skranke
    only:
672 673
      refs:
        - master
674 675
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "skranke/**/*"]

676 677 678 679 680 681 682
  push-test-ldap-latest:
    extends:
      - .push
    when: on_success
    variables:
      APPLICATION_NAME: test-ldap
    only:
683 684
      refs:
        - master
685 686 687 688 689 690 691 692 693
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "test-ldap/**/*"]

  push-tjenestekatalog-latest:
    extends:
      - .push
    when: on_success
    variables:
      APPLICATION_NAME: tjenestekatalog
    only:
694 695
      refs:
        - master
696
      changes: [ "Makefile", ".gitlab-ci.yml", "*.sh", "tjenestekatalog/**/*"]