From 3d3782f205fa949c6cc05cd54da3074581a731d9 Mon Sep 17 00:00:00 2001 From: Domenico Date: Thu, 9 Jul 2020 18:39:09 +0200 Subject: [PATCH 1/3] Integrated keycloak image --- .env | 4 ++ docker-compose.yml | 59 +++++++++++++++--- docker-mariadb/encryption/keyfile | 5 ++ docker-mariadb/encryption/keyfile.enc | Bin 0 -> 368 bytes docker-mariadb/encryption/keyfile.key | 1 + docker-mariadb/my_custom.cnf | 18 ++++++ .../scripts/create-multiple-databases.sh | 23 +++++++ docker-mariadb/scripts/secure-database.sh | 8 +++ 8 files changed, 108 insertions(+), 10 deletions(-) create mode 100644 docker-mariadb/encryption/keyfile create mode 100644 docker-mariadb/encryption/keyfile.enc create mode 100644 docker-mariadb/encryption/keyfile.key create mode 100644 docker-mariadb/my_custom.cnf create mode 100644 docker-mariadb/scripts/create-multiple-databases.sh create mode 100644 docker-mariadb/scripts/secure-database.sh diff --git a/.env b/.env index e389089..5ef6d78 100644 --- a/.env +++ b/.env @@ -18,6 +18,10 @@ DATABASE_ROOT_PASSWORD=gk_admin DATABASE_USER=gk_admin DATABASE_PASSWORD=gk_admin +#keyclock +KEYCLOAK_USER=gk_admin +KEYCLOAK_PASSWORD=gk_admin + # Note: all dollar signs in the hash need to be doubled for escaping. # To create user:password pair, it's possible to use this command: # echo $(htpasswd -nb user password) | sed -e s/\\$/\\$\\$/g diff --git a/docker-compose.yml b/docker-compose.yml index 3d327a5..d6f7bd8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,7 +20,7 @@ services: volumes: - gk-fhir-server-data:/usr/local/tomcat/target depends_on: - - fhir-db + - db #networks: # - backend @@ -34,16 +34,52 @@ services: - rdf4j_data:/opt/eclipse-rdf4j-${RDF4J_VERSION}/data #networks: #- backend - - fhir-db: + + #keycloak + authz: + image: 'jboss/keycloak' + container_name: authz + ports: + - '8082:8080' + environment: + DB_VENDOR: mariadb + DB_ADDR: db + DB_DATABASE: keycloak + DB_USER: ${DATABASE_USER} + DB_PASSWORD: ${DATABASE_PASSWORD} + KEYCLOAK_USER: ${KEYCLOAK_USER} + KEYCLOAK_PASSWORD: ${KEYCLOAK_PASSWORD} + #KEYCLOAK_ALWAYS_HTTPS: "false" + #PROXY_ADDRESS_FORWARDING: "false" + labels: + # Explicitly tell Traefik to expose this container + - "traefik.enable=true" + - "traefik.docker.network=traefik_proxy" + # The domain the service will respond to + - "traefik.http.routers.authz.rule=Host(`${PUBLIC_DOMAIN}`) && PathPrefix(`/auth`)" + - "traefik.http.routers.authz.entrypoints=web,websecure" + #- "traefik.http.services.authz.loadbalancer.server.scheme=http" + # - "traefik.http.routers.authz.tls.certresolver=le" # Uses the Host rule to define which certificate to issue + depends_on: + - db + #command: + #- "-b 0.0.0.0" + #- "-Dkeycloak.profile.feature.docker=enabled" + # networks: + # - traefik_proxy + #- db + + + db: # entrypoint: ["echo", "Service 'db' disabled"] # restart: unless-stopped image: 'bitnami/mariadb:10.4-debian-10' # ~90MB - container_name: fhir-db + container_name: db ports: - '3306:3306' environment: - - MARIADB_DATABASE=gk-fhir-db + #- MARIADB_DATABASE=gk-fhir-db + - MARIADB_MULTIPLE_DATABASES=gk-fhir-db,keycloak - ALLOW_EMPTY_PASSWORD=no # 'yes' is recommended only for development. - MARIADB_ROOT_PASSWORD=${DATABASE_ROOT_PASSWORD} - MARIADB_USER=${DATABASE_USER} @@ -57,9 +93,10 @@ services: - ./docker-mariadb/encryption:/etc/encryption # Encryption keys - ./docker-mariadb/scripts:/docker-entrypoint-initdb.d # Create DBs - db-data:/bitnami/mariadb - # networks: - # - backend-network - + #networks: + # - db + + # # Volumes # @@ -69,5 +106,7 @@ volumes: db-data: #networks: - # backend: - # driver: "bridge" \ No newline at end of file + # traefik_proxy: + # external: true + #db: + # driver: bridge \ No newline at end of file diff --git a/docker-mariadb/encryption/keyfile b/docker-mariadb/encryption/keyfile new file mode 100644 index 0000000..c13dde9 --- /dev/null +++ b/docker-mariadb/encryption/keyfile @@ -0,0 +1,5 @@ +1;cb1f71fccf369f6a29d41c7ddd9a3e76128d9af8804675d53f3379bec7a53103 +2;62a30924c834c9731bc61fea99c1118d3d9c388bdea4049e5b3418ef7d99df3a +3;0996977829150099b0edb95e8868858d8e6698ce598fad7a4735226181a6f851 +4;6de77273424e4b05d07fee1e62bd45e7a3d0fb9506580838ea7a0c316ddd793a +100;dc49bc6943290ff6bebe880876e831a1b44d764aabec386688c1aa2aa1936069 diff --git a/docker-mariadb/encryption/keyfile.enc b/docker-mariadb/encryption/keyfile.enc new file mode 100644 index 0000000000000000000000000000000000000000..0ff39cc44aca545ac1f0a55e594a5f6ff668b7b4 GIT binary patch literal 368 zcmWGe%qdAtiI4xg!t-nXU11}QJVnK0hs7A$?;L4;@!jxF>Tb~|g#v8{zV$agtT-K? zQFGls{Qt3r%H)Z4p-XjtmpFddXZB`Md}GYdQ!5NU{5s_s&YP0A?~Q;_;ng+jwZ_q> zf7*q6?mm2i;ps#Vt-H?i=D2Ix@zp;6KjCr~|HKyF3!B;EHXCs;E=atc7&=2=sxC-k za;W{jj@uPpNnCMz0xxb8-yvWpT5{ug!S4FqFhlR?H&|HS4yJ?fXa zQOwdQRZp&b`63hl{qpX(&WBl-=N$Djy?;Za;M1)~{5$`*@CZCMXS{o<`b9+a!u{GT zPkO`Bm(FEuyK-DMcUImMc163xOBs8W{a%;GIsMKykn4=uH*LR7&1>D@+VXjCB7OgD e^V_h^=%8X(u})=1=;3R@irHcsft<(VP5}V%jke(c literal 0 HcmV?d00001 diff --git a/docker-mariadb/encryption/keyfile.key b/docker-mariadb/encryption/keyfile.key new file mode 100644 index 0000000..729f64a --- /dev/null +++ b/docker-mariadb/encryption/keyfile.key @@ -0,0 +1 @@ +e4f70694bc12f11229eb5330e8c724538e1824ba586f23406f3559d1eb769da79d5d203dce337d1e1b3751243f065e0087f2543b10a5fd6690a18c8e7eb6f4f195ca8ba1ae6209fdc0da7fbcd921a0592e95dd41e400f42721056c8fa7061877f134d73a4e36f85be20a3f56893abf90e88332c68887f740291a152ed070074f diff --git a/docker-mariadb/my_custom.cnf b/docker-mariadb/my_custom.cnf new file mode 100644 index 0000000..534001e --- /dev/null +++ b/docker-mariadb/my_custom.cnf @@ -0,0 +1,18 @@ +[mysqld] +# max_allowed_packet=32M + +[mariadb] +# File Key Management +plugin_load_add = file_key_management +file_key_management_filename = /etc/encryption/keyfile.enc +file_key_management_filekey = FILE:/etc/encryption/keyfile.key +file_key_management_encryption_algorithm = AES_CTR +encrypt_binlog = 1 + +# InnoDB/XtraDB Encryption +# innodb_encrypt_tables = ON +innodb_encrypt_tables = FORCE +innodb_encrypt_temporary_tables = ON +innodb_encrypt_log = ON +innodb_encryption_threads = 4 +innodb_encryption_rotate_key_age = 1 \ No newline at end of file diff --git a/docker-mariadb/scripts/create-multiple-databases.sh b/docker-mariadb/scripts/create-multiple-databases.sh new file mode 100644 index 0000000..d29c8e4 --- /dev/null +++ b/docker-mariadb/scripts/create-multiple-databases.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +#set -e +set -u + +function create_user_and_database() { + local database=$1 + echo " Creating user $MARIADB_USER and database '$database'" + + mysql -u root -p$MARIADB_ROOT_PASSWORD <<-EOSQL + CREATE USER IF NOT EXISTS '$MARIADB_USER'@'%' IDENTIFIED BY '$MARIADB_PASSWORD'; + CREATE DATABASE IF NOT EXISTS \`$database\`; + GRANT ALL ON \`$database\`.* TO '$MARIADB_USER'@'%'; + EOSQL +} + +if [ -n "$MARIADB_MULTIPLE_DATABASES" ]; then + echo "Multiple database creation requested: $MARIADB_MULTIPLE_DATABASES" + for db in $(echo $MARIADB_MULTIPLE_DATABASES | tr ',' ' '); do + create_user_and_database $db + done + echo "Multiple databases created" +fi diff --git a/docker-mariadb/scripts/secure-database.sh b/docker-mariadb/scripts/secure-database.sh new file mode 100644 index 0000000..4c3885d --- /dev/null +++ b/docker-mariadb/scripts/secure-database.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -e +set -u + +mysql -u root -p$MARIADB_ROOT_PASSWORD <<-EOSQL + DROP DATABASE IF EXISTS test; +EOSQL -- GitLab From fa5902bddd43986335bc48dc2e51e81ff283aff5 Mon Sep 17 00:00:00 2001 From: Domenico Date: Wed, 22 Jul 2020 16:26:05 +0200 Subject: [PATCH 2/3] Updated repository to download the gk-fhir-server and gk-integration engine images. Now they are pulled from private docker registry --- .env | 4 ++-- docker-compose.yml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.env b/.env index 5ef6d78..10af57b 100644 --- a/.env +++ b/.env @@ -1,5 +1,5 @@ -#DOCKER_IMAGE_REPO=217.172.12.153:5000 -DOCKER_IMAGE_REPO=localhost:5000 +DOCKER_IMAGE_REPO=gk.eng.it:5000 +#DOCKER_IMAGE_REPO=localhost:5000 #Domain PUBLIC_DOMAIN=localhost diff --git a/docker-compose.yml b/docker-compose.yml index d6f7bd8..d39b862 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,16 +5,16 @@ services: gk-integration-engine: container_name: gk-integration-engine - build: ../gk-integration-engine - #image: ${DOCKER_IMAGE_REPO}/gk-docker_gk-integration-engine_v0.1 + #build: ../gk-integration-engine + image: ${DOCKER_IMAGE_REPO}/gk-integration-engine ports: - "${GK_CONNECTOR_PORT}:8080" gk-fhir-server: container_name: gk-fhir-server - build: ../gk-fhir-server - #image: ${DOCKER_IMAGE_REPO}/gk-docker_gk-fhir-serve_v0.1 + #build: ../gk-fhir-server + image: ${DOCKER_IMAGE_REPO}/gk-fhir-server ports: - "${GK_FHIR_SERVER_PORT}:8080" volumes: -- GitLab From f3c5faa55256280e48d31725acdad9d553e9e76e Mon Sep 17 00:00:00 2001 From: Domenico Date: Wed, 5 Aug 2020 13:17:03 +0200 Subject: [PATCH 3/3] Updated instructions to run docker-compose pulling images from private docker-registry --- README.md | 40 +++---- docker-compose for local build/.env | 29 +++++ docker-compose for local build/README.md | 54 +++++++++ .../docker-compose.yml | 112 ++++++++++++++++++ 4 files changed, 212 insertions(+), 23 deletions(-) create mode 100644 docker-compose for local build/.env create mode 100644 docker-compose for local build/README.md create mode 100644 docker-compose for local build/docker-compose.yml diff --git a/README.md b/README.md index a763777..06eb73e 100644 --- a/README.md +++ b/README.md @@ -1,47 +1,41 @@ # gk-docker -Docker compose that starts all services needed for Semantic Data Lake Workbench +Docker compose that starts all services needed for Data Federation pulling docker images from private docker-registry -#### Clone projects gk-integration-engine, gk-fhir-server and gk-docker in the same folder +#### Clone project gk-docker Open a terminal and run the following (you'll be asked for your GitLab credentials): - ```bash -git clone https://production.eng.it/gitlab/GTKEEPER_EU/gk-integration-engine.git -git clone https://production.eng.it/gitlab/GTKEEPER_EU/gk-fhir-server.git git clone https://production.eng.it/gitlab/GTKEEPER_EU/gk-docker.git ``` -## Compile gk-integration-engine and gk-fhir-server with the docker profile -Move to the gk-integration-engine folder and run the following command: -```bash -mvn clean install -``` -Move to the gk-fhir-server folder and run the following command: -```bash -mvn clean install -``` -## Build and start Semantic Data Lake Workbench -Move to the gk-docker folder and run the following command: +#### Login to docker registry +Run the following command: ```bash -docker-compose up --build -d +docker login http://gk.eng.it:5000 -u -p ``` +Replace and with the real docker registry credentials -## Start Semantic Data Lake Workbench (without any build process) -Move to the gk-docker folder and run the following command: +#### Run Data Federation +Move to the gk-docker folder, edit .env file with your custom configuration and run the following command: ```bash docker-compose up -d ``` - +All running services can be listed with the following command: +```bash +docker ps -a +``` Stop the docker-compose using the following command: ```bash docker-compose down ``` ## Services are avaible to the following endpoint -Fhir integration-engine: [http://localhost:8087/swagger/index.html](http://localhost:8087/swagger/index.html)
-Fhir API: [http://localhost:8085/gk-fhir-server](http://localhost:8085/gk-fhir-server)
-RDF4J Workbench: [http://localhost:8080/rdf4j-workbench](http://localhost:8080/rdf4j-workbench) +Fhir integration-engine: [http://localhost:8087/swagger/index.html](http://localhost:8087/swagger/index.html)
+Fhir API: [http://localhost:8085/gk-fhir-server](http://localhost:8085/gk-fhir-server)
+RDF4J Workbench: [http://localhost:8080/rdf4j-workbench](http://localhost:8080/rdf4j-workbench)
+Keyclock: [http://localhost:8082/auth](http://localhost:8082/auth) +# If you want to build and run all services without download the images from private docker registry, follow instructions contained in the folder "docker-compose for local build" ##### Maintainers diff --git a/docker-compose for local build/.env b/docker-compose for local build/.env new file mode 100644 index 0000000..10af57b --- /dev/null +++ b/docker-compose for local build/.env @@ -0,0 +1,29 @@ +DOCKER_IMAGE_REPO=gk.eng.it:5000 +#DOCKER_IMAGE_REPO=localhost:5000 + +#Domain +PUBLIC_DOMAIN=localhost +#PUBLIC_DOMAIN=217.172.12.153 + +# Port +GK_CONNECTOR_PORT=8087 +GK_FHIR_SERVER_PORT=8085 +RDF4J_WORKBENCH=8080 + +# rdf4j version +RDF4J_VERSION=2.5.1 + +# mariadb +DATABASE_ROOT_PASSWORD=gk_admin +DATABASE_USER=gk_admin +DATABASE_PASSWORD=gk_admin + +#keyclock +KEYCLOAK_USER=gk_admin +KEYCLOAK_PASSWORD=gk_admin + +# Note: all dollar signs in the hash need to be doubled for escaping. +# To create user:password pair, it's possible to use this command: +# echo $(htpasswd -nb user password) | sed -e s/\\$/\\$\\$/g +# IMPORTANT: with ENV it is not required variable escaping (i.e "| sed ...") +#DASHBOARD_CREDENTIALS=gk_admin:$apr1$dfdfdffddf/Bddffdfddfd/ diff --git a/docker-compose for local build/README.md b/docker-compose for local build/README.md new file mode 100644 index 0000000..9e98b90 --- /dev/null +++ b/docker-compose for local build/README.md @@ -0,0 +1,54 @@ +# gk-docker + +Docker compose that starts all services needed for Data Federation + +#### Clone projects gk-integration-engine, gk-fhir-server and gk-docker in the same folder + +Open a terminal and run the following (you'll be asked for your GitLab credentials): + +```bash +git clone https://production.eng.it/gitlab/GTKEEPER_EU/gk-integration-engine.git +git clone https://production.eng.it/gitlab/GTKEEPER_EU/gk-fhir-server.git +git clone https://production.eng.it/gitlab/GTKEEPER_EU/gk-docker.git +``` +## Compile gk-integration-engine and gk-fhir-server with the docker profile +Move to the gk-integration-engine folder and run the following command: +```bash +mvn clean install +``` +Move to the gk-fhir-server folder and run the following command: +```bash +mvn clean install +``` +## Build and start Semantic Data Lake Workbench +Move to the gk-docker folder and run the following command: +```bash +docker-compose up --build -d +``` + +## Start Semantic Data Lake Workbench (without any build process) +Move to the gk-docker folder and run the following command: +```bash +docker-compose up -d +``` + +Stop the docker-compose using the following command: +```bash +docker-compose down +``` +## Services are avaible to the following endpoint + +Fhir integration-engine: [http://localhost:8087/swagger/index.html](http://localhost:8087/swagger/index.html)
+Fhir API: [http://localhost:8085/gk-fhir-server](http://localhost:8085/gk-fhir-server)
+RDF4J Workbench: [http://localhost:8080/rdf4j-workbench](http://localhost:8080/rdf4j-workbench) + + + +##### Maintainers + * domenico.martino@eng.it + * vincenzo.falanga@eng.it + * federica.sacca@eng.it + * paolo.zampognaro@eng.it + +## License +[GNU AGPLv3](https://choosealicense.com/licenses/agpl-3.0/) \ No newline at end of file diff --git a/docker-compose for local build/docker-compose.yml b/docker-compose for local build/docker-compose.yml new file mode 100644 index 0000000..5f07b3e --- /dev/null +++ b/docker-compose for local build/docker-compose.yml @@ -0,0 +1,112 @@ +# Docker Compose file Reference (https://docs.docker.com/compose/compose-file/) +version: '3.7' + +services: + + gk-integration-engine: + container_name: gk-integration-engine + build: ../../gk-integration-engine + #image: ${DOCKER_IMAGE_REPO}/gk-integration-engine + ports: + - "${GK_CONNECTOR_PORT}:8080" + + + gk-fhir-server: + container_name: gk-fhir-server + build: ../../gk-fhir-server + #image: ${DOCKER_IMAGE_REPO}/gk-fhir-server + ports: + - "${GK_FHIR_SERVER_PORT}:8080" + volumes: + - gk-fhir-server-data:/usr/local/tomcat/target + depends_on: + - db + #networks: + # - backend + + + rdf4j: + container_name: rdf4j + image: yyz1989/rdf4j + ports: + - "${RDF4J_WORKBENCH}:8080" + volumes: + - rdf4j_data:/opt/eclipse-rdf4j-${RDF4J_VERSION}/data + #networks: + #- backend + + #keycloak + authz: + image: 'jboss/keycloak' + container_name: authz + ports: + - '8082:8080' + environment: + DB_VENDOR: mariadb + DB_ADDR: db + DB_DATABASE: keycloak + DB_USER: ${DATABASE_USER} + DB_PASSWORD: ${DATABASE_PASSWORD} + KEYCLOAK_USER: ${KEYCLOAK_USER} + KEYCLOAK_PASSWORD: ${KEYCLOAK_PASSWORD} + #KEYCLOAK_ALWAYS_HTTPS: "false" + #PROXY_ADDRESS_FORWARDING: "false" + labels: + # Explicitly tell Traefik to expose this container + - "traefik.enable=true" + - "traefik.docker.network=traefik_proxy" + # The domain the service will respond to + - "traefik.http.routers.authz.rule=Host(`${PUBLIC_DOMAIN}`) && PathPrefix(`/auth`)" + - "traefik.http.routers.authz.entrypoints=web,websecure" + #- "traefik.http.services.authz.loadbalancer.server.scheme=http" + # - "traefik.http.routers.authz.tls.certresolver=le" # Uses the Host rule to define which certificate to issue + depends_on: + - db + #command: + #- "-b 0.0.0.0" + #- "-Dkeycloak.profile.feature.docker=enabled" + # networks: + # - traefik_proxy + #- db + + + db: + # entrypoint: ["echo", "Service 'db' disabled"] + # restart: unless-stopped + image: 'bitnami/mariadb:10.4-debian-10' # ~90MB + container_name: db + ports: + - '3306:3306' + environment: + #- MARIADB_DATABASE=gk-fhir-db + - MARIADB_MULTIPLE_DATABASES=gk-fhir-db,keycloak + - ALLOW_EMPTY_PASSWORD=no # 'yes' is recommended only for development. + - MARIADB_ROOT_PASSWORD=${DATABASE_ROOT_PASSWORD} + - MARIADB_USER=${DATABASE_USER} + - MARIADB_PASSWORD=${DATABASE_PASSWORD} + # Copy-pasted from https://github.com/docker-library/mariadb/issues/94 + healthcheck: + test: ["CMD", "mysqladmin", "ping", "--silent"] + # restart: always + volumes: + - ./../docker-mariadb/my_custom.cnf:/opt/bitnami/mariadb/conf/my_custom.cnf:ro # Enable encryption + - ./../docker-mariadb/encryption:/etc/encryption # Encryption keys + - ./../docker-mariadb/scripts:/docker-entrypoint-initdb.d # Create DBs + - db-data:/bitnami/mariadb + #networks: + # - db + + +# +# Volumes +# +volumes: + rdf4j_data: + gk-fhir-server-data: + db-data: + +#networks: + # traefik_proxy: + # external: true + #db: + # driver: bridge \ No newline at end of file -- GitLab