From ceb0f7c8154d02a5cca547eabcf5b5d8dc471adf Mon Sep 17 00:00:00 2001 From: Iuliia Volkova Date: Sun, 29 Nov 2020 21:35:24 +0300 Subject: [PATCH] Update to 1.10.12 (#6) * add retry, update version to 1.10.12 * update readme.md --- README.md | 19 ++++++++++ docker-compose-with-celery-executor.yml | 46 +++++++++++++++++++------ 2 files changed, 55 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index d06dc31..dcc248b 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,15 @@ +# How to use + +```bash + + git clone https://github.com/xnuinside/airflow_in_docker_compose + cd airflow_in_docker_compose + docker-compose -f docker-compose-with-celery-executor.yml up --build + +``` + +Wait until all services will succesfull up & open http://localhost:8080/admin. + # Apache Airflow with Docker Compose examples **UPD from July 2020: Those articles was created before release of official Apache Airflow Docker image and they use puckel/docker-airflow. @@ -29,6 +41,13 @@ Source files for article with description on Medium. ![Main Apache Airflow UI](/docs/img/main.png?raw=true "Main Apache Airflow UI") ![Version](/docs/img/version.png?raw=true "Version Screen") + +### 29.11.2020: +1. Update Apache Airflow version to 1.10.12 +2. Update PostgreSQL DB to 13.1 +3. Added restart_policy to services in docker-compose + + ### 07.2020: 1. All compose files with puckel_image moved to docker_with_puckel_image 2. Creted docker-compose config based on official image (required docker-compose version 3.7 and higher): diff --git a/docker-compose-with-celery-executor.yml b/docker-compose-with-celery-executor.yml index 32a8f1c..18a72a5 100644 --- a/docker-compose-with-celery-executor.yml +++ b/docker-compose-with-celery-executor.yml @@ -4,7 +4,7 @@ networks: services: postgres: - image: postgres:11.5 + image: postgres:13.1 environment: - POSTGRES_USER=airflow - POSTGRES_DB=airflow @@ -36,14 +36,19 @@ services: webserver: env_file: - .env - image: apache/airflow:1.10.11 + image: apache/airflow:1.10.12 ports: - 8080:8080 volumes: - - ./dags:/opt/airflow/dags + - ./airflow_files/dags:/opt/airflow/dags - ./logs:/opt/airflow/logs - ./files:/opt/airflow/files - /var/run/docker.sock:/var/run/docker.sock + deploy: + restart_policy: + condition: on-failure + delay: 8s + max_attempts: 3 depends_on: - postgres - redis @@ -56,54 +61,75 @@ services: networks: - airflow flower: - image: apache/airflow:1.10.11 + image: apache/airflow:1.10.12 env_file: - .env ports: - 5555:5555 depends_on: - redis + deploy: + restart_policy: + condition: on-failure + delay: 8s + max_attempts: 3 volumes: - ./logs:/opt/airflow/logs command: flower networks: - airflow scheduler: - image: apache/airflow:1.10.11 + image: apache/airflow:1.10.12 env_file: - .env volumes: - - ./dags:/opt/airflow/dags + - ./airflow_files/dags:/opt/airflow/dags - ./logs:/opt/airflow/logs - ./files:/opt/airflow/files - /var/run/docker.sock:/var/run/docker.sock command: scheduler + deploy: + restart_policy: + condition: on-failure + delay: 8s + max_attempts: 3 networks: - airflow worker: - image: apache/airflow:1.10.11 + image: apache/airflow:1.10.12 env_file: - .env volumes: - - ./dags:/opt/airflow/dags + - ./airflow_files/dags:/opt/airflow/dags - ./logs:/opt/airflow/logs - ./files:/opt/airflow/files - /var/run/docker.sock:/var/run/docker.sock command: worker depends_on: - scheduler + + deploy: + restart_policy: + condition: on-failure + delay: 8s + max_attempts: 3 networks: - airflow initdb: - image: apache/airflow:1.10.11 + image: apache/airflow:1.10.12 env_file: - .env volumes: - - ./dags:/opt/airflow/dags + - ./airflow_files/dags:/opt/airflow/dags - ./logs:/opt/airflow/logs - ./files:/opt/airflow/files - /var/run/docker.sock:/var/run/docker.sock entrypoint: /bin/bash + deploy: + restart_policy: + condition: on-failure + delay: 8s + max_attempts: 5 command: -c "airflow initdb" depends_on: - redis