diff --git a/README.md b/README.md index fa64fdc..3db9ef9 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,31 @@ # How to use +**Apache Airflow version 2.0.0** +(2.0 not 100% bacward compatible to 1.10+ this is because I move it to separate compose file): + +By default now RBAC is turn on and this mean, that to use Airflow UI you need create user first, for this in db_init service was added also command to create default user: +>> *airflow users create --firstname admin --lastname admin --email admin --password admin --username admin --role Admin* + +Change your user password and login as you want. By default it is login: admin, password: admin. + +![New Apache Airflow 2.0](/docs/img/2.0.png?raw=true "Apache Airflow 2.0") + + +**[docker-compose-with-celery-executor.yml](docker-compose-2.0-with-celery-executor.yml)** + +**NOTE: if you previous run Airflow 1.10 - remove your DB volume files before run 2.0 or change db init command to db upgrade.** + +```bash + + git clone https://github.com/xnuinside/airflow_in_docker_compose + cd airflow_in_docker_compose + + docker-compose -f docker-compose-2.0-with-celery-executor.yml up --buil + +``` + + +Apache Airflow version 1.10.14: ```bash git clone https://github.com/xnuinside/airflow_in_docker_compose @@ -42,6 +68,10 @@ Source files for article with description on Medium. ![Version](/docs/img/version.png?raw=true "Version Screen") +### 18.12.2020: +1. Added separate docker-compose file for Apache Airflow 2.0 version + + ### 16.12.2020: 1. Update Apache Airflow version to 1.10.14 2. Change init db command to "airflow db init" diff --git a/docker-compose-2.0-with-celery-executor.yml b/docker-compose-2.0-with-celery-executor.yml new file mode 100644 index 0000000..02d5fb4 --- /dev/null +++ b/docker-compose-2.0-with-celery-executor.yml @@ -0,0 +1,138 @@ +version: '3.2' +networks: + airflow: + +services: + postgres: + image: postgres:13.1 + environment: + - POSTGRES_USER=airflow + - POSTGRES_DB=airflow + - POSTGRES_PASSWORD=airflow + - PGDATA=/var/lib/postgresql/data/pgdata + ports: + - 5432:5432 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./database/data:/var/lib/postgresql/data/pgdata + - ./database/logs:/var/lib/postgresql/data/log + command: > + postgres + -c listen_addresses=* + -c logging_collector=on + -c log_destination=stderr + -c max_connections=200 + networks: + - airflow + redis: + image: redis:5.0.5 + environment: + REDIS_HOST: redis + REDIS_PORT: 6379 + ports: + - 6379:6379 + networks: + - airflow + webserver: + env_file: + - .env + image: apache/airflow:2.0.0-python3.8 + ports: + - 8080:8080 + volumes: + - ./airflow_files/dags:/opt/airflow/dags + - ./logs:/opt/airflow/logs + - ./files:/opt/airflow/files + - /var/run/docker.sock:/var/run/docker.sock + deploy: + restart_policy: + condition: on-failure + delay: 8s + max_attempts: 3 + depends_on: + - postgres + - redis + command: webserver + healthcheck: + test: ["CMD-SHELL", "[ -f /opt/airflow/airflow-webserver.pid ]"] + interval: 30s + timeout: 30s + retries: 3 + networks: + - airflow + flower: + image: apache/airflow:2.0.0-python3.8 + env_file: + - .env + ports: + - 5555:5555 + depends_on: + - redis + deploy: + restart_policy: + condition: on-failure + delay: 8s + max_attempts: 3 + volumes: + - ./logs:/opt/airflow/logs + command: celery flower + networks: + - airflow + scheduler: + image: apache/airflow:2.0.0-python3.8 + env_file: + - .env + volumes: + - ./airflow_files/dags:/opt/airflow/dags + - ./logs:/opt/airflow/logs + - ./files:/opt/airflow/files + - /var/run/docker.sock:/var/run/docker.sock + command: scheduler + deploy: + restart_policy: + condition: on-failure + delay: 8s + max_attempts: 3 + networks: + - airflow + worker: + image: apache/airflow:2.0.0-python3.8 + env_file: + - .env + volumes: + - ./airflow_files/dags:/opt/airflow/dags + - ./logs:/opt/airflow/logs + - ./files:/opt/airflow/files + - /var/run/docker.sock:/var/run/docker.sock + command: celery worker + depends_on: + - scheduler + + deploy: + restart_policy: + condition: on-failure + delay: 8s + max_attempts: 3 + networks: + - airflow + initdb: + image: apache/airflow:2.0.0-python3.8 + env_file: + - .env + volumes: + - ./airflow_files/dags:/opt/airflow/dags + - ./logs:/opt/airflow/logs + - ./files:/opt/airflow/files + - /var/run/docker.sock:/var/run/docker.sock + entrypoint: /bin/bash + deploy: + restart_policy: + condition: on-failure + delay: 8s + max_attempts: 5 + command: -c "airflow db init && airflow users create --firstname admin --lastname admin --email admin --password admin --username admin --role Admin" + depends_on: + - redis + - postgres + networks: + - airflow \ No newline at end of file diff --git a/docs/img/2.0.png b/docs/img/2.0.png new file mode 100644 index 0000000..264961c Binary files /dev/null and b/docs/img/2.0.png differ