Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 8e2ccb3544 | |||
|
|
eebe7df201 | ||
|
|
d2aebe4f70 | ||
|
|
75e1d7411c | ||
|
|
6d2912d9d7 |
37
README.md
37
README.md
@@ -1,5 +1,33 @@
|
|||||||
# How to use
|
# How to use
|
||||||
|
|
||||||
|
[](https://ko-fi.com/Z8Z237B0Q)
|
||||||
|
|
||||||
|
### Apache Airflow version 2.0.0
|
||||||
|
(2.0 not 100% bacward compatible to 1.10+ this is because I move it to separate compose file):
|
||||||
|
|
||||||
|
By default now RBAC is turn on and this mean, that to use Airflow UI you need create user first, for this in db_init service was added also command to create default user:
|
||||||
|
>> *airflow users create --firstname admin --lastname admin --email admin --password admin --username admin --role Admin*
|
||||||
|
|
||||||
|
Change your user password and login as you want. By default it is login: admin, password: admin.
|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
|
|
||||||
|
**[docker-compose-with-celery-executor.yml](docker-compose-2.0-with-celery-executor.yml)**
|
||||||
|
|
||||||
|
**NOTE: if you previous run Airflow 1.10 - remove your DB volume files before run 2.0 or change db init command to db upgrade.**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
|
||||||
|
git clone https://github.com/xnuinside/airflow_in_docker_compose
|
||||||
|
cd airflow_in_docker_compose
|
||||||
|
|
||||||
|
docker-compose -f docker-compose-2.0-with-celery-executor.yml up --build
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
### Apache Airflow version 1.10.14:
|
||||||
```bash
|
```bash
|
||||||
|
|
||||||
git clone https://github.com/xnuinside/airflow_in_docker_compose
|
git clone https://github.com/xnuinside/airflow_in_docker_compose
|
||||||
@@ -42,6 +70,15 @@ Source files for article with description on Medium.
|
|||||||

|

|
||||||
|
|
||||||
|
|
||||||
|
### 18.12.2020:
|
||||||
|
1. Added separate docker-compose file for Apache Airflow 2.0 version
|
||||||
|
|
||||||
|
|
||||||
|
### 16.12.2020:
|
||||||
|
1. Update Apache Airflow version to 1.10.14
|
||||||
|
2. Change init db command to "airflow db init"
|
||||||
|
|
||||||
|
|
||||||
### 29.11.2020:
|
### 29.11.2020:
|
||||||
1. Update Apache Airflow version to 1.10.12
|
1. Update Apache Airflow version to 1.10.12
|
||||||
2. Update PostgreSQL DB to 13.1
|
2. Update PostgreSQL DB to 13.1
|
||||||
|
|||||||
138
docker-compose-2.0-with-celery-executor.yml
Normal file
138
docker-compose-2.0-with-celery-executor.yml
Normal file
@@ -0,0 +1,138 @@
|
|||||||
|
version: '3.2'
|
||||||
|
networks:
|
||||||
|
airflow:
|
||||||
|
|
||||||
|
services:
|
||||||
|
postgres:
|
||||||
|
image: postgres:13.1
|
||||||
|
environment:
|
||||||
|
- POSTGRES_USER=airflow
|
||||||
|
- POSTGRES_DB=airflow
|
||||||
|
- POSTGRES_PASSWORD=airflow
|
||||||
|
- PGDATA=/var/lib/postgresql/data/pgdata
|
||||||
|
ports:
|
||||||
|
- 5432:5432
|
||||||
|
volumes:
|
||||||
|
- /var/run/docker.sock:/var/run/docker.sock
|
||||||
|
- ./database/data:/var/lib/postgresql/data/pgdata
|
||||||
|
- ./database/logs:/var/lib/postgresql/data/log
|
||||||
|
command: >
|
||||||
|
postgres
|
||||||
|
-c listen_addresses=*
|
||||||
|
-c logging_collector=on
|
||||||
|
-c log_destination=stderr
|
||||||
|
-c max_connections=200
|
||||||
|
networks:
|
||||||
|
- airflow
|
||||||
|
redis:
|
||||||
|
image: redis:5.0.5
|
||||||
|
environment:
|
||||||
|
REDIS_HOST: redis
|
||||||
|
REDIS_PORT: 6379
|
||||||
|
ports:
|
||||||
|
- 6379:6379
|
||||||
|
networks:
|
||||||
|
- airflow
|
||||||
|
webserver:
|
||||||
|
env_file:
|
||||||
|
- .env
|
||||||
|
image: apache/airflow:2.0.0-python3.8
|
||||||
|
ports:
|
||||||
|
- 8080:8080
|
||||||
|
volumes:
|
||||||
|
- ./airflow_files/dags:/opt/airflow/dags
|
||||||
|
- ./logs:/opt/airflow/logs
|
||||||
|
- ./files:/opt/airflow/files
|
||||||
|
- /var/run/docker.sock:/var/run/docker.sock
|
||||||
|
deploy:
|
||||||
|
restart_policy:
|
||||||
|
condition: on-failure
|
||||||
|
delay: 8s
|
||||||
|
max_attempts: 3
|
||||||
|
depends_on:
|
||||||
|
- postgres
|
||||||
|
- redis
|
||||||
|
command: webserver
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "[ -f /opt/airflow/airflow-webserver.pid ]"]
|
||||||
|
interval: 30s
|
||||||
|
timeout: 30s
|
||||||
|
retries: 3
|
||||||
|
networks:
|
||||||
|
- airflow
|
||||||
|
flower:
|
||||||
|
image: apache/airflow:2.0.0-python3.8
|
||||||
|
env_file:
|
||||||
|
- .env
|
||||||
|
ports:
|
||||||
|
- 5555:5555
|
||||||
|
depends_on:
|
||||||
|
- redis
|
||||||
|
deploy:
|
||||||
|
restart_policy:
|
||||||
|
condition: on-failure
|
||||||
|
delay: 8s
|
||||||
|
max_attempts: 3
|
||||||
|
volumes:
|
||||||
|
- ./logs:/opt/airflow/logs
|
||||||
|
command: celery flower
|
||||||
|
networks:
|
||||||
|
- airflow
|
||||||
|
scheduler:
|
||||||
|
image: apache/airflow:2.0.0-python3.8
|
||||||
|
env_file:
|
||||||
|
- .env
|
||||||
|
volumes:
|
||||||
|
- ./airflow_files/dags:/opt/airflow/dags
|
||||||
|
- ./logs:/opt/airflow/logs
|
||||||
|
- ./files:/opt/airflow/files
|
||||||
|
- /var/run/docker.sock:/var/run/docker.sock
|
||||||
|
command: scheduler
|
||||||
|
deploy:
|
||||||
|
restart_policy:
|
||||||
|
condition: on-failure
|
||||||
|
delay: 8s
|
||||||
|
max_attempts: 3
|
||||||
|
networks:
|
||||||
|
- airflow
|
||||||
|
worker:
|
||||||
|
image: apache/airflow:2.0.0-python3.8
|
||||||
|
env_file:
|
||||||
|
- .env
|
||||||
|
volumes:
|
||||||
|
- ./airflow_files/dags:/opt/airflow/dags
|
||||||
|
- ./logs:/opt/airflow/logs
|
||||||
|
- ./files:/opt/airflow/files
|
||||||
|
- /var/run/docker.sock:/var/run/docker.sock
|
||||||
|
command: celery worker
|
||||||
|
depends_on:
|
||||||
|
- scheduler
|
||||||
|
|
||||||
|
deploy:
|
||||||
|
restart_policy:
|
||||||
|
condition: on-failure
|
||||||
|
delay: 8s
|
||||||
|
max_attempts: 3
|
||||||
|
networks:
|
||||||
|
- airflow
|
||||||
|
initdb:
|
||||||
|
image: apache/airflow:2.0.0-python3.8
|
||||||
|
env_file:
|
||||||
|
- .env
|
||||||
|
volumes:
|
||||||
|
- ./airflow_files/dags:/opt/airflow/dags
|
||||||
|
- ./logs:/opt/airflow/logs
|
||||||
|
- ./files:/opt/airflow/files
|
||||||
|
- /var/run/docker.sock:/var/run/docker.sock
|
||||||
|
entrypoint: /bin/bash
|
||||||
|
deploy:
|
||||||
|
restart_policy:
|
||||||
|
condition: on-failure
|
||||||
|
delay: 8s
|
||||||
|
max_attempts: 5
|
||||||
|
command: -c "airflow db init && airflow users create --firstname admin --lastname admin --email admin --password admin --username admin --role Admin"
|
||||||
|
depends_on:
|
||||||
|
- redis
|
||||||
|
- postgres
|
||||||
|
networks:
|
||||||
|
- airflow
|
||||||
@@ -36,7 +36,7 @@ services:
|
|||||||
webserver:
|
webserver:
|
||||||
env_file:
|
env_file:
|
||||||
- .env
|
- .env
|
||||||
image: apache/airflow:1.10.12
|
image: apache/airflow:1.10.14
|
||||||
ports:
|
ports:
|
||||||
- 8080:8080
|
- 8080:8080
|
||||||
volumes:
|
volumes:
|
||||||
@@ -61,7 +61,7 @@ services:
|
|||||||
networks:
|
networks:
|
||||||
- airflow
|
- airflow
|
||||||
flower:
|
flower:
|
||||||
image: apache/airflow:1.10.12
|
image: apache/airflow:1.10.14
|
||||||
env_file:
|
env_file:
|
||||||
- .env
|
- .env
|
||||||
ports:
|
ports:
|
||||||
@@ -79,7 +79,7 @@ services:
|
|||||||
networks:
|
networks:
|
||||||
- airflow
|
- airflow
|
||||||
scheduler:
|
scheduler:
|
||||||
image: apache/airflow:1.10.12
|
image: apache/airflow:1.10.14
|
||||||
env_file:
|
env_file:
|
||||||
- .env
|
- .env
|
||||||
volumes:
|
volumes:
|
||||||
@@ -96,7 +96,7 @@ services:
|
|||||||
networks:
|
networks:
|
||||||
- airflow
|
- airflow
|
||||||
worker:
|
worker:
|
||||||
image: apache/airflow:1.10.12
|
image: apache/airflow:1.10.14
|
||||||
env_file:
|
env_file:
|
||||||
- .env
|
- .env
|
||||||
volumes:
|
volumes:
|
||||||
@@ -116,7 +116,7 @@ services:
|
|||||||
networks:
|
networks:
|
||||||
- airflow
|
- airflow
|
||||||
initdb:
|
initdb:
|
||||||
image: apache/airflow:1.10.12
|
image: apache/airflow:1.10.14
|
||||||
env_file:
|
env_file:
|
||||||
- .env
|
- .env
|
||||||
volumes:
|
volumes:
|
||||||
@@ -130,7 +130,7 @@ services:
|
|||||||
condition: on-failure
|
condition: on-failure
|
||||||
delay: 8s
|
delay: 8s
|
||||||
max_attempts: 5
|
max_attempts: 5
|
||||||
command: -c "airflow initdb"
|
command: -c "airflow db init"
|
||||||
depends_on:
|
depends_on:
|
||||||
- redis
|
- redis
|
||||||
- postgres
|
- postgres
|
||||||
|
|||||||
BIN
docs/img/2.0.png
Normal file
BIN
docs/img/2.0.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 188 KiB |
Reference in New Issue
Block a user