diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..adad096 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +airflow_files/ +data/ +docs/ +LICENSE +README.md \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 6ae018a..80e0871 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,11 @@ FROM puckel/docker-airflow:1.10.6 RUN pip install --user psycopg2-binary ENV AIRFLOW_HOME=/usr/local/airflow + +# uncomment next 2 lines if you want to use 'docker-compose-volume-packages.yml' +# RUN mkdir /usr/local/airflow/packages +# COPY ./packages.pth /usr/local/lib/python3.7/site-packages + COPY ./airflow.cfg /usr/local/airflow/airflow.cfg -# or ./airflow_celeryexecutor.cfg /usr/local/airflow/airflow.cfg if you need .cfg with CeleryExecutor +# or change to ./airflow_celeryexecutor.cfg /usr/local/airflow/airflow.cfg +# if you need .cfg with CeleryExecutor diff --git a/airflow_files/dags/test_dependency.py b/airflow_files/dags/test_dependency.py new file mode 100644 index 0000000..d955199 --- /dev/null +++ b/airflow_files/dags/test_dependency.py @@ -0,0 +1,11 @@ +from datetime import datetime +from airflow import DAG +from airflow.operators.dummy_operator import DummyOperator + +# uncomment to use for test dependencies installation from packages volume +# from slack import * + +with DAG(dag_id='dependency_dag', + start_date=datetime(2019, 12, 17), + schedule_interval=None) as dag: + DummyOperator(task_id='dummy') diff --git a/docker-compose-volume-packages.yml b/docker-compose-volume-packages.yml new file mode 100644 index 0000000..43cd87b --- /dev/null +++ b/docker-compose-volume-packages.yml @@ -0,0 +1,52 @@ +version: "3" +services: + postgres: + image: "postgres:9.6" + container_name: "postgres" + environment: + - POSTGRES_USER=airflow + - POSTGRES_PASSWORD=airflow + - POSTGRES_DB=airflow + ports: + - "5432:5432" + volumes: + - ./data/postgres:/var/lib/postgresql/data + # uncomment initdb if you need initdb at first run + initdb: + build: . + entrypoint: airflow initdb + depends_on: + - postgres + webserver: + build: . + restart: always + depends_on: + - postgres + volumes: + - ./airflow_files/dags:/usr/local/airflow/dags + - ./airflow_files/logs:/usr/local/airflow/logs + - ./airflow_files/packages:/usr/local/airflow/packages + ports: + - "8080:8080" + entrypoint: airflow webserver + healthcheck: + test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"] + interval: 30s + timeout: 30s + retries: 3 + scheduler: + build: . + restart: always + depends_on: + - postgres + - webserver + volumes: + - ./airflow_files/dags:/usr/local/airflow/dags + - ./airflow_files/logs:/usr/local/airflow/logs + - ./airflow_files/packages:/usr/local/airflow/packages + entrypoint: airflow scheduler + healthcheck: + test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-scheduler.pid ]"] + interval: 30s + timeout: 30s + retries: 3 diff --git a/docker-compose.yml b/docker-compose.yml index 28057c6..427df93 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,7 @@ services: # build: . # entrypoint: airflow initdb # depends_on: - # - postgres + # - postgres webserver: build: . restart: always @@ -40,6 +40,7 @@ services: - webserver volumes: - ./airflow_files/dags:/usr/local/airflow/dags + - ./airflow_files/logs:/usr/local/airflow/logs entrypoint: airflow scheduler healthcheck: test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-scheduler.pid ]"] diff --git a/packages.pth b/packages.pth new file mode 100644 index 0000000..a6a3367 --- /dev/null +++ b/packages.pth @@ -0,0 +1 @@ +/usr/local/airflow/packages \ No newline at end of file