Packages volume (#2)

* added packages volume example

* return comments

.dockerignore

@@ -0,0 +1,5 @@
+airflow_files/
+data/
+docs/
+LICENSE
+README.md

Dockerfile

@@ -1,5 +1,11 @@
 FROM puckel/docker-airflow:1.10.6
 RUN pip install --user psycopg2-binary
 ENV AIRFLOW_HOME=/usr/local/airflow
+
+# uncomment next 2 lines if you want to use 'docker-compose-volume-packages.yml'
+# RUN mkdir /usr/local/airflow/packages
+# COPY ./packages.pth /usr/local/lib/python3.7/site-packages
+
 COPY ./airflow.cfg /usr/local/airflow/airflow.cfg
-# or ./airflow_celeryexecutor.cfg /usr/local/airflow/airflow.cfg if you need .cfg with CeleryExecutor
+# or change to ./airflow_celeryexecutor.cfg /usr/local/airflow/airflow.cfg
+# if you need .cfg with CeleryExecutor

airflow_files/dags/test_dependency.py

@@ -0,0 +1,11 @@
+from datetime import datetime
+from airflow import DAG
+from airflow.operators.dummy_operator import DummyOperator
+
+# uncomment to use for test dependencies installation from packages volume
+# from slack import *
+
+with DAG(dag_id='dependency_dag',
+         start_date=datetime(2019, 12, 17),
+         schedule_interval=None) as dag:
+    DummyOperator(task_id='dummy')

docker-compose-volume-packages.yml

@@ -0,0 +1,52 @@
+version: "3"
+services:
+  postgres:
+    image: "postgres:9.6"
+    container_name: "postgres"
+    environment:
+      - POSTGRES_USER=airflow
+      - POSTGRES_PASSWORD=airflow
+      - POSTGRES_DB=airflow
+    ports:
+      - "5432:5432"
+    volumes:
+      - ./data/postgres:/var/lib/postgresql/data
+  # uncomment initdb if you need initdb at first run
+  initdb:
+    build: .
+    entrypoint: airflow initdb
+    depends_on:
+      - postgres
+  webserver:
+    build: .
+    restart: always
+    depends_on:
+      - postgres
+    volumes:
+      - ./airflow_files/dags:/usr/local/airflow/dags
+      - ./airflow_files/logs:/usr/local/airflow/logs
+      - ./airflow_files/packages:/usr/local/airflow/packages
+    ports:
+      - "8080:8080"
+    entrypoint: airflow webserver
+    healthcheck:
+      test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"]
+      interval: 30s
+      timeout: 30s
+      retries: 3
+  scheduler:
+    build: .
+    restart: always
+    depends_on:
+      - postgres
+      - webserver
+    volumes:
+      - ./airflow_files/dags:/usr/local/airflow/dags
+      - ./airflow_files/logs:/usr/local/airflow/logs
+      - ./airflow_files/packages:/usr/local/airflow/packages
+    entrypoint: airflow scheduler
+    healthcheck:
+      test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-scheduler.pid ]"]
+      interval: 30s
+      timeout: 30s
+      retries: 3

docker-compose.yml

@@ -40,6 +40,7 @@ services:
       - webserver
     volumes:
       - ./airflow_files/dags:/usr/local/airflow/dags
+      - ./airflow_files/logs:/usr/local/airflow/logs
     entrypoint: airflow scheduler
     healthcheck:
       test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-scheduler.pid ]"]

packages.pth

@@ -0,0 +1 @@
+/usr/local/airflow/packages