Adding RBAC

README.md

@@ -1,6 +1,6 @@
 # Using AWS Fargate(ECS) to host Apache Airflow
 
-This repository contains a sample setup for hosting [Apache Ariflow](https://airflow.apache.org/) on AWS ECS using Fargate.  
+This repository contains a sample setup for hosting [Apache Airflow](https://airflow.apache.org/) on AWS ECS using Fargate.  
 This setup uses [AWS Cloud Development Kit](https://github.com/awslabs/aws-cdk) to automate resource creation.
 
 ## Table of Contents
@@ -28,6 +28,7 @@ This setup is based on AWS CDK. So, install CDK first.
 ### Prerequisites
 1. Node.js => 12.x or later
 2. AWS CLI => [Installation Guide](https://aws.amazon.com/cli/)
+3. Docker
 ```
 $ npm install -g aws-cdk
 ```
@@ -38,12 +39,30 @@ $ npm run build  // Build this package and generates Cloudformation Stack with r
 $ cdk deploy     // Deploys the CloudFormation template to AWS account configured for your AWS CLI
 ```
 This will output LoadBalancerDNSName on the terminal, which can be used to access Airflow Webserver UI.
+Along with LoadBalancerDNSName, you will also get password for Admin account, to login into Airflow UI. You can change this password from Profile section after login.
+For details about Admin config, check `airflow/config/webserver_entry.sh`
 
 If you want to delete this stack, run following command:
 ```
 $ cdk destroy
 ```
 
+## DAG Explanation
+This stack creates a worflow/DAG, which has 5 tasks
+```
+start_process >> [odd_task, even_task] >> numbers_task >> on_worker_task
+```
+This DAG showcases how to create parallel tasks and how to use ECSOperator, which spins-up OnDemand Fargate instances for running a task.
+
+Note: Each sub-folder under `tasks` will result in a new Fargate TaskDefinition. These task definition will be used as part of ECSOperator
+
+start_process: It's a dummy task, which will run on default worker of Airflow
+odd_task: This task will execute `odd_numbers.py` file, which is located under `tasks/multi_task`. This will be executed on an OnDemand Fargate task
+even_task: This task will execute `even_numbers.py` file, which is located under `tasks/multi_task`. This will be executed on an OnDemand Fargate task
+numbers_task: This task will execute `numbers_numbers.py` file, which is located under `tasks/number_task`. This will be executed on an OnDemand Fargate instance
+on_worker_task: This task will be executed on the default worker. It showcases how to use PythonOperator to run a task on Airflow worker 
+
+
 ## Configuration Options <a name="Config"></a>
 Once deployed this setup will create following AWS resources with some necessary dependencies:
 * 1 VPC

airflow/config/webserver_entry.sh

@@ -3,5 +3,9 @@
 set -Eeuxo pipefail
 
 airflow initdb
+sleep 10
+
+airflow create_user -r Admin -u admin -f FirstName -l LastName -p ${ADMIN_PASS} -e admin@test.com
 sleep 5
+
 airflow webserver

app/constructs/airflow-construct.ts

@@ -1,4 +1,4 @@
-import { Construct, CfnOutput } from "@aws-cdk/core";
+import {CfnOutput, Construct} from "@aws-cdk/core";
 import { IVpc } from "@aws-cdk/aws-ec2";
 
 import ecs = require('@aws-cdk/aws-ecs');
@@ -8,6 +8,8 @@ import { FargateTaskDefinition } from '@aws-cdk/aws-ecs';
 
 import {airflowTaskConfig, ContainerConfig} from "../config";
 import { ServiceConstruct } from "./service-construct";
+import { v4 as uuidv4 } from 'uuid';
+
 
 export interface AirflowConstructProps {
   readonly vpc: IVpc;
@@ -18,16 +20,20 @@ export interface AirflowConstructProps {
 }
 
 export class AirflowConstruct extends Construct {
-  public readonly loadBalancerDnsName: CfnOutput;
+  public readonly adminPasswordOutput?: CfnOutput;
 
   constructor(parent: Construct, name: string, props: AirflowConstructProps) {
     super(parent, name);
 
+    const adminPassword = uuidv4();
+
     const ENV_VAR = {
       AIRFLOW__CORE__SQL_ALCHEMY_CONN: props.dbConnection,
       AIRFLOW__CELERY__BROKER_URL: "sqs://",
       AIRFLOW__CELERY__RESULT_BACKEND: "db+" + props.dbConnection,
       AIRFLOW__CORE__EXECUTOR: "CeleryExecutor",
+      AIRFLOW__WEBSERVER__RBAC: "True",
+      ADMIN_PASS: adminPassword,
       CLUSTER: props.cluster.clusterName,
       SECURITY_GROUP: props.defaultVpcSecurityGroup.securityGroupId,
       SUBNETS: props.privateSubnets.map(subnet => subnet.subnetId).join(",")
@@ -95,5 +101,9 @@ export class AirflowConstruct extends Construct {
         isWorkerService: true
       });
     }
+
+    this.adminPasswordOutput = new CfnOutput(this, 'AdminPassword', {
+      value: adminPassword
+    });
   }
 }

app/policies.ts

@@ -10,7 +10,6 @@ export class PolicyConstruct extends Construct {
 
         // Both managed policies and policy statements will be attached to Task Role of Airflow Instance
         this.managedPolicies = [
-            ManagedPolicy.fromAwsManagedPolicyName("IAMFullAccess"),
             ManagedPolicy.fromAwsManagedPolicyName("AmazonSQSFullAccess"),
             ManagedPolicy.fromAwsManagedPolicyName("AmazonECS_FullAccess"),
         ];

package-lock.json

@@ -693,6 +693,11 @@
       "integrity": "sha512-76fupxOYVxk36kb7O/6KtrAPZ9jnSK3+qisAX4tQMEuGNdlvl7ycwatlHqjoE6jHfVtXFM3pCrCixZOidc5cuw==",
       "dev": true
     },
+    "@types/uuid": {
+      "version": "8.3.0",
+      "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.0.tgz",
+      "integrity": "sha512-eQ9qFW/fhfGJF8WKHGEHZEyVWfZxrT+6CLIJGBcZPfxUh/+BnEj+UCGYMlr9qZuX/2AltsvwrGqp0LhEW8D0zQ=="
+    },
     "at-least-node": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz",
@@ -775,6 +780,11 @@
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/universalify/-/universalify-1.0.0.tgz",
       "integrity": "sha512-rb6X1W158d7pRQBg5gkR8uPaSfiids68LTJQYOtEUhoJUWBdaQHsuT/EUduxXYxcrt4r5PJ4fuHW1MHT6p0qug=="
+    },
+    "uuid": {
+      "version": "8.3.1",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.1.tgz",
+      "integrity": "sha512-FOmRr+FmWEIG8uhZv6C2bTgEVXsHk08kE7mPlrBbEe+c3r9pjceVPgupIfNIhc4yx55H69OXANrUaSuu9eInKg=="
     }
   }
 }

package.json

@@ -24,6 +24,8 @@
     "@aws-cdk/aws-ecs-patterns": "*",
     "@aws-cdk/aws-ecr-assets": "*",
     "@aws-cdk/aws-rds": "*",
-    "@aws-cdk/core": "*"
+    "@aws-cdk/core": "*",
+    "@types/uuid": "8.3.0",
+    "uuid": "^8.3.0"
   }
 }