From ac9f476d30fbc4cd393497efc4b01a102f779806 Mon Sep 17 00:00:00 2001 From: Chaithanya Maisagoni Date: Sat, 14 Nov 2020 16:12:42 -0800 Subject: [PATCH] Adding RBAC --- README.md | 21 ++++++++++++++++++++- airflow/config/webserver_entry.sh | 4 ++++ app/constructs/airflow-construct.ts | 16 +++++++++++++--- app/policies.ts | 1 - package-lock.json | 10 ++++++++++ package.json | 4 +++- 6 files changed, 50 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 1713bd1..b1589d2 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Using AWS Fargate(ECS) to host Apache Airflow -This repository contains a sample setup for hosting [Apache Ariflow](https://airflow.apache.org/) on AWS ECS using Fargate. +This repository contains a sample setup for hosting [Apache Airflow](https://airflow.apache.org/) on AWS ECS using Fargate. This setup uses [AWS Cloud Development Kit](https://github.com/awslabs/aws-cdk) to automate resource creation. ## Table of Contents @@ -28,6 +28,7 @@ This setup is based on AWS CDK. So, install CDK first. ### Prerequisites 1. Node.js => 12.x or later 2. AWS CLI => [Installation Guide](https://aws.amazon.com/cli/) +3. Docker ``` $ npm install -g aws-cdk ``` @@ -38,12 +39,30 @@ $ npm run build // Build this package and generates Cloudformation Stack with r $ cdk deploy // Deploys the CloudFormation template to AWS account configured for your AWS CLI ``` This will output LoadBalancerDNSName on the terminal, which can be used to access Airflow Webserver UI. +Along with LoadBalancerDNSName, you will also get password for Admin account, to login into Airflow UI. You can change this password from Profile section after login. +For details about Admin config, check `airflow/config/webserver_entry.sh` If you want to delete this stack, run following command: ``` $ cdk destroy ``` +## DAG Explanation +This stack creates a worflow/DAG, which has 5 tasks +``` +start_process >> [odd_task, even_task] >> numbers_task >> on_worker_task +``` +This DAG showcases how to create parallel tasks and how to use ECSOperator, which spins-up OnDemand Fargate instances for running a task. + +Note: Each sub-folder under `tasks` will result in a new Fargate TaskDefinition. These task definition will be used as part of ECSOperator + +start_process: It's a dummy task, which will run on default worker of Airflow +odd_task: This task will execute `odd_numbers.py` file, which is located under `tasks/multi_task`. This will be executed on an OnDemand Fargate task +even_task: This task will execute `even_numbers.py` file, which is located under `tasks/multi_task`. This will be executed on an OnDemand Fargate task +numbers_task: This task will execute `numbers_numbers.py` file, which is located under `tasks/number_task`. This will be executed on an OnDemand Fargate instance +on_worker_task: This task will be executed on the default worker. It showcases how to use PythonOperator to run a task on Airflow worker + + ## Configuration Options Once deployed this setup will create following AWS resources with some necessary dependencies: * 1 VPC diff --git a/airflow/config/webserver_entry.sh b/airflow/config/webserver_entry.sh index 4a32243..0aa4644 100755 --- a/airflow/config/webserver_entry.sh +++ b/airflow/config/webserver_entry.sh @@ -3,5 +3,9 @@ set -Eeuxo pipefail airflow initdb +sleep 10 + +airflow create_user -r Admin -u admin -f FirstName -l LastName -p ${ADMIN_PASS} -e admin@test.com sleep 5 + airflow webserver \ No newline at end of file diff --git a/app/constructs/airflow-construct.ts b/app/constructs/airflow-construct.ts index 29bfc6d..508cf62 100644 --- a/app/constructs/airflow-construct.ts +++ b/app/constructs/airflow-construct.ts @@ -1,4 +1,4 @@ -import { Construct, CfnOutput } from "@aws-cdk/core"; +import {CfnOutput, Construct} from "@aws-cdk/core"; import { IVpc } from "@aws-cdk/aws-ec2"; import ecs = require('@aws-cdk/aws-ecs'); @@ -6,8 +6,10 @@ import ec2 = require("@aws-cdk/aws-ec2"); import { DockerImageAsset } from '@aws-cdk/aws-ecr-assets'; import { FargateTaskDefinition } from '@aws-cdk/aws-ecs'; -import { airflowTaskConfig, ContainerConfig } from "../config"; +import {airflowTaskConfig, ContainerConfig} from "../config"; import { ServiceConstruct } from "./service-construct"; +import { v4 as uuidv4 } from 'uuid'; + export interface AirflowConstructProps { readonly vpc: IVpc; @@ -18,16 +20,20 @@ export interface AirflowConstructProps { } export class AirflowConstruct extends Construct { - public readonly loadBalancerDnsName: CfnOutput; + public readonly adminPasswordOutput?: CfnOutput; constructor(parent: Construct, name: string, props: AirflowConstructProps) { super(parent, name); + const adminPassword = uuidv4(); + const ENV_VAR = { AIRFLOW__CORE__SQL_ALCHEMY_CONN: props.dbConnection, AIRFLOW__CELERY__BROKER_URL: "sqs://", AIRFLOW__CELERY__RESULT_BACKEND: "db+" + props.dbConnection, AIRFLOW__CORE__EXECUTOR: "CeleryExecutor", + AIRFLOW__WEBSERVER__RBAC: "True", + ADMIN_PASS: adminPassword, CLUSTER: props.cluster.clusterName, SECURITY_GROUP: props.defaultVpcSecurityGroup.securityGroupId, SUBNETS: props.privateSubnets.map(subnet => subnet.subnetId).join(",") @@ -95,5 +101,9 @@ export class AirflowConstruct extends Construct { isWorkerService: true }); } + + this.adminPasswordOutput = new CfnOutput(this, 'AdminPassword', { + value: adminPassword + }); } } diff --git a/app/policies.ts b/app/policies.ts index 7e5b143..47120c9 100644 --- a/app/policies.ts +++ b/app/policies.ts @@ -10,7 +10,6 @@ export class PolicyConstruct extends Construct { // Both managed policies and policy statements will be attached to Task Role of Airflow Instance this.managedPolicies = [ - ManagedPolicy.fromAwsManagedPolicyName("IAMFullAccess"), ManagedPolicy.fromAwsManagedPolicyName("AmazonSQSFullAccess"), ManagedPolicy.fromAwsManagedPolicyName("AmazonECS_FullAccess"), ]; diff --git a/package-lock.json b/package-lock.json index 6d913b9..328f8d8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -693,6 +693,11 @@ "integrity": "sha512-76fupxOYVxk36kb7O/6KtrAPZ9jnSK3+qisAX4tQMEuGNdlvl7ycwatlHqjoE6jHfVtXFM3pCrCixZOidc5cuw==", "dev": true }, + "@types/uuid": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.0.tgz", + "integrity": "sha512-eQ9qFW/fhfGJF8WKHGEHZEyVWfZxrT+6CLIJGBcZPfxUh/+BnEj+UCGYMlr9qZuX/2AltsvwrGqp0LhEW8D0zQ==" + }, "at-least-node": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz", @@ -775,6 +780,11 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/universalify/-/universalify-1.0.0.tgz", "integrity": "sha512-rb6X1W158d7pRQBg5gkR8uPaSfiids68LTJQYOtEUhoJUWBdaQHsuT/EUduxXYxcrt4r5PJ4fuHW1MHT6p0qug==" + }, + "uuid": { + "version": "8.3.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.1.tgz", + "integrity": "sha512-FOmRr+FmWEIG8uhZv6C2bTgEVXsHk08kE7mPlrBbEe+c3r9pjceVPgupIfNIhc4yx55H69OXANrUaSuu9eInKg==" } } } diff --git a/package.json b/package.json index c8b8055..597f8d2 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,8 @@ "@aws-cdk/aws-ecs-patterns": "*", "@aws-cdk/aws-ecr-assets": "*", "@aws-cdk/aws-rds": "*", - "@aws-cdk/core": "*" + "@aws-cdk/core": "*", + "@types/uuid": "8.3.0", + "uuid": "^8.3.0" } }