commit ecde2a83a2ca451b5676ba4f564861e84577d0dc Author: Chaithanya Maisagoni Date: Wed Aug 12 16:34:16 2020 -0700 [FarFlow] Using AWS Fargate(ECS) to host Apache Airflow diff --git a/README.md b/README.md new file mode 100644 index 0000000..1713bd1 --- /dev/null +++ b/README.md @@ -0,0 +1,161 @@ +# Using AWS Fargate(ECS) to host Apache Airflow + +This repository contains a sample setup for hosting [Apache Ariflow](https://airflow.apache.org/) on AWS ECS using Fargate. +This setup uses [AWS Cloud Development Kit](https://github.com/awslabs/aws-cdk) to automate resource creation. + +## Table of Contents +1. [What is AWS Fargate](#FargateIntro) +2. [What is Airflow](#AirflowIntro) +3. [How to use this?](#setup) +4. [Configuration Options](#Config) +5. [Understanding Code Structure](#explore) +6. [Some Useful Resources](#resources) + +## What is AWS Fargate +``` +AWS Fargate is a serverless compute engine for containers that works with both Amazon Elastic Container Service (ECS) and Amazon Elastic Kubernetes Service (EKS). Fargate makes it easy for you to focus on building your applications. Fargate removes the need to provision and manage servers, lets you specify and pay for resources per application, and improves security through application isolation by design. +``` +[Source](https://aws.amazon.com/fargate/) + +## What is Airflow +``` +Airflow is a platform created by the community to programmatically author, schedule and monitor workflows. +``` +More info about Airflow can be found [here](https://airflow.apache.org/) + +## How to use this? +This setup is based on AWS CDK. So, install CDK first. +### Prerequisites +1. Node.js => 12.x or later +2. AWS CLI => [Installation Guide](https://aws.amazon.com/cli/) +``` +$ npm install -g aws-cdk +``` +Once Node.js and CDK are installed, pull this repository and run following commands: +``` +$ npm install // Installs all necessary dependencies +$ npm run build // Build this package and generates Cloudformation Stack with resources +$ cdk deploy // Deploys the CloudFormation template to AWS account configured for your AWS CLI +``` +This will output LoadBalancerDNSName on the terminal, which can be used to access Airflow Webserver UI. + +If you want to delete this stack, run following command: +``` +$ cdk destroy +``` + +## Configuration Options +Once deployed this setup will create following AWS resources with some necessary dependencies: +* 1 VPC +* 1 Postgres DB on AWS RDS +* 1 ECS Cluster +* 3 Fargate Task Definitions +* 1 Fargate ECS Service with 3 task instances with one container each for Airflow Webserver, Scheduler and Worker +* 1 EC2 NetworkLoadBalancer +* 1 SecurityGroup: this will be used to restrict access to all of the above resources to VPC. Only webserver can be accessed from outside, using load balancer DNS name + +You can find default config in `config.ts` file. + +### Default Postgres RDS config +``` +export const defaultDBConfig: DBConfig = { + dbName: "farflow", // DB cluster and instance Name + port: 5432, // Port on which db instance runs + masterUsername: "airflow", // Username for master-user. Password will be autogenerated and stored in ParameterStore + instanceType: InstanceType.of(InstanceClass.T2, InstanceSize.SMALL), // Using T2.small for this setup. Upgrade as per your requirements + allocatedStorageInGB: 25, // 25GB of storeage will be allocated + backupRetentionInDays: 30 // Backup will be deleted after 30 days +}; +``` +### Fargate Config +``` +export const defaultWebserverConfig: ContainerConfig = { + name: "WebserverContainer", + containerPort: 8080, + entryPoint: "/webserver_entry.sh" +} + +export const defaultSchedulerConfig: ContainerConfig = { + name: "SchedulerContainer", + containerPort: 8081, + entryPoint: "/scheduler_entry.sh" +} + +export const defaultWorkerConfig: ContainerConfig = { + name: "WorkerContainer", + containerPort: 8082, + entryPoint: "/worker_entry.sh" +} + +export const airflowTaskConfig: AirflowTaskConfig = { + cpu: 2048, + memoryLimitMiB: 4096, + webserverConfig: defaultWebserverConfig, + schedulerConfig: defaultSchedulerConfig, + workerConfig: defaultWorkerConfig, + logRetention: RetentionDays.ONE_MONTH, + // Uncomment this to have dedicated worker pool that can be auto-scaled as per workerAutoScalingConfig + // createWorkerPool: true +}; +``` +Adjust configuration in this file, as per your requirements. +If you need a dedicated Worker pool for resource intense operations that need to available all the time, set `createWorkerPool: true` under `airflowTaskConfig`. +This will create a separate ECS Service which holds Task/Container which holds Airflow Worker. This is configured with auto-scaling to add more task instances, depending on load. +Auto-scaling config can be set as follows: +``` +export const workerAutoScalingConfig: AutoScalingConfig = { + minTaskCount: 1, + maxTaskCount: 5, + cpuUsagePercent: 70 +}; +``` +### AWS Account and Region Setting +This setup uses default account and region that were used in AWS CLI configuration. +In order to install it in different/multiple region or account, follow [this guide](https://docs.aws.amazon.com/cdk/latest/guide/environments.html) + +## Understanding Code Structure +Let's understand the code structure and what each file does. Hope this helps to change things as required +``` +📦FarFlow + ┣ 📂airflow => Top-level directory that holds Airflow related config + ┃ ┣ 📂config + ┃ ┃ ┣ 📜scheduler_entry.sh => Entrypoint for Scheduler Container. + ┃ ┃ ┣ 📜webserver_entry.sh => Entrypoint for Webserver Container. This also initializes backedn database + ┃ ┃ ┗ 📜worker_entry.sh => Entrypoint for Worker Container. + ┃ ┣ 📂dags => Holds all the DAGs for this Airflow instance. Add more DAGs here. + ┃ ┃ ┗ 📜dag.py => Sample DAG + ┃ ┗ 📜Dockerfile => Dockerfile for Airflow Image, with some dependencies + ┣ 📂app => Base folder for CDK application + ┃ ┣ 📂constructs => Holds helper files for CDK setup + ┃ ┃ ┣ 📜airflow-construct.ts => Creates Fargate Service holding Airflow + ┃ ┃ ┣ 📜dag-tasks.ts => Creates fargate tasks containing modules invoked from DAG using ECSOperator + ┃ ┃ ┣ 📜rds.ts => Creates RDS Postgres instance + ┃ ┃ ┣ 📜service-construct.ts => Top level Fargate service helper + ┃ ┃ ┗ 📜task-construct.ts => Helper for Dag-tasks Construct + ┃ ┣ 📜config.ts => Configuration for entire CDK application + ┃ ┣ 📜farflow.ts => Starting point for this CDK application + ┃ ┗ 📜policies.ts => Configure policies that will be attached to Airflow instances + ┣ 📂tasks => Sample tasks that will be invoked from DAG using ECSOperator + ┃ ┣ 📂multi_task => example task 1 + ┃ ┃ ┣ 📜Dockerfile => config for container holding this task + ┃ ┃ ┣ 📜even_numbers.py => module-1 in this container + ┃ ┃ ┗ 📜odd_numbers.py => module-2 in this container + ┃ ┗ 📂number_task => example task 2 + ┃ ┃ ┣ 📜Dockerfile => config for container holding this task + ┃ ┃ ┗ 📜numbers.py => only module for this container + ┣ 📜README.md => YOU ARE READIN IT + ┣ 📜cdk.json => CDK config + ┣ 📜package-lock.json => npm package info (auto-generated) + ┣ 📜package.json => npm dependencies + ┗ 📜tsconfig.json => Typescript config + ``` + Code-tree generated using [this plugin](https://marketplace.visualstudio.com/items?itemName=Shinotatwu-DS.file-tree-generator) + + ## Some Useful Resources + * [CDK](https://docs.aws.amazon.com/cdk/latest/guide/getting_started.html) + * [Airflow Docker](https://hub.docker.com/r/apache/airflow) + * [CDK Examples](https://github.com/aws-samples/aws-cdk-examples) + * [ECS Operator](https://airflow.readthedocs.io/en/latest/_api/airflow/providers/amazon/aws/operators/ecs/index.html?highlight=ECSOperator#airflow.providers.amazon.aws.operators.ecs.ECSOperator): This is the basis for on-demand Fargate tasks + * [Airflow Config and Environment Variables](https://airflow.apache.org/docs/stable/configurations-ref.html): Pass Environment Variables to Docker + * [Default Airflow Config](https://github.com/apache/airflow/blob/master/airflow/config_templates/default_airflow.cfg) + \ No newline at end of file diff --git a/airflow/Dockerfile b/airflow/Dockerfile new file mode 100755 index 0000000..f52846b --- /dev/null +++ b/airflow/Dockerfile @@ -0,0 +1,26 @@ +FROM apache/airflow:1.10.11 + +ENV AIRFLOW_HOME=/usr/local/airflow + +USER root + +RUN apt-get update && apt-get install -y python3-pip \ + libcurl4-gnutls-dev \ + librtmp-dev \ + python3-dev \ + libpq-dev + +RUN python3 -m pip install PyGreSQL argcomplete pycurl + +COPY ./config/* / +COPY ./dags ${AIRFLOW_HOME}/dags + +RUN chown -R airflow: ${AIRFLOW_HOME} + +EXPOSE 8080 + +USER airflow + +WORKDIR ${AIRFLOW_HOME} + +# ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file diff --git a/airflow/config/scheduler_entry.sh b/airflow/config/scheduler_entry.sh new file mode 100755 index 0000000..ad8374f --- /dev/null +++ b/airflow/config/scheduler_entry.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -Eeuxo pipefail +sleep 30 +airflow scheduler \ No newline at end of file diff --git a/airflow/config/webserver_entry.sh b/airflow/config/webserver_entry.sh new file mode 100755 index 0000000..4a32243 --- /dev/null +++ b/airflow/config/webserver_entry.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -Eeuxo pipefail + +airflow initdb +sleep 5 +airflow webserver \ No newline at end of file diff --git a/airflow/config/worker_entry.sh b/airflow/config/worker_entry.sh new file mode 100755 index 0000000..b41238c --- /dev/null +++ b/airflow/config/worker_entry.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -Eeuxo pipefail +sleep 30 +airflow worker \ No newline at end of file diff --git a/airflow/dags/dag.py b/airflow/dags/dag.py new file mode 100644 index 0000000..f576976 --- /dev/null +++ b/airflow/dags/dag.py @@ -0,0 +1,110 @@ +import os +import sys +from datetime import datetime +from datetime import timedelta +from pprint import pprint + +from airflow import DAG + +from airflow.operators.dummy_operator import DummyOperator +from airflow.operators.python_operator import PythonOperator +from airflow.contrib.operators.ecs_operator import ECSOperator + +DAG_NAME = 'Test_Dag' + +default_args = { + 'owner': 'CM', + 'start_date': datetime(2019, 6, 8), + 'email': ['xyz@amazon.com'], + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 3, + 'retry_delay': timedelta(minutes=1) +} + + +def get_ecs_operator_args(taskDefinitionName, taskContainerName, entryFile, param): + return dict( + launch_type="FARGATE", + # The name of your task as defined in ECS + task_definition=taskDefinitionName, + # The name of your ECS cluster + cluster=os.environ['CLUSTER'], + network_configuration={ + 'awsvpcConfiguration': { + 'securityGroups': [os.environ['SECURITY_GROUP']], + 'subnets': os.environ['SUBNETS'].split(","), + 'assignPublicIp': "DISABLED" + } + }, + overrides={ + 'containerOverrides': [ + { + 'name': taskContainerName, + 'command': ["python", entryFile, param] + } + ] + } + ) + +oddTaskConfig = { + 'taskDefinitionName': 'FarFlowCombinedTask', + 'taskContainerName': 'MultiTaskContainer', + 'entryFile': 'odd_numbers.py', + 'param': '10' +} +evenTaskConfig = { + 'taskDefinitionName': 'FarFlowCombinedTask', + 'taskContainerName': 'MultiTaskContainer', + 'entryFile': 'even_numbers.py', + 'param': '10' +} +numbersTaskConfig = { + 'taskDefinitionName': 'FarFlowNumbersTask', + 'taskContainerName': 'NumbersContainer', + 'entryFile': 'numbers.py', + 'param': '10' +} + +oddTask_args = get_ecs_operator_args(**oddTaskConfig) +evenTask_args = get_ecs_operator_args(**evenTaskConfig) +numbersTask_args = get_ecs_operator_args(**numbersTaskConfig) + +dag = DAG( DAG_NAME, + schedule_interval=None, + default_args=default_args) + +start_process = DummyOperator(task_id="start_process", dag=dag) + +# Following tasks will get triggered from worker and runs on OnDemand Fargate Task +odd_task = ECSOperator(task_id="odd_task", **oddTask_args, dag=dag) +even_task = ECSOperator(task_id="even_task", **evenTask_args, dag=dag) +numbers_task = ECSOperator(task_id="numbers_task", **numbersTask_args, dag=dag) + + +# [START howto_operator_python] +# Pulled from : https://github.com/apache/airflow/blob/master/airflow/example_dags/example_python_operator.py#L40 +def print_context(ds, **kwargs): + """Print the Airflow context and ds variable from the context.""" + pprint(kwargs) + print(ds) + return 'Whatever you return gets printed in the logs' + + +task_config = { + "key1": "value1", + "key2": "value2", + "key3": "value3", + "key4": "value4" +} + +on_worker_task = PythonOperator( + task_id='runs_on_worker', + python_callable=print_context, + dag=dag, + op_args=[task_config] +) +# [END howto_operator_python] + + +start_process >> [odd_task, even_task] >> numbers_task >> on_worker_task diff --git a/app/config.ts b/app/config.ts new file mode 100644 index 0000000..547b95c --- /dev/null +++ b/app/config.ts @@ -0,0 +1,73 @@ +import { DBConfig } from "./constructs/rds"; +import {InstanceClass, InstanceSize, InstanceType} from "@aws-cdk/aws-ec2"; +import { RetentionDays } from "@aws-cdk/aws-logs"; + +export interface AirflowTaskConfig { + readonly cpu: number; + readonly memoryLimitMiB: number; + readonly webserverConfig: ContainerConfig; + readonly schedulerConfig: ContainerConfig; + readonly workerConfig: ContainerConfig; + readonly logRetention: RetentionDays; + readonly createWorkerPool?: boolean; +} + +export interface AutoScalingConfig { + readonly maxTaskCount: number; + readonly minTaskCount: number; + readonly cpuUsagePercent?: number; + readonly memUsagePercent?: number; +} + +export interface ContainerConfig { + readonly name: string; + readonly cpu?: number; + readonly memoryLimitMiB?: number; + readonly containerPort: number; + readonly entryPoint: string; +} + +export const workerAutoScalingConfig: AutoScalingConfig = { + minTaskCount: 1, + maxTaskCount: 5, + cpuUsagePercent: 70 +}; + +export const defaultWebserverConfig: ContainerConfig = { + name: "WebserverContainer", + containerPort: 8080, + entryPoint: "/webserver_entry.sh" +} + +export const defaultSchedulerConfig: ContainerConfig = { + name: "SchedulerContainer", + containerPort: 8081, + entryPoint: "/scheduler_entry.sh" +} + +export const defaultWorkerConfig: ContainerConfig = { + name: "WorkerContainer", + containerPort: 8082, + entryPoint: "/worker_entry.sh" +} + +export const airflowTaskConfig: AirflowTaskConfig = { + cpu: 2048, + memoryLimitMiB: 4096, + webserverConfig: defaultWebserverConfig, + schedulerConfig: defaultSchedulerConfig, + workerConfig: defaultWorkerConfig, + logRetention: RetentionDays.ONE_MONTH, + // Uncomment this to have dedicated worker pool that can be auto-scaled as per workerAutoScalingConfig + // createWorkerPool: true +}; + +export const defaultDBConfig: DBConfig = { + dbName: "farflow", + port: 5432, + masterUsername: "airflow", + instanceType: InstanceType.of(InstanceClass.T2, InstanceSize.SMALL), + allocatedStorageInGB: 25, + backupRetentionInDays: 30 +}; + diff --git a/app/constructs/airflow-construct.ts b/app/constructs/airflow-construct.ts new file mode 100644 index 0000000..29bfc6d --- /dev/null +++ b/app/constructs/airflow-construct.ts @@ -0,0 +1,99 @@ +import { Construct, CfnOutput } from "@aws-cdk/core"; +import { IVpc } from "@aws-cdk/aws-ec2"; + +import ecs = require('@aws-cdk/aws-ecs'); +import ec2 = require("@aws-cdk/aws-ec2"); +import { DockerImageAsset } from '@aws-cdk/aws-ecr-assets'; +import { FargateTaskDefinition } from '@aws-cdk/aws-ecs'; + +import { airflowTaskConfig, ContainerConfig } from "../config"; +import { ServiceConstruct } from "./service-construct"; + +export interface AirflowConstructProps { + readonly vpc: IVpc; + readonly cluster: ecs.ICluster; + readonly dbConnection: string; + readonly defaultVpcSecurityGroup: ec2.ISecurityGroup; + readonly privateSubnets: ec2.ISubnet[]; +} + +export class AirflowConstruct extends Construct { + public readonly loadBalancerDnsName: CfnOutput; + + constructor(parent: Construct, name: string, props: AirflowConstructProps) { + super(parent, name); + + const ENV_VAR = { + AIRFLOW__CORE__SQL_ALCHEMY_CONN: props.dbConnection, + AIRFLOW__CELERY__BROKER_URL: "sqs://", + AIRFLOW__CELERY__RESULT_BACKEND: "db+" + props.dbConnection, + AIRFLOW__CORE__EXECUTOR: "CeleryExecutor", + CLUSTER: props.cluster.clusterName, + SECURITY_GROUP: props.defaultVpcSecurityGroup.securityGroupId, + SUBNETS: props.privateSubnets.map(subnet => subnet.subnetId).join(",") + }; + + const logging = new ecs.AwsLogDriver({ + streamPrefix: 'FarFlowLogging', + logRetention: airflowTaskConfig.logRetention + }); + + // Build Airflow docker image from Dockerfile + const airflowImageAsset = new DockerImageAsset(this, 'AirflowBuildImage', { + directory: './airflow', + }); + + const airflowTask = new FargateTaskDefinition(this, 'AirflowTask', { + cpu: airflowTaskConfig.cpu, + memoryLimitMiB: airflowTaskConfig.memoryLimitMiB + }); + + let workerTask = airflowTask; + if (airflowTaskConfig.createWorkerPool) { + workerTask = new FargateTaskDefinition(this, 'WorkerTask', { + cpu: airflowTaskConfig.cpu, + memoryLimitMiB: airflowTaskConfig.memoryLimitMiB + }); + } + + let mmap = new Map(); + mmap.set(airflowTaskConfig.webserverConfig, airflowTask); + mmap.set(airflowTaskConfig.schedulerConfig, airflowTask); + mmap.set(airflowTaskConfig.workerConfig, workerTask); + + // Add containers to corresponding Tasks + for (let entry of mmap.entries()) { + let containerInfo: ContainerConfig = entry[0]; + let task: FargateTaskDefinition = entry[1]; + + task.addContainer(containerInfo.name, { + image: ecs.ContainerImage.fromDockerImageAsset(airflowImageAsset), + logging: logging, + environment: ENV_VAR, + entryPoint: [containerInfo.entryPoint], + cpu: containerInfo.cpu, + memoryLimitMiB: containerInfo.cpu + }).addPortMappings({ + containerPort: containerInfo.containerPort + }); + } + + new ServiceConstruct(this, "AirflowService", { + cluster: props.cluster, + defaultVpcSecurityGroup: props.defaultVpcSecurityGroup, + vpc: props.vpc, + taskDefinition: airflowTask, + isWorkerService: false + }); + + if (airflowTaskConfig.createWorkerPool) { + new ServiceConstruct(this, "WorkerService", { + cluster: props.cluster, + defaultVpcSecurityGroup: props.defaultVpcSecurityGroup, + vpc: props.vpc, + taskDefinition: workerTask, + isWorkerService: true + }); + } + } +} diff --git a/app/constructs/dag-tasks.ts b/app/constructs/dag-tasks.ts new file mode 100644 index 0000000..f71f201 --- /dev/null +++ b/app/constructs/dag-tasks.ts @@ -0,0 +1,44 @@ +import { Construct } from "@aws-cdk/core"; +import { AwsLogDriver } from "@aws-cdk/aws-ecs"; +import { RetentionDays } from "@aws-cdk/aws-logs"; + +import { AirflowDagTaskDefinition } from "./task-construct" + +export class DagTasks extends Construct { + + constructor( + scope: Construct, + taskName: string, + ) { + super(scope, taskName + "-TaskConstruct"); + + const logging = new AwsLogDriver({ + streamPrefix: 'FarFlowDagTaskLogging', + logRetention: RetentionDays.ONE_MONTH + }); + + // Task Container with multiple python executables + new AirflowDagTaskDefinition(this, 'FarFlowCombinedTask', { + containerInfo: { + assetDir: "./tasks/multi_task", + name: "MultiTaskContainer" + }, + cpu: 512, + memoryLimitMiB: 1024, + taskFamilyName: "FarFlowCombinedTask", + logging: logging + }); + + // Task Container with single python executable + new AirflowDagTaskDefinition(this, 'FarFlowNumbersTask', { + containerInfo: { + assetDir: "./tasks/number_task", + name: "NumbersContainer" + }, + cpu: 256, + memoryLimitMiB: 512, + taskFamilyName: "FarFlowNumbersTask", + logging: logging + }); + } +} diff --git a/app/constructs/rds.ts b/app/constructs/rds.ts new file mode 100644 index 0000000..319f547 --- /dev/null +++ b/app/constructs/rds.ts @@ -0,0 +1,95 @@ +import { Duration, Construct } from "@aws-cdk/core"; +import { + DatabaseInstance, + DatabaseInstanceEngine, + StorageType +} from "@aws-cdk/aws-rds"; +import { ISecret, Secret } from "@aws-cdk/aws-secretsmanager"; +import { + InstanceType, + ISecurityGroup, + IVpc, + SubnetType +} from "@aws-cdk/aws-ec2"; + +import { defaultDBConfig } from "../config"; + +export interface DBConfig { + readonly dbName: string; + readonly masterUsername: string; + readonly port: number; + readonly instanceType: InstanceType; + readonly allocatedStorageInGB: number; + readonly backupRetentionInDays: number; +} + +export interface RDSConstructProps { + readonly vpc: IVpc; + readonly defaultVpcSecurityGroup: ISecurityGroup; + readonly dbConfig?: DBConfig; +} + +export class RDSConstruct extends Construct { + public readonly dbConnection: string; + public readonly rdsInstance: DatabaseInstance; + + constructor(parent: Construct, name: string, props: RDSConstructProps) { + super(parent, name); + + const backendSecret: ISecret = new Secret(this, "DatabseSecret", { + secretName: name + "Secret", + description: "airflow RDS secrets", + generateSecretString: { + secretStringTemplate: JSON.stringify({ + username: defaultDBConfig.masterUsername + }), + generateStringKey: "password", + excludeUppercase: false, + requireEachIncludedType: false, + includeSpace: false, + excludePunctuation: true, + excludeLowercase: false, + excludeNumbers: false, + passwordLength: 16 + } + }); + + const databasePasswordSecret = backendSecret.secretValueFromJson( + "password" + ); + + this.rdsInstance = new DatabaseInstance(this, "RDSInstance", { + engine: DatabaseInstanceEngine.POSTGRES, + instanceType: defaultDBConfig.instanceType, + instanceIdentifier: defaultDBConfig.dbName, + vpc: props.vpc, + securityGroups: [props.defaultVpcSecurityGroup], + vpcPlacement: { subnetType: SubnetType.PRIVATE }, + storageEncrypted: true, + multiAz: false, + autoMinorVersionUpgrade: false, + allocatedStorage: defaultDBConfig.allocatedStorageInGB, + storageType: StorageType.GP2, + backupRetention: Duration.days(defaultDBConfig.backupRetentionInDays), + deletionProtection: false, + masterUsername: defaultDBConfig.masterUsername, + databaseName: defaultDBConfig.dbName, + masterUserPassword: databasePasswordSecret, + port: defaultDBConfig.port + }); + + this.dbConnection = this.getDBConnection( + defaultDBConfig, + this.rdsInstance.dbInstanceEndpointAddress, + databasePasswordSecret.toString() + ); + } + + public getDBConnection( + dbConfig: DBConfig, + endpoint: string, + password: string + ): string { + return `postgresql+pygresql://${dbConfig.masterUsername}:${password}@${endpoint}:${dbConfig.port}/${dbConfig.dbName}`; + } +} diff --git a/app/constructs/service-construct.ts b/app/constructs/service-construct.ts new file mode 100644 index 0000000..10899ca --- /dev/null +++ b/app/constructs/service-construct.ts @@ -0,0 +1,115 @@ +import { Construct, CfnOutput, Duration } from "@aws-cdk/core"; +import { IVpc } from "@aws-cdk/aws-ec2"; + +import ecs = require('@aws-cdk/aws-ecs'); +import ec2 = require("@aws-cdk/aws-ec2"); +import elbv2 = require("@aws-cdk/aws-elasticloadbalancingv2"); +import { FargateTaskDefinition } from '@aws-cdk/aws-ecs'; + +import { PolicyConstruct } from "../policies"; +import { workerAutoScalingConfig } from "../config"; + +export interface ServiceConstructProps { + readonly vpc: IVpc; + readonly cluster: ecs.ICluster; + readonly defaultVpcSecurityGroup: ec2.ISecurityGroup; + readonly taskDefinition: FargateTaskDefinition; + readonly isWorkerService?: boolean; +} + +export class ServiceConstruct extends Construct { + private readonly fargateService: ecs.FargateService; + public readonly loadBalancerDnsName?: CfnOutput; + + constructor(parent: Construct, name: string, props: ServiceConstructProps) { + super(parent, name); + + // Attach required policies to Task Role + let policies = new PolicyConstruct(this, "AIrflowTaskPolicies"); + if (policies.managedPolicies) { + policies.managedPolicies.forEach(managedPolicy => props.taskDefinition.taskRole.addManagedPolicy(managedPolicy)); + } + if (policies.policyStatements) { + policies.policyStatements.forEach(policyStatement => props.taskDefinition.taskRole.addToPolicy(policyStatement)); + } + + // Create Fargate Service for Airflow + this.fargateService = new ecs.FargateService(this, name, { + cluster: props.cluster, + taskDefinition: props.taskDefinition, + securityGroup: props.defaultVpcSecurityGroup + }); + const allowedPorts = new ec2.Port({ + protocol: ec2.Protocol.TCP, + fromPort: 0, + toPort: 65535, + stringRepresentation: "All" + }); + this.fargateService.connections.allowFromAnyIpv4(allowedPorts); + + if (props.isWorkerService) { + this.configureAutoScaling(); + } + else { + // Export Load Balancer DNS Name, which will be used to access Airflow UI + this.loadBalancerDnsName = new CfnOutput(this, 'LoadBalanceDNSName', { + value: this.attachLoadBalancer(props.vpc), + }); + } + } + + private attachLoadBalancer(vpc: IVpc): string { + let loadBalancer = new elbv2.NetworkLoadBalancer( + this, + "NetworkLoadBalancer", + { + vpc: vpc, + internetFacing: true, + crossZoneEnabled: true + } + ); + + const listener = loadBalancer.addListener("Listener", { + port: 80 + }); + + const targetGroup = listener.addTargets( + "AirflowFargateServiceTargetGroup", + { + healthCheck: { + port: "traffic-port", + protocol: elbv2.Protocol.HTTP, + path: "/health" + }, + port: 80, + targets: [this.fargateService] + } + ); + targetGroup.setAttribute("deregistration_delay.timeout_seconds", "60"); + + return loadBalancer.loadBalancerDnsName; + } + + private configureAutoScaling(): void { + const scaling = this.fargateService.autoScaleTaskCount({ + maxCapacity: workerAutoScalingConfig.maxTaskCount, + minCapacity: workerAutoScalingConfig.minTaskCount + }); + + if (workerAutoScalingConfig.cpuUsagePercent) { + scaling.scaleOnCpuUtilization("CpuScaling", { + targetUtilizationPercent: workerAutoScalingConfig.cpuUsagePercent, + scaleInCooldown: Duration.seconds(60), + scaleOutCooldown: Duration.seconds(60) + }); + } + + if (workerAutoScalingConfig.memUsagePercent) { + scaling.scaleOnMemoryUtilization("MemoryScaling", { + targetUtilizationPercent: workerAutoScalingConfig.memUsagePercent, + scaleInCooldown: Duration.seconds(60), + scaleOutCooldown: Duration.seconds(60) + }); + } + } +} diff --git a/app/constructs/task-construct.ts b/app/constructs/task-construct.ts new file mode 100644 index 0000000..f2a024e --- /dev/null +++ b/app/constructs/task-construct.ts @@ -0,0 +1,46 @@ +import { Construct } from "@aws-cdk/core"; + +import ecs = require('@aws-cdk/aws-ecs'); +import { DockerImageAsset } from '@aws-cdk/aws-ecr-assets'; +import { FargateTaskDefinition } from '@aws-cdk/aws-ecs'; + +export interface AirflowDagTaskDefinitionProps { + readonly taskFamilyName: string; + readonly containerInfo: ContainerInfo; + readonly cpu: number; + readonly memoryLimitMiB: number; + readonly logging: ecs.LogDriver; +} + +export interface ContainerInfo { + readonly name: string; + readonly assetDir: string; +} + +export class AirflowDagTaskDefinition extends Construct { + + constructor( + scope: Construct, + taskName: string, + props: AirflowDagTaskDefinitionProps + ) { + super(scope, taskName + "-TaskConstruct"); + + // Create a new task with given requirements + const workerTask = new FargateTaskDefinition(this, taskName + '-TaskDef', { + cpu: props.cpu, + memoryLimitMiB: props.memoryLimitMiB, + family: props.taskFamilyName + }); + + const workerImageAsset = new DockerImageAsset(this, props.containerInfo.name + '-BuildImage', { + directory: props.containerInfo.assetDir, + }); + + workerTask.addContainer(props.containerInfo.name, { + image: ecs.ContainerImage.fromDockerImageAsset(workerImageAsset), + logging: props.logging + }); + + } +} diff --git a/app/farflow.ts b/app/farflow.ts new file mode 100644 index 0000000..25ed245 --- /dev/null +++ b/app/farflow.ts @@ -0,0 +1,45 @@ +import ec2 = require('@aws-cdk/aws-ec2'); +import ecs = require('@aws-cdk/aws-ecs'); +import cdk = require('@aws-cdk/core'); +import {RDSConstruct} from "./constructs/rds"; +import {AirflowConstruct} from "./constructs/airflow-construct"; +import { DagTasks } from './constructs/dag-tasks'; + +class FarFlow extends cdk.Stack { + + constructor(scope: cdk.App, id: string, props?: cdk.StackProps) { + super(scope, id, props); + + // Create VPC and Fargate Cluster + // NOTE: Limit AZs to avoid reaching resource quotas + let vpc = new ec2.Vpc(this, 'Vpc', { maxAzs: 2 }); + let cluster = new ecs.Cluster(this, 'ECSCluster', { vpc: vpc }); + + // Setting default SecurityGroup to use across all the resources + let defaultVpcSecurityGroup = new ec2.SecurityGroup(this, "SecurityGroup", {vpc: vpc}); + + // Create RDS instance for Airflow backend + const rds = new RDSConstruct(this, "RDS-Postgres", { + defaultVpcSecurityGroup: defaultVpcSecurityGroup, + vpc: vpc + }); + + // Create Airflow service: Webserver, Scheduler and minimal Worker + new AirflowConstruct(this, "AirflowService", { + cluster: cluster, + vpc: vpc, + dbConnection: rds.dbConnection, + defaultVpcSecurityGroup: defaultVpcSecurityGroup, + privateSubnets: vpc.privateSubnets + }); + + // Create TaskDefinitions for on-demand Fargate tasks, invoked from DAG + new DagTasks(this, "DagTasks"); + } +} + +const app = new cdk.App(); + +new FarFlow(app, 'FarFlow'); + +app.synth(); diff --git a/app/policies.ts b/app/policies.ts new file mode 100644 index 0000000..7e5b143 --- /dev/null +++ b/app/policies.ts @@ -0,0 +1,36 @@ +import { Construct } from "@aws-cdk/core"; +import { IManagedPolicy, ManagedPolicy, PolicyStatement } from "@aws-cdk/aws-iam"; + +export class PolicyConstruct extends Construct { + public readonly policyStatements?: PolicyStatement[]; + public readonly managedPolicies?: IManagedPolicy[]; + + constructor(app: Construct, name: string,) { + super(app, name); + + // Both managed policies and policy statements will be attached to Task Role of Airflow Instance + this.managedPolicies = [ + ManagedPolicy.fromAwsManagedPolicyName("IAMFullAccess"), + ManagedPolicy.fromAwsManagedPolicyName("AmazonSQSFullAccess"), + ManagedPolicy.fromAwsManagedPolicyName("AmazonECS_FullAccess"), + ]; + + /* + You can add custom Policy Statements as well. + Sample code for SQS and IAM Full Access would like like: + + this.policyStatements = [ + new PolicyStatement({ + actions: ["sqs:*"], + effect: Effect.ALLOW, + resources: ["*"] + }), + new PolicyStatement({ + actions: ["iam:*"], + effect: Effect.ALLOW, + resources: ["*"] + }) + ] + */ + } +} \ No newline at end of file diff --git a/cdk.json b/cdk.json new file mode 100644 index 0000000..793096a --- /dev/null +++ b/cdk.json @@ -0,0 +1,3 @@ +{ + "app": "node app/farflow" +} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..6d913b9 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,780 @@ +{ + "name": "airflow-on-fargate", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "@aws-cdk/assets": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/assets/-/assets-1.57.0.tgz", + "integrity": "sha512-+yn4YWU7LCDbL6GsvnXgY0pe7TaH8Ib+o4zowsLuMnS2l5ShtHy7dpqJj+VVO0/YR9upvmlvMg6QY6UQRpA0cA==", + "requires": { + "@aws-cdk/core": "1.57.0", + "@aws-cdk/cx-api": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-apigateway": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-apigateway/-/aws-apigateway-1.57.0.tgz", + "integrity": "sha512-y04kG/h4lQq5TQTyOogLzdt61mdDOaXcSnQrOEBMKgMyDOpcT9mETlfxPfvn55tdrw6XwlC8d1qB6Jo6GJQf4w==", + "requires": { + "@aws-cdk/assets": "1.57.0", + "@aws-cdk/aws-certificatemanager": "1.57.0", + "@aws-cdk/aws-ec2": "1.57.0", + "@aws-cdk/aws-elasticloadbalancingv2": "1.57.0", + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/aws-lambda": "1.57.0", + "@aws-cdk/aws-logs": "1.57.0", + "@aws-cdk/aws-s3": "1.57.0", + "@aws-cdk/aws-s3-assets": "1.57.0", + "@aws-cdk/core": "1.57.0", + "@aws-cdk/cx-api": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-applicationautoscaling": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-applicationautoscaling/-/aws-applicationautoscaling-1.57.0.tgz", + "integrity": "sha512-frlqBtUcrJfZ5NTyW4DpAPnQFcHAioUQKj65/XEwCZvq6g44S9z+ARAIT0hEisUfFDFnJ1p6Fm/Wn+M6fyLQFA==", + "requires": { + "@aws-cdk/aws-autoscaling-common": "1.57.0", + "@aws-cdk/aws-cloudwatch": "1.57.0", + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-autoscaling": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-autoscaling/-/aws-autoscaling-1.57.0.tgz", + "integrity": "sha512-UYFW/ksiOPV7aHFixOYZMqddduf39skeBb8zxZBNPEKQ2PmUzSx3ygPf5XQMKngnuf47IUKd0O7Za4fBHex9Mg==", + "requires": { + "@aws-cdk/aws-autoscaling-common": "1.57.0", + "@aws-cdk/aws-cloudwatch": "1.57.0", + "@aws-cdk/aws-ec2": "1.57.0", + "@aws-cdk/aws-elasticloadbalancing": "1.57.0", + "@aws-cdk/aws-elasticloadbalancingv2": "1.57.0", + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/aws-sns": "1.57.0", + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-autoscaling-common": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-autoscaling-common/-/aws-autoscaling-common-1.57.0.tgz", + "integrity": "sha512-FYcqcby+ObhZEaP4za+JhwShQVAWAXQDq0DoDlYd30RgWcr7yz6gbz56G2N+/ngxFWQ42SkS+T1BqR3hsQdTmg==", + "requires": { + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-autoscaling-hooktargets": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-autoscaling-hooktargets/-/aws-autoscaling-hooktargets-1.57.0.tgz", + "integrity": "sha512-PIkBKiZSN+OpeSYdFcBWKgnyYrBvmhsgAAQmKiV/OP0FeM87+jh7hjlaREA/WuRIbv3vj/rcEpWcVih8vzgTLw==", + "requires": { + "@aws-cdk/aws-autoscaling": "1.57.0", + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/aws-lambda": "1.57.0", + "@aws-cdk/aws-sns": "1.57.0", + "@aws-cdk/aws-sns-subscriptions": "1.57.0", + "@aws-cdk/aws-sqs": "1.57.0", + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-batch": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-batch/-/aws-batch-1.57.0.tgz", + "integrity": "sha512-jckHpmHVEdfEBFLAk8u+W9UYOTkDvefbrTmdW9wA0361jujxC6lYyp2lkF53i75L0kdBpJpGlhlSyyofESTk3w==", + "requires": { + "@aws-cdk/aws-ec2": "1.57.0", + "@aws-cdk/aws-ecr": "1.57.0", + "@aws-cdk/aws-ecs": "1.57.0", + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-certificatemanager": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-certificatemanager/-/aws-certificatemanager-1.57.0.tgz", + "integrity": "sha512-CMRhikcHziPrKhe/CHgQ4boCa+rG19QtF7wzTMHlAeEd7E8ruJMH9yNFtS1+MFrwLgp0hMS+H/Gyz9arJtfCrA==", + "requires": { + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/aws-lambda": "1.57.0", + "@aws-cdk/aws-route53": "1.57.0", + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-cloudformation": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-cloudformation/-/aws-cloudformation-1.57.0.tgz", + "integrity": "sha512-rEjjXOoXNJ4kPSkdlKuhA1VYIRhByqZJ5PqWhGCYB+PlAlOFiAzOH63xbDkhn4g4S+PIi2h9AYNIkqniqtjceA==", + "requires": { + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/aws-lambda": "1.57.0", + "@aws-cdk/aws-s3": "1.57.0", + "@aws-cdk/aws-sns": "1.57.0", + "@aws-cdk/core": "1.57.0", + "@aws-cdk/cx-api": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-cloudfront": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-cloudfront/-/aws-cloudfront-1.57.0.tgz", + "integrity": "sha512-OWXJ6X2mxtN/EbXavr4icidamSmj588Pcjb2vvcK8XC//wOOlUZh/mAA26zTilFbFeGrCxRh62itJauRjlA6hw==", + "requires": { + "@aws-cdk/aws-certificatemanager": "1.57.0", + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/aws-kms": "1.57.0", + "@aws-cdk/aws-lambda": "1.57.0", + "@aws-cdk/aws-s3": "1.57.0", + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-cloudwatch": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-cloudwatch/-/aws-cloudwatch-1.57.0.tgz", + "integrity": "sha512-sKS0LURGmL1jbgrG9UGUxh0q+yxGVDmsds3eoJWLc0qyVrD9KN3wxU+QuSLqQdZGGun/tDIv39UQ1l+Y6vodsw==", + "requires": { + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-codebuild": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-codebuild/-/aws-codebuild-1.57.0.tgz", + "integrity": "sha512-oZhaSP1xvJsMmL1g6NWJB4GZzcKhgailwitA1VT9sNKbAabSrUE8X7x7tOBprdtRDiFdeAtO7oR1gdyuITErtA==", + "requires": { + "@aws-cdk/assets": "1.57.0", + "@aws-cdk/aws-cloudwatch": "1.57.0", + "@aws-cdk/aws-codecommit": "1.57.0", + "@aws-cdk/aws-ec2": "1.57.0", + "@aws-cdk/aws-ecr": "1.57.0", + "@aws-cdk/aws-ecr-assets": "1.57.0", + "@aws-cdk/aws-events": "1.57.0", + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/aws-kms": "1.57.0", + "@aws-cdk/aws-s3": "1.57.0", + "@aws-cdk/aws-s3-assets": "1.57.0", + "@aws-cdk/aws-secretsmanager": "1.57.0", + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-codecommit": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-codecommit/-/aws-codecommit-1.57.0.tgz", + "integrity": "sha512-AAXoDZvvKAfTpnRbqSaR2nkXRLpCdcbv+geJX2HLycgch/wIyMfXKJZXuTtQAtlt5elPGrNt3CT02kvfQKBB4w==", + "requires": { + "@aws-cdk/aws-events": "1.57.0", + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-codeguruprofiler": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-codeguruprofiler/-/aws-codeguruprofiler-1.57.0.tgz", + "integrity": "sha512-69KWy67Lo20DcFXSuSxlD5Px08jnDtkLIMroF8Zp9dEF5maxgy41i26FoY4hQQnXiTKopA8IPVxHvkouRxNutQ==", + "requires": { + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/core": "1.57.0" + } + }, + "@aws-cdk/aws-codepipeline": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-codepipeline/-/aws-codepipeline-1.57.0.tgz", + "integrity": "sha512-oTqN8MdubKKVL24/oU3erA3vBpYUeFQvDPiV1T0ODfvuFxqbgu7iVjw0h70DGyoHoeq1mi8LmdaddkgE6e8RIw==", + "requires": { + "@aws-cdk/aws-events": "1.57.0", + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/aws-kms": "1.57.0", + "@aws-cdk/aws-s3": "1.57.0", + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-cognito": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-cognito/-/aws-cognito-1.57.0.tgz", + "integrity": "sha512-+r9VFKOl1n0/7XuOP9B9DoXZx5IhnkB9y6vgrD1oOBgpOtePf6z8VSyW2nR+slZWQCTgbe2fN3E3afQY9dS1lQ==", + "requires": { + "@aws-cdk/aws-certificatemanager": "1.57.0", + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/aws-lambda": "1.57.0", + "@aws-cdk/core": "1.57.0", + "@aws-cdk/custom-resources": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-ec2": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-ec2/-/aws-ec2-1.57.0.tgz", + "integrity": "sha512-ptqXTCm0YE87YyLovVCmMVoqJ3V4yCTRJPXbA9j+h1eJcnKamTI2FmYqVDGrysAGYLAJyzi233sstfMFbks94g==", + "requires": { + "@aws-cdk/aws-cloudwatch": "1.57.0", + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/aws-kms": "1.57.0", + "@aws-cdk/aws-logs": "1.57.0", + "@aws-cdk/aws-s3": "1.57.0", + "@aws-cdk/aws-ssm": "1.57.0", + "@aws-cdk/cloud-assembly-schema": "1.57.0", + "@aws-cdk/core": "1.57.0", + "@aws-cdk/cx-api": "1.57.0", + "@aws-cdk/region-info": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-ecr": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-ecr/-/aws-ecr-1.57.0.tgz", + "integrity": "sha512-wZWvYrN1M9bn7txtdertDHcGfKYT/eTzQp/MyH/cu/ZKs+IQQ/Iq1Zh+t8d3nrkfoFPP8LPI3AyVOLAiPw8pGg==", + "requires": { + "@aws-cdk/aws-events": "1.57.0", + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/core": "1.57.0", + "@aws-cdk/custom-resources": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-ecr-assets": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-ecr-assets/-/aws-ecr-assets-1.57.0.tgz", + "integrity": "sha512-5i/b77ImDp+WF3GIIwjO1HHI3o/HPrLXbS/j44v9jxecBlpMj+DroOSn5jnq+J2wGguUS9UrIdgstts0lqCnYA==", + "requires": { + "@aws-cdk/assets": "1.57.0", + "@aws-cdk/aws-ecr": "1.57.0", + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/aws-lambda": "1.57.0", + "@aws-cdk/aws-s3": "1.57.0", + "@aws-cdk/core": "1.57.0", + "@aws-cdk/cx-api": "1.57.0", + "constructs": "^3.0.2", + "minimatch": "^3.0.4" + }, + "dependencies": { + "minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "requires": { + "brace-expansion": "^1.1.7" + } + } + } + }, + "@aws-cdk/aws-ecs": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-ecs/-/aws-ecs-1.57.0.tgz", + "integrity": "sha512-uSUNe+K10WcvWhYKY3rvN0C/93yuajbdHMqybemNVU29PFnqTQ6djKsieIAxlPctM6jwQditakLH8S01xscTxw==", + "requires": { + "@aws-cdk/aws-applicationautoscaling": "1.57.0", + "@aws-cdk/aws-autoscaling": "1.57.0", + "@aws-cdk/aws-autoscaling-hooktargets": "1.57.0", + "@aws-cdk/aws-certificatemanager": "1.57.0", + "@aws-cdk/aws-cloudwatch": "1.57.0", + "@aws-cdk/aws-ec2": "1.57.0", + "@aws-cdk/aws-ecr": "1.57.0", + "@aws-cdk/aws-ecr-assets": "1.57.0", + "@aws-cdk/aws-elasticloadbalancing": "1.57.0", + "@aws-cdk/aws-elasticloadbalancingv2": "1.57.0", + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/aws-lambda": "1.57.0", + "@aws-cdk/aws-logs": "1.57.0", + "@aws-cdk/aws-route53": "1.57.0", + "@aws-cdk/aws-route53-targets": "1.57.0", + "@aws-cdk/aws-secretsmanager": "1.57.0", + "@aws-cdk/aws-servicediscovery": "1.57.0", + "@aws-cdk/aws-sns": "1.57.0", + "@aws-cdk/aws-sqs": "1.57.0", + "@aws-cdk/aws-ssm": "1.57.0", + "@aws-cdk/core": "1.57.0", + "@aws-cdk/cx-api": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-ecs-patterns": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-ecs-patterns/-/aws-ecs-patterns-1.57.0.tgz", + "integrity": "sha512-FPKNi17th6d88wNkl6WzNi9pEsffTZiWwZ27yazKzP6UzkF+LtV6qK7DKyPnaGdrKxMgVcDNQJGsHMHcyrNLfA==", + "requires": { + "@aws-cdk/aws-applicationautoscaling": "1.57.0", + "@aws-cdk/aws-certificatemanager": "1.57.0", + "@aws-cdk/aws-ec2": "1.57.0", + "@aws-cdk/aws-ecs": "1.57.0", + "@aws-cdk/aws-elasticloadbalancingv2": "1.57.0", + "@aws-cdk/aws-events": "1.57.0", + "@aws-cdk/aws-events-targets": "1.57.0", + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/aws-route53": "1.57.0", + "@aws-cdk/aws-route53-targets": "1.57.0", + "@aws-cdk/aws-servicediscovery": "1.57.0", + "@aws-cdk/aws-sqs": "1.57.0", + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-efs": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-efs/-/aws-efs-1.57.0.tgz", + "integrity": "sha512-vrxQAdG3c4KYGPnE6+eppyKHnqpdP/15VIC8ihFxpqdKBeQK2CguTZuN/dKMHWvhN6F2iyvRoGWSDzqI5uTbrg==", + "requires": { + "@aws-cdk/aws-ec2": "1.57.0", + "@aws-cdk/aws-kms": "1.57.0", + "@aws-cdk/cloud-assembly-schema": "1.57.0", + "@aws-cdk/core": "1.57.0", + "@aws-cdk/cx-api": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-elasticloadbalancing": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-elasticloadbalancing/-/aws-elasticloadbalancing-1.57.0.tgz", + "integrity": "sha512-pK8fFu7g7MnGz4Dp+qlDEEXF2Vq2UQyVRgVt+wvcMWHbZN35wkhRM8XVvZ/QO7jyk4Dz5+hRy+vUbi5SuVha9w==", + "requires": { + "@aws-cdk/aws-ec2": "1.57.0", + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-elasticloadbalancingv2": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-elasticloadbalancingv2/-/aws-elasticloadbalancingv2-1.57.0.tgz", + "integrity": "sha512-fSoqsgWBawL6eYIoAoF7O0QVwqlvAmtKltlUJ4mKS0Umor3MXEGrsdaMKitgNx6/mToJdYtfSe6UDMWI3HdjyQ==", + "requires": { + "@aws-cdk/aws-certificatemanager": "1.57.0", + "@aws-cdk/aws-cloudwatch": "1.57.0", + "@aws-cdk/aws-ec2": "1.57.0", + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/aws-lambda": "1.57.0", + "@aws-cdk/aws-s3": "1.57.0", + "@aws-cdk/core": "1.57.0", + "@aws-cdk/region-info": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-events": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-events/-/aws-events-1.57.0.tgz", + "integrity": "sha512-d2681DHe+gFQq+o4GXglnaEKmEorCpTYPQIbbRRO3RQzggSLdJMYFoIio3Vft3xeuaxpZCpNwyJ6mROdgiq3eg==", + "requires": { + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-events-targets": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-events-targets/-/aws-events-targets-1.57.0.tgz", + "integrity": "sha512-iwCY3CmxdSt0OknZdIO7aGIc/Q3sWLCFDTS1m2K+pf58d+Mim/g/QeifNje/UpUCBbBJkgufFPj3lFQa7w92lA==", + "requires": { + "@aws-cdk/aws-batch": "1.57.0", + "@aws-cdk/aws-codebuild": "1.57.0", + "@aws-cdk/aws-codepipeline": "1.57.0", + "@aws-cdk/aws-ec2": "1.57.0", + "@aws-cdk/aws-ecs": "1.57.0", + "@aws-cdk/aws-events": "1.57.0", + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/aws-kinesis": "1.57.0", + "@aws-cdk/aws-lambda": "1.57.0", + "@aws-cdk/aws-sns": "1.57.0", + "@aws-cdk/aws-sns-subscriptions": "1.57.0", + "@aws-cdk/aws-sqs": "1.57.0", + "@aws-cdk/aws-stepfunctions": "1.57.0", + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-iam": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-iam/-/aws-iam-1.57.0.tgz", + "integrity": "sha512-vOltgt7CDEZ9Zko5tBhxP6bwuOAJA5ev04w1rwAcDD8W4UGHnz+2D1b0eAHfLhsICVagwfNEG+KmDyyKzrzyFw==", + "requires": { + "@aws-cdk/core": "1.57.0", + "@aws-cdk/region-info": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-kinesis": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-kinesis/-/aws-kinesis-1.57.0.tgz", + "integrity": "sha512-x7ySlxBV+8I82MeYdNHLOMFi7mO19mkMabP9MrDlINcXpHRwXtUO6LMN+Bl/sda5v2FeVK8tpZm1PQ7lJ3jVCA==", + "requires": { + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/aws-kms": "1.57.0", + "@aws-cdk/aws-logs": "1.57.0", + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-kms": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-kms/-/aws-kms-1.57.0.tgz", + "integrity": "sha512-kBoOCPEZTTCU2JN9LfVy7qYKI9+XGyCPybcxbrXhYjM1eC2tKEcCub7fQiQ3He+H4QEgfEqoKhwJlLHUDMSKtQ==", + "requires": { + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-lambda": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-lambda/-/aws-lambda-1.57.0.tgz", + "integrity": "sha512-LNYO32T6ljUbfIEh1rOPFePQfUGF8GzAmazQ8A/G0aE8e919aJexWJ4k2YVcKAoKQRl7nPAZMXWWg2VfxEocDQ==", + "requires": { + "@aws-cdk/aws-cloudwatch": "1.57.0", + "@aws-cdk/aws-codeguruprofiler": "1.57.0", + "@aws-cdk/aws-ec2": "1.57.0", + "@aws-cdk/aws-efs": "1.57.0", + "@aws-cdk/aws-events": "1.57.0", + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/aws-logs": "1.57.0", + "@aws-cdk/aws-s3": "1.57.0", + "@aws-cdk/aws-s3-assets": "1.57.0", + "@aws-cdk/aws-sqs": "1.57.0", + "@aws-cdk/core": "1.57.0", + "@aws-cdk/cx-api": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-logs": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-logs/-/aws-logs-1.57.0.tgz", + "integrity": "sha512-q8hmcT7f6ZSHs0qypncTvIag7onKvt3gQIknPDnn3712rP3yz0zJlDfcN/TXRKEPxGVjGB9VCM/9q9pBKwbzQQ==", + "requires": { + "@aws-cdk/aws-cloudwatch": "1.57.0", + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-rds": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-rds/-/aws-rds-1.57.0.tgz", + "integrity": "sha512-FU6PSJzMQOLBnY8FzDMYngZ/uf7QCCSB1QkHIn9geoaMaMtqRadRwrBYei6U5fdhFFLxAi94UYNwlQa8scXkyw==", + "requires": { + "@aws-cdk/aws-cloudwatch": "1.57.0", + "@aws-cdk/aws-ec2": "1.57.0", + "@aws-cdk/aws-events": "1.57.0", + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/aws-kms": "1.57.0", + "@aws-cdk/aws-lambda": "1.57.0", + "@aws-cdk/aws-logs": "1.57.0", + "@aws-cdk/aws-s3": "1.57.0", + "@aws-cdk/aws-secretsmanager": "1.57.0", + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-route53": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-route53/-/aws-route53-1.57.0.tgz", + "integrity": "sha512-SR+Q6fxDNzhsrc0gKfdXH5CZYH++7qwj5Hcd5CXKKHi9hw01GrYadTz8eXLFRb6Bt8coY5FN+KDbXc/wCI0H4g==", + "requires": { + "@aws-cdk/aws-ec2": "1.57.0", + "@aws-cdk/aws-logs": "1.57.0", + "@aws-cdk/cloud-assembly-schema": "1.57.0", + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-route53-targets": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-route53-targets/-/aws-route53-targets-1.57.0.tgz", + "integrity": "sha512-1qE5Zr/y3vF/8h/7coJoyDcjacjVeI3ASXECC/O8hEpDeevqQSTza/l/xyeM8FRpYw/TqtFpJ7EAZbnYk3nJFw==", + "requires": { + "@aws-cdk/aws-apigateway": "1.57.0", + "@aws-cdk/aws-cloudfront": "1.57.0", + "@aws-cdk/aws-cognito": "1.57.0", + "@aws-cdk/aws-ec2": "1.57.0", + "@aws-cdk/aws-elasticloadbalancing": "1.57.0", + "@aws-cdk/aws-elasticloadbalancingv2": "1.57.0", + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/aws-route53": "1.57.0", + "@aws-cdk/aws-s3": "1.57.0", + "@aws-cdk/core": "1.57.0", + "@aws-cdk/region-info": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-s3": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-s3/-/aws-s3-1.57.0.tgz", + "integrity": "sha512-pwqLL6D+2saDNAlue74iBphPJ9E4ejXxrXg3llMLbud6GIjtzXz4G8H7z6OwkgdceEK5Sx3eVQNR4MPREdcO5g==", + "requires": { + "@aws-cdk/aws-events": "1.57.0", + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/aws-kms": "1.57.0", + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-s3-assets": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-s3-assets/-/aws-s3-assets-1.57.0.tgz", + "integrity": "sha512-zX6xvP9m+InbD0IT40yCBRCtg0EW66WGMrd7yi9p4/kK8SSDdRP5LITJTzNrche24LUV2jWpzSBPVAm8QCp7JQ==", + "requires": { + "@aws-cdk/assets": "1.57.0", + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/aws-kms": "1.57.0", + "@aws-cdk/aws-s3": "1.57.0", + "@aws-cdk/core": "1.57.0", + "@aws-cdk/cx-api": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-sam": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-sam/-/aws-sam-1.57.0.tgz", + "integrity": "sha512-QDfNwvE9fXk6gf99H6GMEMaWUOX6xSfnqLZaRACGD95saNwTtRT00j9gepIxbf8AA1vvZLpw222U4vTnMnU01g==", + "requires": { + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-secretsmanager": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-secretsmanager/-/aws-secretsmanager-1.57.0.tgz", + "integrity": "sha512-URWlLfF1PR2BHgwcB1zGMkW1AsF9SrkA7qwyV/MmVvsQf3c2LOR6LWyCWJBRo3QSjrl/J8XVo6V5vBehE8hjow==", + "requires": { + "@aws-cdk/aws-ec2": "1.57.0", + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/aws-kms": "1.57.0", + "@aws-cdk/aws-lambda": "1.57.0", + "@aws-cdk/aws-sam": "1.57.0", + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-servicediscovery": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-servicediscovery/-/aws-servicediscovery-1.57.0.tgz", + "integrity": "sha512-buRrZNRIrb1X7SHt+s8n2x6Pnj4fQ4L5do4NyC2yvzJk1na7I2quY8Po95GghZAi0JHraq6aI/AR4mOY10uI9w==", + "requires": { + "@aws-cdk/aws-ec2": "1.57.0", + "@aws-cdk/aws-elasticloadbalancingv2": "1.57.0", + "@aws-cdk/aws-route53": "1.57.0", + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-sns": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-sns/-/aws-sns-1.57.0.tgz", + "integrity": "sha512-kJGbpOiUJBJext6DuWPkF6N3k/TzGeS/jXCCTWRbJoNFhyfor0G3IqHsuenLioNFz5IhPXJHEjW2vmGy5zjAlw==", + "requires": { + "@aws-cdk/aws-cloudwatch": "1.57.0", + "@aws-cdk/aws-events": "1.57.0", + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/aws-kms": "1.57.0", + "@aws-cdk/aws-sqs": "1.57.0", + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-sns-subscriptions": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-sns-subscriptions/-/aws-sns-subscriptions-1.57.0.tgz", + "integrity": "sha512-86+fW/BKKyiNqZCTdmgf7RJcVrzhwLmZaCp/SatAnf6CPQ7vcf+14JAa8FmRjB0C5kSAeFDKx9eAb5amo9zTzA==", + "requires": { + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/aws-lambda": "1.57.0", + "@aws-cdk/aws-sns": "1.57.0", + "@aws-cdk/aws-sqs": "1.57.0", + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-sqs": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-sqs/-/aws-sqs-1.57.0.tgz", + "integrity": "sha512-Yu75GtLOykW/LPTp6fArLMsz8msTnnD0afIYcifzMbIU+EoqGtsGLkJat1rqHyU4t6g3cbcK8f7CVWSy+YyTZQ==", + "requires": { + "@aws-cdk/aws-cloudwatch": "1.57.0", + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/aws-kms": "1.57.0", + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-ssm": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-ssm/-/aws-ssm-1.57.0.tgz", + "integrity": "sha512-nzDqMlqCooRbIZ3BazJMIXmoQWTMjJxP0Xn7bIfyjiTQwVj76h8r7n20uajXfCVMNcDj1Mpq+QBvNdyZPG/R6g==", + "requires": { + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/aws-kms": "1.57.0", + "@aws-cdk/cloud-assembly-schema": "1.57.0", + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/aws-stepfunctions": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-stepfunctions/-/aws-stepfunctions-1.57.0.tgz", + "integrity": "sha512-4Age9vLLC5s7oeVRIOW/wB+EvAgoIZxO/nlkZ3i5bRWvvRpesELS3DUkSXicbEJP0mn3yCujGnMo85LnTT7o2g==", + "requires": { + "@aws-cdk/aws-cloudwatch": "1.57.0", + "@aws-cdk/aws-events": "1.57.0", + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/aws-logs": "1.57.0", + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/cloud-assembly-schema": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/cloud-assembly-schema/-/cloud-assembly-schema-1.57.0.tgz", + "integrity": "sha512-IJyV3pgMvpbaIVYpUkBGsxIWh+VK7TxuTbEyHfBra5+VgXLoFdSG2UO80b4v6ou+occRqUGbqAnc/VfYr1uuvw==", + "requires": { + "jsonschema": "^1.2.5", + "semver": "^7.2.2" + } + }, + "@aws-cdk/core": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/core/-/core-1.57.0.tgz", + "integrity": "sha512-NOE9u2tVwEF+EN5CYaoC34csBgKLA9rjCWCB6R64RPQ2MlhBFCvQxdG2ZO26nOOvH+yv/3zlylM7F546mfQMHg==", + "requires": { + "@aws-cdk/cloud-assembly-schema": "1.57.0", + "@aws-cdk/cx-api": "1.57.0", + "constructs": "^3.0.2", + "fs-extra": "^9.0.1", + "minimatch": "^3.0.4" + } + }, + "@aws-cdk/custom-resources": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/custom-resources/-/custom-resources-1.57.0.tgz", + "integrity": "sha512-EEK2aRFRfC5uKWRQLFLjLApipb0NU07fDuATgdhSd1oqDRUvzvpQGJe0QL411c5rZGZa1aUAU9t/rnY1zmd1yQ==", + "requires": { + "@aws-cdk/aws-cloudformation": "1.57.0", + "@aws-cdk/aws-iam": "1.57.0", + "@aws-cdk/aws-lambda": "1.57.0", + "@aws-cdk/aws-logs": "1.57.0", + "@aws-cdk/aws-sns": "1.57.0", + "@aws-cdk/core": "1.57.0", + "constructs": "^3.0.2" + } + }, + "@aws-cdk/cx-api": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/cx-api/-/cx-api-1.57.0.tgz", + "integrity": "sha512-h2qTTofE8cE1rSZB9ny+7AjjmxEdKoxhq+GsQebQ5NZkrN/Rbc+DL77S/GG53koPG4u/ZoA4UcdLz/JqiGgdPA==", + "requires": { + "@aws-cdk/cloud-assembly-schema": "1.57.0", + "semver": "^7.2.2" + }, + "dependencies": { + "semver": { + "version": "7.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz", + "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==" + } + } + }, + "@aws-cdk/region-info": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/region-info/-/region-info-1.57.0.tgz", + "integrity": "sha512-U2V2f/PdD2VDGVwGGShb+7dqhmEEmShlOOdvDQvTmAg2SW6HwOt/rsz+va6EzvWV7fmxtFEfMiJVpnheZ1Vwyg==" + }, + "@types/node": { + "version": "8.10.62", + "resolved": "https://registry.npmjs.org/@types/node/-/node-8.10.62.tgz", + "integrity": "sha512-76fupxOYVxk36kb7O/6KtrAPZ9jnSK3+qisAX4tQMEuGNdlvl7ycwatlHqjoE6jHfVtXFM3pCrCixZOidc5cuw==", + "dev": true + }, + "at-least-node": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz", + "integrity": "sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==" + }, + "balanced-match": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" + }, + "constructs": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/constructs/-/constructs-3.0.4.tgz", + "integrity": "sha512-CDvg7gMjphE3DFX4pzkF6j73NREbR8npPFW8Mx/CLRnMR035+Y1o1HrXIsNSss/dq3ZUnNTU9jKyd3fL9EOlfw==" + }, + "fs-extra": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.0.1.tgz", + "integrity": "sha512-h2iAoN838FqAFJY2/qVpzFXy+EBxfVE220PalAqQLDVsFOHLJrZvut5puAbCdNv6WJk+B8ihI+k0c7JK5erwqQ==", + "requires": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^1.0.0" + } + }, + "graceful-fs": { + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.4.tgz", + "integrity": "sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw==" + }, + "jsonfile": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.0.1.tgz", + "integrity": "sha512-jR2b5v7d2vIOust+w3wtFKZIfpC2pnRmFAhAC/BuweZFQR8qZzxH1OyrQ10HmdVYiXWkYUqPVsz91cG7EL2FBg==", + "requires": { + "graceful-fs": "^4.1.6", + "universalify": "^1.0.0" + } + }, + "jsonschema": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/jsonschema/-/jsonschema-1.2.6.tgz", + "integrity": "sha512-SqhURKZG07JyKKeo/ir24QnS4/BV7a6gQy93bUSe4lUdNp0QNpIz2c9elWJQ9dpc5cQYY6cvCzgRwy0MQCLyqA==" + }, + "minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "semver": { + "version": "7.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz", + "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==" + }, + "typescript": { + "version": "3.7.5", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.7.5.tgz", + "integrity": "sha512-/P5lkRXkWHNAbcJIiHPfRoKqyd7bsyCma1hZNUGfn20qm64T6ZBlrzprymeu918H+mB/0rIg2gGK/BXkhhYgBw==", + "dev": true + }, + "universalify": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-1.0.0.tgz", + "integrity": "sha512-rb6X1W158d7pRQBg5gkR8uPaSfiids68LTJQYOtEUhoJUWBdaQHsuT/EUduxXYxcrt4r5PJ4fuHW1MHT6p0qug==" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..c8b8055 --- /dev/null +++ b/package.json @@ -0,0 +1,29 @@ +{ + "name": "airflow-on-fargate", + "version": "1.0.0", + "description": "Running an application load balanced Airflow on Fargate", + "private": true, + "scripts": { + "build": "tsc", + "watch": "tsc -w", + "cdk": "cdk" + }, + "author": { + "name": "Amazon Web Services", + "url": "https://aws.amazon.com", + "organization": true + }, + "license": "Apache-2.0", + "devDependencies": { + "@types/node": "^8.10.38", + "typescript": "~3.7.2" + }, + "dependencies": { + "@aws-cdk/aws-ec2": "*", + "@aws-cdk/aws-ecs": "*", + "@aws-cdk/aws-ecs-patterns": "*", + "@aws-cdk/aws-ecr-assets": "*", + "@aws-cdk/aws-rds": "*", + "@aws-cdk/core": "*" + } +} diff --git a/tasks/multi_task/Dockerfile b/tasks/multi_task/Dockerfile new file mode 100644 index 0000000..f51538d --- /dev/null +++ b/tasks/multi_task/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.8-slim + +ENV USER_HOME=/usr/local/farflow +COPY . ${USER_HOME}/app +WORKDIR ${USER_HOME}/app + +#CMD ["python","numbers.py", '10'] \ No newline at end of file diff --git a/tasks/multi_task/even_numbers.py b/tasks/multi_task/even_numbers.py new file mode 100644 index 0000000..c960e76 --- /dev/null +++ b/tasks/multi_task/even_numbers.py @@ -0,0 +1,13 @@ +from argparse import ArgumentParser + +parser = ArgumentParser(description='Airflow Fargate Example') +parser.add_argument('number', help='number', type=int) + +if __name__ == '__main__': + args = parser.parse_args() + number = args.number + + print("Printing Even numbers in given range") + for i in range(int(number)): + if(i % 2 == 0): + print(i) diff --git a/tasks/multi_task/odd_numbers.py b/tasks/multi_task/odd_numbers.py new file mode 100644 index 0000000..486e7bd --- /dev/null +++ b/tasks/multi_task/odd_numbers.py @@ -0,0 +1,13 @@ +from argparse import ArgumentParser + +parser = ArgumentParser(description='Airflow Fargate Example') +parser.add_argument('number', help='number', type=int) + +if __name__ == '__main__': + args = parser.parse_args() + number = args.number + + print("Printing Odd numbers in given range") + for i in range(int(number)): + if(i % 2 != 0): + print(i) diff --git a/tasks/number_task/Dockerfile b/tasks/number_task/Dockerfile new file mode 100644 index 0000000..fa39f9c --- /dev/null +++ b/tasks/number_task/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.8-slim + +ENV USER_HOME=/usr/local/farflow +COPY . ${USER_HOME}/app +WORKDIR ${USER_HOME}/app + +CMD ["python","numbers.py", '10'] \ No newline at end of file diff --git a/tasks/number_task/numbers.py b/tasks/number_task/numbers.py new file mode 100644 index 0000000..be548ff --- /dev/null +++ b/tasks/number_task/numbers.py @@ -0,0 +1,11 @@ +from argparse import ArgumentParser + +parser = ArgumentParser(description='Airflow Fargate Example') +parser.add_argument('number', help='number', type=int) + +if __name__ == '__main__': + args = parser.parse_args() + number = args.number + print("Printing all numbers in given range") + for i in range(int(number)): + print(i) diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..b2a0d11 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "target":"ES2018", + "module": "commonjs", + "lib": ["es2016", "es2017.object", "es2017.string"], + "strict": true, + "noImplicitAny": true, + "strictNullChecks": true, + "noImplicitThis": true, + "alwaysStrict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": false, + "inlineSourceMap": true, + "inlineSources": true, + "experimentalDecorators": true, + "strictPropertyInitialization":false + } +}