Files
Airflow-on-Fargate/app/constructs/task-construct.ts
Chaithanya Maisagoni 7814377342 Adding support for EFS mounts.
Changelog:
* Upgraded CDK version to support EFS usage
* Upgraded Fargate PlatformVersion to support EFS mounts
* Refacored RDS contruct as per new CDK
* Created a new LogGroup for OnDemand DagTasks
* Added TAG for stack, to track resources belonging to this setup
* Updated sample DAG to utilize EFS. Tasks Odd and Even will publish to EFS and Numbers will read from EFS
* Now you can see logs from OnDemand tasks on Airflow UI, once task run finishes
2020-12-08 11:58:26 -08:00

73 lines
2.1 KiB
TypeScript

import { Construct } from "@aws-cdk/core";
import ecs = require('@aws-cdk/aws-ecs');
import { DockerImageAsset } from '@aws-cdk/aws-ecr-assets';
import { FargateTaskDefinition } from '@aws-cdk/aws-ecs';
import {ManagedPolicy} from "@aws-cdk/aws-iam";
export interface AirflowDagTaskDefinitionProps {
readonly taskFamilyName: string;
readonly containerInfo: ContainerInfo;
readonly cpu: number;
readonly memoryLimitMiB: number;
readonly logging: ecs.LogDriver;
readonly efsVolumeInfo?: EfsVolumeInfo;
}
export interface ContainerInfo {
readonly name: string;
readonly assetDir: string;
}
export interface EfsVolumeInfo {
readonly volumeName: string;
readonly efsFileSystemId: string;
readonly containerPath: string;
}
export class AirflowDagTaskDefinition extends Construct {
constructor(
scope: Construct,
taskName: string,
props: AirflowDagTaskDefinitionProps
) {
super(scope, taskName + "-TaskConstruct");
// Create a new task with given requirements
const workerTask = new FargateTaskDefinition(this, taskName + '-TaskDef', {
cpu: props.cpu,
memoryLimitMiB: props.memoryLimitMiB,
family: props.taskFamilyName
});
if (props.efsVolumeInfo) {
workerTask.addVolume({
name: props.efsVolumeInfo.volumeName,
efsVolumeConfiguration: {
fileSystemId: props.efsVolumeInfo.efsFileSystemId
}
});
workerTask.taskRole.addManagedPolicy(ManagedPolicy.fromAwsManagedPolicyName("AmazonElasticFileSystemClientReadWriteAccess"));
}
const workerImageAsset = new DockerImageAsset(this, props.containerInfo.name + '-BuildImage', {
directory: props.containerInfo.assetDir,
});
let container = workerTask.addContainer(props.containerInfo.name, {
image: ecs.ContainerImage.fromDockerImageAsset(workerImageAsset),
logging: props.logging
});
if (props.efsVolumeInfo) {
container.addMountPoints({
containerPath: props.efsVolumeInfo.containerPath,
sourceVolume: props.efsVolumeInfo.volumeName,
readOnly: false
});
}
}
}