AWSTemplateFormatVersion: '2010-09-09'
Description: 'EMR Cluster with Spark, Hive, Hadoop, Iceberg, JupyterHub, and JupyterEnterpriseGateway'

Parameters:
  ClusterName:
    Type: String
    Default: emr-cluster-for-s3tables-demo
    Description: Name of the EMR cluster

  Environment:
    Type: String
    Default: dev
    AllowedValues: [dev, test, prod]
    Description: Environment type


  KeyName:
    Type: AWS::EC2::KeyPair::KeyName
    Description: Name of an existing EC2 KeyPair

  SubnetId:
    Type: AWS::EC2::Subnet::Id
    Description: Subnet ID for EMR cluster

  VpcId:
    Type: AWS::EC2::VPC::Id
    Description: VPC ID for the EMR cluster

  SourceCidrForNotebook:
    Type: String
    Description: IMPORTANT- Please enter your IP or VPN range for Notebook access and security groups for EMR cluster (e.g. your IP/22). Avoid using 0.0.0.0/0 as IP range as you may encounter errors/failures while creating EMR clusters.
    Default: '0.0.0.0/0'
    AllowedPattern: '(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})'



Resources:

  EMRMasterSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: Security group for EMR Master Node
      VpcId: !Ref VpcId
      Tags:
        - Key: Name
          Value: !Sub emr-master-sg-${Environment}
        - Key: Environment
          Value: !Ref Environment
      SecurityGroupIngress:
        # JupyterHub access
        - IpProtocol: tcp
          FromPort: 9443
          ToPort: 9443
          CidrIp: !Ref SourceCidrForNotebook
          Description: JupyterHub HTTPS
        # Jupyter Enterprise Gateway
        - IpProtocol: tcp
          FromPort: 8888
          ToPort: 8888
          CidrIp: !Ref SourceCidrForNotebook
          Description: Jupyter Enterprise Gateway
        # YARN ResourceManager
        - IpProtocol: tcp
          FromPort: 8088
          ToPort: 8088
          CidrIp: !Ref SourceCidrForNotebook
          Description: YARN ResourceManager web UI
        # Spark History Server
        - IpProtocol: tcp
          FromPort: 18080
          ToPort: 18080
          CidrIp: !Ref SourceCidrForNotebook
          Description: Spark History Server
        # Livy Server
        - IpProtocol: tcp
          FromPort: 8998
          ToPort: 8998
          CidrIp: !Ref SourceCidrForNotebook
          Description: Livy Server
        # Ganglia
        - IpProtocol: tcp
          FromPort: 80
          ToPort: 80
          CidrIp: !Ref SourceCidrForNotebook
          Description: Ganglia web UI

  EMRSlaveSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: Security group for EMR Slave Nodes
      VpcId: !Ref VpcId
      Tags:
        - Key: Name
          Value: !Sub emr-slave-sg-${Environment}
        - Key: Environment
          Value: !Ref Environment

  # Allow all traffic between master and slave nodes
  MasterToSlaveIngress:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId: !Ref EMRSlaveSecurityGroup
      SourceSecurityGroupId: !Ref EMRMasterSecurityGroup
      IpProtocol: -1
      FromPort: -1
      ToPort: -1
      Description: Allow all traffic from master node

  SlaveToMasterIngress:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId: !Ref EMRMasterSecurityGroup
      SourceSecurityGroupId: !Ref EMRSlaveSecurityGroup
      IpProtocol: -1
      FromPort: -1
      ToPort: -1
      Description: Allow all traffic from slave nodes

  # Allow communication between slave nodes
  SlaveToSlaveIngress:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId: !Ref EMRSlaveSecurityGroup
      SourceSecurityGroupId: !Ref EMRSlaveSecurityGroup
      IpProtocol: -1
      FromPort: -1
      ToPort: -1
      Description: Allow all traffic between slave nodes

  # Add egress rules
  MasterSecurityGroupEgress:
    Type: AWS::EC2::SecurityGroupEgress
    Properties:
      GroupId: !Ref EMRMasterSecurityGroup
      IpProtocol: -1
      FromPort: -1
      ToPort: -1
      CidrIp: 0.0.0.0/0
      Description: Allow all outbound traffic

  SlaveSecurityGroupEgress:
    Type: AWS::EC2::SecurityGroupEgress
    Properties:
      GroupId: !Ref EMRSlaveSecurityGroup
      IpProtocol: -1
      FromPort: -1
      ToPort: -1
      CidrIp: 0.0.0.0/0
      Description: Allow all outbound traffic


  EMRServiceRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: elasticmapreduce.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceRole
        

  EMRInstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Roles:
        - !Ref EMRInstanceRole

  EMRInstanceRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: ec2.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforEC2Role
        - arn:aws:iam::aws:policy/AmazonS3FullAccess
        - arn:aws:iam::aws:policy/AmazonS3TablesFullAccess
      

  EMRSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: Security group for EMR cluster
      VpcId: !Ref VpcId


  EMRCluster:
    Type: AWS::EMR::Cluster
    Properties:
      Name: !Sub ${ClusterName}-${Environment}
      ReleaseLabel: "emr-7.5.0"


      ServiceRole: !Ref EMRServiceRole
      JobFlowRole: !Ref EMRInstanceProfile
      VisibleToAllUsers: true
      
      Applications:
        - Name: Spark
        - Name: JupyterEnterpriseGateway
        - Name: JupyterHub
        
            

      Configurations:
        - Classification: spark-defaults
          ConfigurationProperties:
            spark.driver.memory: "5692M"
            spark.executor.memory: "5692M"
            spark.yarn.am.memory: "5692M"
            spark.dynamicAllocation.enabled: "true"
            spark.shuffle.service.enabled: "true"
            
        - Classification: yarn-site
          ConfigurationProperties:
            yarn.nodemanager.resource.memory-mb: "57344"
            yarn.scheduler.maximum-allocation-mb: "57344"
            yarn.scheduler.minimum-allocation-mb: "1024"
            
        - Classification: iceberg-defaults
          ConfigurationProperties:
            iceberg.enabled: "true"

      Instances:
        Ec2KeyName: !Ref KeyName  # Changed from KeyName to Ec2KeyName
        Ec2SubnetId: !Ref SubnetId
        EmrManagedMasterSecurityGroup: !Ref EMRMasterSecurityGroup
        EmrManagedSlaveSecurityGroup: !Ref EMRSlaveSecurityGroup
        TerminationProtected: false
        
        MasterInstanceGroup:
          InstanceCount: 1
          InstanceType: "r5.8xlarge"
          Market: ON_DEMAND
          Name: Master
          EbsConfiguration:
            EbsBlockDeviceConfigs:
              - VolumeSpecification:
                  VolumeType: gp3
                  SizeInGB: 100
                  Iops: 3000
                VolumesPerInstance: 1
        
        CoreInstanceGroup:
          InstanceCount: 1
          InstanceType: "r5.16xlarge"
          Market: ON_DEMAND
          Name: Core
          EbsConfiguration:
            EbsBlockDeviceConfigs:
              - VolumeSpecification:
                  VolumeType: gp3
                  SizeInGB: 100
                  Iops: 3000
                VolumesPerInstance: 1

                
                

      Tags:
        - Key: Environment
          Value: !Ref Environment
        - Key: Name
          Value: !Sub ${ClusterName}-${Environment}

      ScaleDownBehavior: TERMINATE_AT_TASK_COMPLETION
      AutoTerminationPolicy:
        IdleTimeout: 7200

  

Outputs:
  ClusterId:
    Description: EMR Cluster ID
    Value: !Ref EMRCluster

  MasterPublicDNS:
    Description: Master node public DNS
    Value: !GetAtt EMRCluster.MasterPublicDNS

  JupyterHubURL:
    Description: JupyterHub URL
    Value: !Sub 'https://${EMRCluster.MasterPublicDNS}:9443'

  JupyterEnterpriseGatewayURL:
    Description: Jupyter Enterprise Gateway URL
    Value: !Sub 'https://${EMRCluster.MasterPublicDNS}:8888'


  MasterSecurityGroupId:
    Description: Security Group ID for EMR Master Node
    Value: !Ref EMRMasterSecurityGroup
    Export:
      Name: !Sub ${AWS::StackName}-master-sg

  SlaveSecurityGroupId:
    Description: Security Group ID for EMR Slave Nodes
    Value: !Ref EMRSlaveSecurityGroup
    Export:
      Name: !Sub ${AWS::StackName}-slave-sg

  MasterSecurityGroupName:
    Description: Security Group Name for EMR Master Node
    Value: !GetAtt EMRMasterSecurityGroup.GroupName

  SlaveSecurityGroupName:
    Description: Security Group Name for EMR Slave Nodes
    Value: !GetAtt EMRSlaveSecurityGroup.GroupName
