Step 4:Once the consuming accounts get the events, they can set up event rules on their side triggering their AWS processes like Lambdas, Glue Jobs, EMR scripts etc.

*/
Terraform Code for setting up a custom event bus and event rule that sends
events to a clodwatch log group so users can see the events being recieved.
*/

#Creates a custom eventbus 
resource "aws_cloudwatch_event_bus" "messenger" {
  name = "custom_event_name"
}

/* 
Creates a CloudWatch event rule that sends all events that have
their source as the source defined in the event_pattern.
*/
resource "aws_cloudwatch_event_rule" "eventrule" {
  name        = "event_rule_name"
  description = "Rule to see events being published from the custom eventbus to cloudwatch"
  event_bus_name = aws_cloudwatch_event_bus.messenger.name

  event_pattern = jsonencode(
   {
     "source": [ 
       "source_of_data"
     ]
   }
  ) 
}

/*
Allows root users in the account to send events to the custom eventbus.

Consuming accounts can add the producer accounts in the "producer_account_ids"
variable allowing them to send events to the consuming accounts eventbus.

Allows root users to view and put rules and resources in the custom eventbus. 
*/
data "aws_iam_policy_document" "eventbus" {
    statement {
      sid    = "allow_account_to_put_events"
      effect = "Allow"
      actions = [
        "events:PutEvents",
      ]
      resources = [
        aws_cloudwatch_event_bus.messenger.arn
      ]

      principals {
        type        = "AWS"
        identifiers = compact(concat([var.aws_account_id], var.producer_account_ids))
      }
    }
      statement {
      sid    = "allow_account_to_manage_rules_they_created"
      effect = "Allow"
      actions = [
        "events:DescribeRule",
        "events:ListRules",
        "events:ListTagsForResource",
        "events:PutRule", 
        "events:PutTargets",
        "events:DeleteRule", 
        "events:RemoveTargets", 
        "events:DisableRule", 
        "events:EnableRule", 
        "events:TagResource", 
        "events:UntagResource", 
        "events:ListTargetsByRule",
        "events:ListTagsForResource"
      ]
      resources = [
        aws_cloudwatch_event_bus.messenger.arn,
        aws_cloudwatch_event_rule.eventrule.arn
      ]

      principals {
        type        = "AWS"
        identifiers = [var.aws_account_id]
      }  
    }
}

resource "aws_cloudwatch_event_bus_policy" "eventbus" {
  policy         = data.aws_iam_policy_document.eventbus.json
  event_bus_name = aws_cloudwatch_event_bus.messenger.name
}

#event bridge event target CloudWatch

#Creates CloudWatch log group 
resource "aws_cloudwatch_log_group" "logs" {
  name              = "/aws/events/eventbus_events/logs"
  retention_in_days = 5
}

#Policy allows EventBridge to publish events to the log group
data "aws_iam_policy_document" "eventbridge_log_policy" {
 statement {
   actions = [
    "logs:*",
    ]

    resources = [
      "${aws_cloudwatch_log_group.logs.arn}:*"
    ]

    principals {
      identifiers = ["events.amazonaws.com", "delivery.logs.amazonaws.com"]
      type        = "Service"
    }
    principals {
        type        = "AWS"
        identifiers = [var.aws_account_id]
    }
  }
}

resource "aws_cloudwatch_log_resource_policy" "logs" {
  policy_document = data.aws_iam_policy_document.eventbridge_log_policy.json
  policy_name     = "log-publishing-policy"
}

#Creates event target for the event rule to push matching events to the log group.
resource "aws_cloudwatch_event_target" "eventrule" {
  rule = aws_cloudwatch_event_rule.eventrule.name
  event_bus_name = aws_cloudwatch_event_bus.messenger.name
  arn  = aws_cloudwatch_log_group.logs.arn
}

Considerations:

• We considered using S3 notifications instead of sending custom events through our bash scripts. We went the custom event route as we wanted to include specific information about the Hadoop source and AWS glue tables that our consuming accounts would find useful instead of the generic S3 event that AWS provides.
• Tools like airflow/ prefect etc weren't used in the Distcp to central AWS account process as the Hadoop team and AWS central data account teams are different and their processes are required to be decoupled. We also needed a method to notify the consuming data accounts that their data has been updated. This solution satisfies those requirements.
• The "source" for all the custom events is the same and the central data account sends all the events to the consumer accounts. It's up to the consumers to tailor their EventBridge Rules to the specific datasets they're interested in. This allows the architecture to be more scalable and maintainable as the central data account doesn't have to write custom code for each consuming account and it makes onboarding new consumers easier since datasets used and requirements are constantly changing.
• We are using a Custom Event Buses in both the central data and consuming accounts. We opted not to use the default event bus to receive the events because according to AWS best practices the default bus should be kept localized to your account since all the events from your AWS services flow to this event bus. This architecture also requires the consuming accounts to give the central data account access to their event bus. Doing this on the default event bus poses a security risk.