[PROD] Daily DB backup by CI & "Resource Dashboard" in left sidebar

.circleci/config.yml

@@ -43,7 +43,37 @@ executors:
   machine-executor:
     machine:
       image: ubuntu-2204:current
+  aws-executor:
+    docker:
+      - image: cimg/aws:2024.03
 commands:
+  sparse_checkout:
+    description: "Checkout a sparse directory from a specific branch."
+    parameters:
+      directory:
+        type: string
+        description: "Directory to checkout sparsely"
+      branch:
+        type: string
+        description: "Branch to checkout"
+    steps:
+      - run:
+          name: Install Git
+          command: |
+            sudo apt-get update && sudo apt-get install -y git
+      - run:
+          name: Clone Repository
+          command: |
+            git clone --no-checkout --filter=blob:none << pipeline.project.git_url >>.git .
+      - run:
+          name: Setup Sparse Checkout
+          command: |
+            git config core.sparseCheckout true
+            echo "<< parameters.directory >>/*" >> .git/info/sparse-checkout
+      - run:
+          name: Checkout Branch
+          command: |
+            git checkout << parameters.branch >>
   create_combined_yarnlock:
     description: "Concatenate all yarn.json files into single file.
       File is used as checksum source for part of caching key."
@@ -253,6 +283,108 @@ commands:
       #     name: Push maintenance application
       #     command: |
       #       cd maintenance_page && cf push -s cflinuxfs4 --vars-file ../<<parameters.deploy_config_file >>
+  cf_backup:
+    description: "Login to cloud foundry space with service account credentials, Connect to DB & S3, backup DB to S3"
+    parameters:
+      auth_client_secret:
+        description: "Name of CircleCi project environment variable that
+          holds authentication client secret, a required application variable"
+        type: env_var_name
+      cloudgov_username:
+        description: "Name of CircleCi project environment variable that
+          holds deployer username for cloudgov space"
+        type: env_var_name
+      cloudgov_password:
+        description: "Name of CircleCi project environment variable that
+          holds deployer password for cloudgov space"
+        type: env_var_name
+      cloudgov_space:
+        description: "Name of CircleCi project environment variable that
+          holds name of cloudgov space to target for application deployment"
+        type: env_var_name
+      rds_service_name:
+        description: "Name of the rds service to backup"
+        type: string
+      s3_service_name:
+        description: "Name of the s3 service access"
+        type: string
+      backup_prefix:
+        description: "prefix name to use for backups"
+        type: string
+    steps:
+      - run:
+          name: Install Dependencies
+          command: |
+            set -e
+            set -u
+            set -o pipefail
+            set -o noglob
+            set -o noclobber
+
+            # update
+            sudo apt-get update
+            # Install uuid-runtime to have access to uuidgen
+            # Install pv wget
+            sudo apt-get install pv uuid-runtime wget
+            # Install Cloud Foundry CLI
+            wget -q -O - https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key | sudo apt-key add -
+            echo "deb https://packages.cloudfoundry.org/debian stable main" | sudo tee /etc/apt/sources.list.d/cloudfoundry-cli.list
+            sudo apt-get update
+            sudo apt-get install cf8-cli
+            # Install plugin needed for connect-to-service
+            cf install-plugin -f https://github.com/cloud-gov/cf-service-connect/releases/download/v1.1.3/cf-service-connect_linux_amd64
+
+            # The line you want to ensure exists in the /etc/hosts file
+            line="127.0.0.1        localhost"
+
+            # Check if the line already exists
+            if ! grep -qF "$line" /etc/hosts; then
+                # If the line does not exist, append it
+                echo "$line" | sudo tee -a /etc/hosts > /dev/null
+                echo "Line added to /etc/hosts"
+            else
+                echo "Line already exists in /etc/hosts"
+            fi
+
+            # cleanup
+            sudo rm -rf /var/lib/apt/lists/*
+      - run:
+          name: Login with service account
+          command: |
+            cf login -a << pipeline.parameters.cg_api >> \
+              -u ${<< parameters.cloudgov_username >>} \
+              -p ${<< parameters.cloudgov_password >>} \
+              -o << pipeline.parameters.cg_org >> \
+              -s ${<< parameters.cloudgov_space >>}
+      - run:
+          name: cf_lambda - script to trigger backup
+          command: |
+            set -x
+            json_data=$(jq -n \
+              --arg automation_dir "./automation" \
+              --arg manifest "manifest.yml" \
+              --arg task_name "backup" \
+              --arg command "cd /home/vcap/app/db-backup/scripts; bash ./db_backup.sh" \
+              --argjson args '["<< parameters.backup_prefix >>", "<< parameters.rds_service_name >>", "<< parameters.s3_service_name >>"]' \
+              '{
+                automation_dir: $automation_dir,
+                manifest: $manifest,
+                task_name: $task_name,
+                command: $command,
+                args: $args
+              }')
+
+            # Set execute permission
+            find ./automation -name "*.sh" -exec chmod +x {} \;
+
+            ./automation/ci/scripts/cf_lambda.sh "$json_data"
+          environment:
+            CF_RDS_SERVICE_NAME: ttahub-prod
+            CF_S3_SERVICE_NAME: ttahub-db-backups
+      - run:
+          name: Logout of service account
+          command: |
+            cf logout
 parameters:
   cg_org:
     description: "Cloud Foundry cloud.gov organization name"
@@ -305,6 +437,9 @@ parameters:
   sandbox_new_relic_app_id:
     default: "867346799"
     type: string
+  manual-trigger:
+    type: boolean
+    default: false
 jobs:
   build_and_lint:
     executor: docker-executor
@@ -888,8 +1023,25 @@ jobs:
                 slack_bot_token: $SLACK_BOT_TOKEN
                 slack_channel: "acf-ohs-ttahub--contractor-customer-team"
     resource_class: large
+  backup_upload_production:
+    docker:
+      - image: cimg/base:2024.05
+    steps:
+    - sparse_checkout:
+          directory: 'automation'
+          branch: << pipeline.git.branch >>
+    - cf_backup:
+          auth_client_secret: PROD_AUTH_CLIENT_SECRET
+          cloudgov_username: CLOUDGOV_PROD_USERNAME
+          cloudgov_password: CLOUDGOV_PROD_PASSWORD
+          cloudgov_space: CLOUDGOV_PROD_SPACE
+          rds_service_name: ttahub-prod
+          s3_service_name: ttahub-db-backups
+          backup_prefix: production
 workflows:
   build_test_deploy:
+    when:
+      equal: [false, << pipeline.parameters.manual-trigger >>]
     jobs:
       - build_and_lint
       - build_and_lint_similarity_api
@@ -968,3 +1120,18 @@ workflows:
       - dynamic_security_scan:
           requires:
             - build_and_lint
+  daily_backup_upload_production:
+    triggers:
+      - schedule:
+          cron: "0 10 * * 1-5"
+          filters:
+            branches:
+              only:
+                - << pipeline.parameters.prod_git_branch >>
+    jobs:
+      - backup_upload_production
+  manual_backup_upload_production:
+    when:
+      equal: [true, << pipeline.parameters.manual-trigger >>]
+    jobs:
+      - backup_upload_production

automation/.cfignore

@@ -0,0 +1,7 @@
+# Ignore everything
+*
+
+!cf/scripts/*
+!common/scripts/*
+!db-backup/scripts/*
+!.cfignore

automation/cf/scripts/idol.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+while true; do
+  # Check if the 'stop' file exists in the root or /tmp directory
+  if [ -f /stop ] || [ -f /tmp/stop ]; then
+    echo "Stop file found. Exiting loop."
+    break
+  fi
+  sleep 1
+done

automation/ci/scripts/README.md

@@ -0,0 +1,75 @@
+# CF Lambda Deployment Tool
+
+The `cf_lambda.sh` script automates the deployment of applications to Cloud Foundry (CF) and manages tasks such as pushing applications, managing service keys, and handling various app states.
+
+## Features
+
+- **Automated App Deployment**: Streamlines the deployment process for applications using Cloud Foundry.
+- **Service Key Management**: Automates the creation and deletion of service keys.
+- **Application Lifecycle Management**: Provides functionalities to start, stop, and delete applications as needed.
+- **Task Execution and Monitoring**: Facilitates running and monitoring tasks within the deployed applications.
+- **Logging**: Enhanced logging capabilities that provide detailed feedback and timestamps for tracking the deployment process.
+
+## Prerequisites
+
+Before using this script, ensure the following prerequisites are met:
+
+- **Cloud Foundry CLI**: Installed and configured with access to your Cloud Foundry environment.
+- **jq**: Installed on the system to handle JSON data manipulation.
+- **Properly Configured JSON Input**: Input parameters for the script must be provided in JSON format including details such as directory paths, manifest details, and task commands.
+
+## Installation
+
+1. Download the script to your system where you have Cloud Foundry CLI installed.
+2. Ensure the script is executable:
+   ```bash
+   chmod +x cf_lambda.sh
+   ```
+
+3. Verify all dependencies mentioned are correctly installed and accessible in your environment.
+
+## Usage
+
+To use this script, you need to provide a JSON input file containing the necessary configuration. Here’s how you can run the script:
+
+```bash
+./cf_lambda.sh 'your_json_input_here'
+```
+
+### Sample JSON Input
+
+```json
+{
+  "automation_dir": "./path_to_automation_directory",
+  "manifest": "manifest.yml",
+  "task_name": "deploy-task",
+  "command": "bash deploy_script.sh",
+  "args": "argument1 argument2"
+}
+```
+
+- **automation_dir**: Directory where your automation scripts and manifest file are located.
+- **manifest**: Name of the manifest file used by Cloud Foundry to deploy the application.
+- **task_name**: A specific name for the task to be executed as part of the app deployment.
+- **command**: The command to be executed as a task.
+- **args**: Arguments to be passed along with the command.
+
+## Detailed Workflow
+
+1. **Preparation**: Validate and parse the JSON input.
+2. **Deployment**: Use the manifest file to push the application to Cloud Foundary.
+3. **Task Management**: Execute specified tasks within the context of the deployed application and monitor their execution.
+4. **Service Key Handling**: Automatically manage service keys needed for various services like databases or third-party APIs.
+5. **Cleanup**: Optionally stop or delete the application post-task execution.
+
+## Logging and Monitoring
+
+Logs provide detailed information including timestamps for each step of the deployment and task execution process, which is crucial for debugging and verifying the deployment.
+
+## Contributions
+
+Contributions to this script are welcome. Please ensure that any modifications maintain the integrity of the deployment processes and include adequate error handling and validation.
+
+## License
+
+Specify the license under which this script is shared (if applicable), ensuring users are aware of their rights to use, modify, and distribute the script.