Cost Optimization for CI/CD

# ci_cost_tracker.py
import json
from datetime import datetime, timedelta

class CICostTracker:
    # GitHub Actions pricing (as of 2025)
    PRICING = {
        'ubuntu': 0.008,      # per minute
        'windows': 0.016,     # per minute
        'macos': 0.08,        # per minute
        'macos_m1': 0.16,     # per minute (ARM)
        'storage_gb': 0.25,   # per GB/month
        'network_gb': 0.50    # per GB transfer
    }

    def calculate_workflow_cost(self, workflow_data):
        """Calculate cost for a single workflow run"""
        total_cost = 0

        for job in workflow_data['jobs']:
            runner_type = job['runs_on']
            duration_minutes = job['duration_seconds'] / 60

            # Compute cost
            compute_cost = duration_minutes * self.PRICING.get(runner_type, 0.008)
            total_cost += compute_cost

        return {
            'workflow_id': workflow_data['id'],
            'total_cost': round(total_cost, 4),
            'duration_minutes': sum(j['duration_seconds'] for j in workflow_data['jobs']) / 60,
            'runner_breakdown': self._calculate_runner_breakdown(workflow_data['jobs'])
        }

    def analyze_cost_trends(self, days=30):
        """Analyze cost trends over time"""
        workflows = self.fetch_workflows(days)

        daily_costs = {}
        for workflow in workflows:
            date = workflow['created_at'].split('T')[0]
            if date not in daily_costs:
                daily_costs[date] = 0
            daily_costs[date] += self.calculate_workflow_cost(workflow)['total_cost']

        # Identify cost spikes
        avg_daily_cost = sum(daily_costs.values()) / len(daily_costs)
        spikes = {
            date: cost for date, cost in daily_costs.items()
            if cost > avg_daily_cost * 1.5
        }

        return {
            'total_cost': sum(daily_costs.values()),
            'average_daily': round(avg_daily_cost, 2),
            'peak_day': max(daily_costs.items(), key=lambda x: x[1]),
            'cost_spikes': spikes,
            'projection_monthly': round(avg_daily_cost * 30, 2)
        }

    def identify_cost_hotspots(self):
        """Identify workflows/jobs with highest costs"""
        workflows = self.fetch_workflows(days=7)

        workflow_costs = {}
        for wf in workflows:
            name = wf['name']
            cost = self.calculate_workflow_cost(wf)['total_cost']

            if name not in workflow_costs:
                workflow_costs[name] = {'cost': 0, 'runs': 0}
            workflow_costs[name]['cost'] += cost
            workflow_costs[name]['runs'] += 1

        # Calculate cost per run
        for name, data in workflow_costs.items():
            data['cost_per_run'] = data['cost'] / data['runs']

        # Sort by total cost
        hotspots = sorted(
            workflow_costs.items(),
            key=lambda x: x[1]['cost'],
            reverse=True
        )[:10]

        return hotspots

# .github/workflows/cost-report.yml
name: Weekly Cost Report

on:
  schedule:
    - cron: '0 0 * * 1'  # Every Monday
  workflow_dispatch:

jobs:
  generate-report:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: Fetch workflow data
        run: |
          gh api /repos/${{ github.repository }}/actions/runs \
            --paginate \
            --jq '.workflow_runs[] | {id, name, created_at, conclusion, run_started_at}' \
            > workflows.json

      - name: Calculate costs
        run: |
          python3 scripts/ci_cost_tracker.py \
            --input workflows.json \
            --output cost-report.html \
            --format html

      - name: Upload report
        uses: actions/upload-artifact@v3
        with:
          name: cost-report
          path: cost-report.html

      - name: Post to Slack
        run: |
          TOTAL_COST=$(jq '.total_cost' cost-summary.json)
          INCREASE=$(jq '.week_over_week_change' cost-summary.json)

          curl -X POST ${{ secrets.SLACK_WEBHOOK }} \
            -H 'Content-Type: application/json' \
            -d "{\"text\": \"📊 Weekly CI/CD Cost Report\n💰 Total: \$$TOTAL_COST\n📈 Change: ${INCREASE}%\n🔗 Full report: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\"}"

# Before: Sequential tests (60 minutes)
jobs:
  test:
    runs-on: ubuntu-latest
    steps:
      - run: npm test  # Runs all 10,000 tests

# After: Parallel tests (15 minutes) - 4x faster, same cost
jobs:
  test:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        shard: [1, 2, 3, 4]
    steps:
      - run: npm test -- --shard=${{ matrix.shard }}/4

# test_selector.py
import subprocess
import json

def get_changed_files():
    """Get files changed in current commit"""
    result = subprocess.run(
        ['git', 'diff', '--name-only', 'HEAD~1'],
        capture_output=True, text=True
    )
    return result.stdout.strip().split('\n')

def select_tests(changed_files):
    """Select only tests affected by changes"""
    test_mapping = json.load(open('test-mapping.json'))

    tests_to_run = set()
    for file in changed_files:
        # Find tests that depend on this file
        if file in test_mapping:
            tests_to_run.update(test_mapping[file])

    # Always run critical tests
    tests_to_run.update(get_critical_tests())

    return list(tests_to_run)

# In CI
changed = get_changed_files()
if not changed or 'core/' in changed:
    # Run all tests for core changes
    run_command('npm test')
else:
    # Run only affected tests (potential 70% reduction)
    selected_tests = select_tests(changed)
    run_command(f'npm test {" ".join(selected_tests)}')

# Before: 2GB image, 10-minute build
FROM node:18
WORKDIR /app
COPY . .
RUN npm install
RUN npm run build
CMD ["npm", "start"]

# After: 200MB image, 3-minute build
FROM node:18-alpine AS builder
WORKDIR /app
COPY package*.json ./
RUN npm ci --only=production
COPY . .
RUN npm run build

FROM node:18-alpine
WORKDIR /app
COPY --from=builder /app/dist ./dist
COPY --from=builder /app/node_modules ./node_modules
CMD ["node", "dist/index.js"]

- name: Build with cache
  uses: docker/build-push-action@v5
  with:
    context: .
    cache-from: type=gha
    cache-to: type=gha,mode=max
    push: true
    tags: myapp:latest

# Reduces build time from 10min to 2min (80% reduction)

jobs:
  lint:
    runs-on: ubuntu-latest  # $0.008/min
    steps:
      - run: npm run lint

  test-unit:
    runs-on: ubuntu-latest  # $0.008/min
    steps:
      - run: npm test

  test-e2e:
    runs-on: ubuntu-latest-4-cores  # $0.016/min but 2x faster
    steps:
      - run: npm run test:e2e

  build-mac:
    runs-on: macos-latest  # $0.08/min - only when necessary
    if: contains(github.event.head_commit.message, '[build-mac]')
    steps:
      - run: npm run build:mac

name: Smart CI

on: [push, pull_request]

jobs:
  changes:
    runs-on: ubuntu-latest
    outputs:
      backend: ${{ steps.filter.outputs.backend }}
      frontend: ${{ steps.filter.outputs.frontend }}
      docs: ${{ steps.filter.outputs.docs }}
    steps:
      - uses: actions/checkout@v4
      - uses: dorny/paths-filter@v2
        id: filter
        with:
          filters: |
            backend:
              - 'src/backend/**'
              - 'package.json'
            frontend:
              - 'src/frontend/**'
              - 'package.json'
            docs:
              - 'docs/**'
              - '**.md'

  test-backend:
    needs: changes
    if: needs.changes.outputs.backend == 'true'
    runs-on: ubuntu-latest
    steps:
      - run: npm run test:backend

  test-frontend:
    needs: changes
    if: needs.changes.outputs.frontend == 'true'
    runs-on: ubuntu-latest
    steps:
      - run: npm run test:frontend

  # Skip expensive builds for docs-only changes
  build:
    needs: changes
    if: needs.changes.outputs.docs != 'true'
    runs-on: ubuntu-latest
    steps:
      - run: npm run build

# Before: Storing 5GB of artifacts per build
- uses: actions/upload-artifact@v3
  with:
    name: build-output
    path: |
      dist/
      logs/
      coverage/
    retention-days: 90  # Expensive!

# After: Selective storage with shorter retention
- uses: actions/upload-artifact@v3
  with:
    name: build-output
    path: dist/
    retention-days: 7  # 90% cost reduction

- uses: actions/upload-artifact@v3
  if: failure()  # Only upload logs on failure
  with:
    name: debug-logs
    path: logs/
    retention-days: 14

# Cost comparison for 10,000 minutes/month

# GitHub hosted:
# 10,000 min × $0.008 = $80/month

# Self-hosted (AWS EC2 t3.large):
# $75/month + $5 storage = $80/month
# But handles 20,000+ minutes/month = $0.004/min effective

# Self-hosted (AWS EC2 c6i.4xlarge with spot):
# $150/month spot price
# Handles 100,000+ minutes/month = $0.0015/min
# Savings: 81% vs GitHub hosted

- name: Cache dependencies
  uses: actions/cache@v3
  with:
    path: |
      ~/.npm
      ~/.cache
      node_modules
    key: ${{ runner.os }}-deps-${{ hashFiles('**/package-lock.json') }}
    restore-keys: |
      ${{ runner.os }}-deps-

- name: Cache build
  uses: actions/cache@v3
  with:
    path: dist
    key: ${{ runner.os }}-build-${{ github.sha }}
    restore-keys: |
      ${{ runner.os }}-build-

# Reduces npm install from 2min to 10sec (90% reduction)

# budget_monitor.py
def check_budget(current_spend, budget):
    utilization = (current_spend / budget) * 100

    if utilization >= 90:
        send_alert('CRITICAL', f'At {utilization}% of budget')
        disable_non_critical_workflows()
    elif utilization >= 75:
        send_alert('WARNING', f'At {utilization}% of budget')

    return utilization

# Run daily
if __name__ == '__main__':
    monthly_budget = 5000  # $5,000
    current_spend = calculate_month_to_date_spend()
    check_budget(current_spend, monthly_budget)

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true  # Cancel old runs on new push

# Saves costs on rapid pushes
# Example: 5 pushes in 10 minutes
# Before: 5 × 10min = 50 minutes
# After: 1 × 10min = 10 minutes (80% savings)

# Run expensive jobs during off-hours
name: Nightly Full Test Suite

on:
  schedule:
    - cron: '0 2 * * *'  # 2 AM UTC
  workflow_dispatch:

# Use spot instances for scheduled jobs (60-90% cheaper)