Test Reporting в CI/CD

<?xml version="1.0" encoding="UTF-8"?>
<testsuites name="Test Suite" tests="10" failures="2" errors="0" time="45.231">
  <testsuite name="UserAuthentication" tests="5" failures="1" time="12.456">
    <testcase name="test_login_valid_credentials" classname="auth.test" time="2.345">
      <system-out>User logged in successfully</system-out>
    </testcase>
    <testcase name="test_login_invalid_password" classname="auth.test" time="1.987">
      <failure message="AssertionError: Expected 401, got 500" type="AssertionError">
        Traceback (most recent call last):
          File "auth/test.py", line 45, in test_login_invalid_password
            assert response.status_code == 401
        AssertionError: Expected 401, got 500
      </failure>
    </testcase>
  </testsuite>
</testsuites>

{
  "jest": {
    "reporters": [
      "default",
      ["jest-junit", {
        "outputDirectory": "test-results",
        "outputName": "junit.xml",
        "classNameTemplate": "{classname}",
        "titleTemplate": "{title}",
        "ancestorSeparator": " › "
      }]
    ]
  }
}

pytest --junitxml=test-results/junit.xml --html=test-results/report.html

go test -v ./... | go-junit-report > test-results/junit.xml

name: Test and Report

on: [push, pull_request]

jobs:
  test:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: Run tests
        run: npm test -- --coverage

      - name: Publish Test Results
        uses: EnricoMi/publish-unit-test-result-action@v2
        if: always()
        with:
          files: test-results/**/*.xml
          check_name: Test Results
          comment_title: Test Report

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v3
        with:
          files: ./coverage/coverage.xml
          flags: unittests
          name: codecov-umbrella

      - name: Generate Job Summary
        if: always()
        run: |
          echo "## Test Results" >> $GITHUB_STEP_SUMMARY
          echo "Total: $(grep -o 'tests="[0-9]*"' test-results/junit.xml | head -1 | grep -o '[0-9]*')" >> $GITHUB_STEP_SUMMARY
          echo "Failures: $(grep -o 'failures="[0-9]*"' test-results/junit.xml | head -1 | grep -o '[0-9]*')" >> $GITHUB_STEP_SUMMARY

// report-publisher.js
const { InfluxDB, Point } = require('@influxdata/influxdb-client');

async function publishTestMetrics(results) {
  const client = new InfluxDB({
    url: process.env.INFLUX_URL,
    token: process.env.INFLUX_TOKEN
  });

  const writeApi = client.getWriteApi(
    process.env.INFLUX_ORG,
    process.env.INFLUX_BUCKET
  );

  const point = new Point('test_run')
    .tag('branch', process.env.BRANCH_NAME)
    .tag('environment', process.env.ENV)
    .intField('total_tests', results.total)
    .intField('passed', results.passed)
    .intField('failed', results.failed)
    .floatField('duration_seconds', results.duration)
    .floatField('pass_rate', (results.passed / results.total) * 100);

  writeApi.writePoint(point);
  await writeApi.close();
}

# flakiness_tracker.py
import json
from datetime import datetime, timedelta
from collections import defaultdict

class FlakinessTracker:
    def __init__(self, history_file='test_history.json'):
        self.history_file = history_file
        self.load_history()

    def load_history(self):
        try:
            with open(self.history_file, 'r') as f:
                self.history = json.load(f)
        except FileNotFoundError:
            self.history = defaultdict(list)

    def record_result(self, test_name, passed, duration):
        self.history[test_name].append({
            'timestamp': datetime.now().isoformat(),
            'passed': passed,
            'duration': duration
        })
        # Сохраняйте только последние 100 запусков
        self.history[test_name] = self.history[test_name][-100:]
        self.save_history()

    def calculate_flakiness(self, test_name, lookback_days=7):
        if test_name not in self.history:
            return 0.0

        cutoff = datetime.now() - timedelta(days=lookback_days)
        recent_runs = [
            r for r in self.history[test_name]
            if datetime.fromisoformat(r['timestamp']) > cutoff
        ]

        if len(recent_runs) < 10:  # Нужны минимальные данные
            return 0.0

        # Вычислить нестабильность: переходы между прошло/упало
        transitions = 0
        for i in range(1, len(recent_runs)):
            if recent_runs[i]['passed'] != recent_runs[i-1]['passed']:
                transitions += 1

        return transitions / len(recent_runs)

    def get_flaky_tests(self, threshold=0.2):
        flaky = {}
        for test_name in self.history:
            flakiness = self.calculate_flakiness(test_name)
            if flakiness > threshold:
                flaky[test_name] = flakiness
        return sorted(flaky.items(), key=lambda x: x[1], reverse=True)

# .github/workflows/parallel-tests.yml
name: Parallel Testing with Aggregation

jobs:
  test:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        shard: [1, 2, 3, 4]

    steps:
      - uses: actions/checkout@v4

      - name: Run test shard
        run: |
          npm test -- --shard=${{ matrix.shard }}/4 \
            --reporter=junit \
            --outputFile=test-results/junit-${{ matrix.shard }}.xml

      - name: Upload shard results
        uses: actions/upload-artifact@v3
        with:
          name: test-results-${{ matrix.shard }}
          path: test-results/

  aggregate:
    needs: test
    runs-on: ubuntu-latest
    if: always()

    steps:
      - name: Download all results
        uses: actions/download-artifact@v3
        with:
          path: all-results/

      - name: Merge and analyze results
        run: |
          python scripts/merge_reports.py all-results/ merged-report.xml
          python scripts/analyze_trends.py merged-report.xml

      - name: Publish aggregated report
        uses: EnricoMi/publish-unit-test-result-action@v2
        with:
          files: merged-report.xml

// visual-regression-reporter.js
const { compareScreenshots } = require('pixelmatch');
const fs = require('fs');

async function generateVisualReport(baseline, current, output) {
  const diff = await compareScreenshots(baseline, current, {
    threshold: 0.1,
    includeAA: true
  });

  const report = {
    timestamp: new Date().toISOString(),
    baseline: baseline,
    current: current,
    diff: output,
    pixelsDifferent: diff.pixelsDifferent,
    percentageDifferent: diff.percentage,
    passed: diff.percentage < 0.5
  };

  // Генерировать HTML отчет
  const html = `
    <!DOCTYPE html>
    <html>
    <head><title>Visual Regression Report</title></head>
    <body>
      <h1>Visual Regression Results</h1>
      <p>Difference: ${diff.percentage.toFixed(2)}%</p>
      <div style="display: flex;">
        <div>
          <h2>Baseline</h2>
          <img src="${baseline}" />
        </div>
        <div>
          <h2>Current</h2>
          <img src="${current}" />
        </div>
        <div>
          <h2>Diff</h2>
          <img src="${output}" />
        </div>
      </div>
    </body>
    </html>
  `;

  fs.writeFileSync('visual-report.html', html);
  return report;
}

# Пример отчета о chaos тесте в стиле Netflix
chaos_test_results:
  scenario: "Database Primary Failover"
  duration: 300s
  outcome: PASS
  metrics:
    - error_rate: 0.02%  # В пределах 5% порога
    - latency_p99: 245ms  # Ниже порога 500ms
    - traffic_success: 99.98%
  events:
    - timestamp: "10:30:15"
      action: "Terminated primary DB instance"
    - timestamp: "10:30:17"
      observation: "Automatic failover initiated"
    - timestamp: "10:30:22"
      observation: "All traffic routed to secondary"
  recommendation: "System resilient to DB primary failures"

// canary-report.js
const canaryReport = {
  deployment_id: "deploy-12345",
  canary_percentage: 5,
  duration_minutes: 30,
  metrics_comparison: {
    error_rate: {
      baseline: 0.1,
      canary: 0.12,
      threshold: 0.15,
      status: "PASS"
    },
    latency_p50: {
      baseline: 45,
      canary: 48,
      threshold: 60,
      status: "PASS"
    },
    latency_p99: {
      baseline: 250,
      canary: 310,
      threshold: 300,
      status: "FAIL"
    }
  },
  decision: "ROLLBACK",
  reason: "P99 latency exceeded threshold by 10ms"
};

# Стратегии оптимизации отчетов
optimization:
  # Хранить детальные логи только для сбоев
  log_level:
    passed: summary
    failed: detailed

  # Сжимать вложения
  attachments:
    screenshots: webp  # На 30% меньше чем PNG
    videos: h264      # Сжатый формат
    logs: gzip        # Сжимать текстовые логи

  # Политика хранения
  retention:
    passing_builds: 30_days
    failing_builds: 90_days
    critical_failures: 1_year

<!-- Пример сворачиваемого отчета о тесте -->
<div class="test-suite">
  <h2>Authentication Tests (5/6 passed) ❌</h2>
  <details>
    <summary>✅ test_login_valid_credentials (2.3s)</summary>
    <pre>Логи доступны по требованию</pre>
  </details>
  <details open>
    <summary>❌ test_password_reset (FAILED)</summary>
    <pre class="error">
      AssertionError at line 67
      Expected: 200
      Actual: 500
      Stack trace: ...
    </pre>
    <img src="screenshot.png" alt="Failure screenshot" />
  </details>
</div>

# quality_metrics.py
metrics_to_track = {
    'test_count': 'Total number of tests',
    'pass_rate': 'Percentage of passing tests',
    'avg_duration': 'Average test suite duration',
    'flaky_test_count': 'Number of flaky tests',
    'code_coverage': 'Percentage of code covered',
    'time_to_fix': 'Average time from failure to fix'
}

# Оповещать если метрики деградируют
thresholds = {
    'pass_rate': {'min': 95.0, 'trend': 'up'},
    'avg_duration': {'max': 600, 'trend': 'down'},
    'flaky_test_count': {'max': 10, 'trend': 'down'}
}

// Умная фильтрация отчетов
const reportView = {
  default: {
    show: ['failed_tests', 'flaky_tests', 'new_failures'],
    hide: ['passed_tests', 'skipped_tests']
  },
  detailed: {
    show: ['all_tests', 'coverage', 'performance'],
    expandable: true
  },
  executive: {
    show: ['summary_stats', 'trends', 'quality_score'],
    format: 'high_level'
  }
};

- name: Check test performance
  run: |
    CURRENT_DURATION=$(jq '.duration' test-results/summary.json)
    BASELINE_DURATION=$(curl -s $BASELINE_URL | jq '.duration')
    INCREASE=$(echo "scale=2; ($CURRENT_DURATION - $BASELINE_DURATION) / $BASELINE_DURATION * 100" | bc)

    if (( $(echo "$INCREASE > 20" | bc -l) )); then
      echo "⚠️ Test duration increased by ${INCREASE}%"
      exit 1
    fi

failure_categories = {
    'BLOCKER': {
        'criteria': ['security', 'data_loss', 'service_down'],
        'priority': 1,
        'notify': ['team_lead', 'on_call']
    },
    'CRITICAL': {
        'criteria': ['core_feature', 'payment', 'authentication'],
        'priority': 2,
        'notify': ['team_lead']
    },
    'MAJOR': {
        'criteria': ['user_facing', 'performance'],
        'priority': 3,
        'notify': ['developer']
    },
    'MINOR': {
        'criteria': ['edge_case', 'cosmetic'],
        'priority': 4,
        'notify': ['developer']
    }
}