Monitoring and Maintenance

Comprehensive monitoring and maintenance procedures ensure the Stratpoint Timesheet Application operates reliably, performs optimally, and maintains high availability. This section covers monitoring strategies, alerting systems, and maintenance procedures.

Monitoring Architecture

Monitoring Stack Overview

graph TB
    A[Application Servers] --> B[Metrics Collection]
    C[Database Servers] --> B
    D[Cache Servers] --> B
    E[Queue Workers] --> B

    B --> F[Prometheus]
    F --> G[Grafana Dashboards]
    F --> H[AlertManager]

    I[Log Aggregation] --> J[ELK Stack]
    J --> K[Kibana Dashboards]

    L[APM Monitoring] --> M[New Relic/DataDog]

    H --> N[Notification Channels]
    N --> O[Email Alerts]
    N --> P[Slack Notifications]
    N --> Q[SMS Alerts]
    N --> R[PagerDuty]

Monitoring Components

Infrastructure Monitoring: - Server resource utilization (CPU, Memory, Disk, Network) - Database performance and connections - Cache hit rates and memory usage - Load balancer health and distribution

Application Monitoring: - Response times and throughput - Error rates and exception tracking - Queue job processing and failures - User session and authentication metrics

Business Monitoring: - Timesheet submission rates - Approval workflow performance - Integration sync status - User activity patterns

Application Performance Monitoring (APM)

Laravel Application Metrics

Performance Metrics Collection:

<?php

namespace App\Http\Middleware;

use Illuminate\Http\Request;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Redis;

class PerformanceMonitoring
{
    public function handle(Request $request, Closure $next)
    {
        $startTime = microtime(true);
        $startMemory = memory_get_usage(true);

        $response = $next($request);

        $endTime = microtime(true);
        $endMemory = memory_get_usage(true);

        $metrics = [
            'endpoint' => $request->path(),
            'method' => $request->method(),
            'response_time' => round(($endTime - $startTime) * 1000, 2), // milliseconds
            'memory_usage' => $endMemory - $startMemory,
            'status_code' => $response->getStatusCode(),
            'user_id' => auth()->id(),
            'timestamp' => now()->toISOString()
        ];

        // Send metrics to monitoring system
        $this->sendMetrics($metrics);

        // Log slow requests
        if ($metrics['response_time'] > 1000) {
            Log::warning('Slow request detected', $metrics);
        }

        return $response;
    }

    private function sendMetrics($metrics)
    {
        // Send to Prometheus
        Redis::lpush('metrics:api_requests', json_encode($metrics));

        // Send to APM service
        if (app()->bound('apm')) {
            app('apm')->recordMetric('api.response_time', $metrics['response_time'], [
                'endpoint' => $metrics['endpoint'],
                'method' => $metrics['method']
            ]);
        }
    }
}

Database Performance Monitoring

Query Performance Tracking:

<?php

namespace App\Providers;

use Illuminate\Database\Events\QueryExecuted;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\ServiceProvider;

class DatabaseMonitoringServiceProvider extends ServiceProvider
{
    public function boot()
    {
        DB::listen(function (QueryExecuted $query) {
            $this->logSlowQueries($query);
            $this->trackQueryMetrics($query);
        });
    }

    private function logSlowQueries(QueryExecuted $query)
    {
        if ($query->time > 1000) { // Log queries taking more than 1 second
            Log::warning('Slow database query detected', [
                'sql' => $query->sql,
                'bindings' => $query->bindings,
                'time' => $query->time,
                'connection' => $query->connectionName
            ]);
        }
    }

    private function trackQueryMetrics(QueryExecuted $query)
    {
        $metrics = [
            'query_time' => $query->time,
            'connection' => $query->connectionName,
            'query_type' => $this->getQueryType($query->sql),
            'timestamp' => now()->toISOString()
        ];

        Redis::lpush('metrics:database_queries', json_encode($metrics));
    }

    private function getQueryType($sql)
    {
        $sql = strtoupper(trim($sql));
        if (str_starts_with($sql, 'SELECT')) return 'SELECT';
        if (str_starts_with($sql, 'INSERT')) return 'INSERT';
        if (str_starts_with($sql, 'UPDATE')) return 'UPDATE';
        if (str_starts_with($sql, 'DELETE')) return 'DELETE';
        return 'OTHER';
    }
}

Health Checks and Endpoints

Application Health Check

<?php

namespace App\Http\Controllers;

use Illuminate\Http\Request;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Redis;
use Illuminate\Support\Facades\Cache;

class HealthCheckController extends Controller
{
    public function index()
    {
        $checks = [
            'database' => $this->checkDatabase(),
            'redis' => $this->checkRedis(),
            'cache' => $this->checkCache(),
            'queue' => $this->checkQueue(),
            'storage' => $this->checkStorage(),
            'external_apis' => $this->checkExternalAPIs()
        ];

        $overall = collect($checks)->every(fn($check) => $check['status'] === 'healthy');

        return response()->json([
            'status' => $overall ? 'healthy' : 'unhealthy',
            'timestamp' => now()->toISOString(),
            'checks' => $checks,
            'version' => config('app.version'),
            'environment' => config('app.env')
        ], $overall ? 200 : 503);
    }

    private function checkDatabase()
    {
        try {
            $start = microtime(true);
            DB::select('SELECT 1');
            $responseTime = round((microtime(true) - $start) * 1000, 2);

            return [
                'status' => 'healthy',
                'response_time' => $responseTime,
                'message' => 'Database connection successful'
            ];
        } catch (\Exception $e) {
            return [
                'status' => 'unhealthy',
                'message' => 'Database connection failed: ' . $e->getMessage()
            ];
        }
    }

    private function checkRedis()
    {
        try {
            $start = microtime(true);
            Redis::ping();
            $responseTime = round((microtime(true) - $start) * 1000, 2);

            return [
                'status' => 'healthy',
                'response_time' => $responseTime,
                'message' => 'Redis connection successful'
            ];
        } catch (\Exception $e) {
            return [
                'status' => 'unhealthy',
                'message' => 'Redis connection failed: ' . $e->getMessage()
            ];
        }
    }

    private function checkQueue()
    {
        try {
            $failedJobs = DB::table('failed_jobs')->count();
            $queueSize = Redis::llen('queues:default');

            $status = 'healthy';
            $message = 'Queue system operational';

            if ($failedJobs > 100) {
                $status = 'warning';
                $message = "High number of failed jobs: {$failedJobs}";
            }

            if ($queueSize > 1000) {
                $status = 'warning';
                $message = "Large queue backlog: {$queueSize} jobs";
            }

            return [
                'status' => $status,
                'failed_jobs' => $failedJobs,
                'queue_size' => $queueSize,
                'message' => $message
            ];
        } catch (\Exception $e) {
            return [
                'status' => 'unhealthy',
                'message' => 'Queue check failed: ' . $e->getMessage()
            ];
        }
    }

    private function checkExternalAPIs()
    {
        $apis = [
            'netsuite' => $this->checkNetSuiteAPI(),
            'sso' => $this->checkSSOAPI()
        ];

        $overallStatus = collect($apis)->every(fn($api) => $api['status'] === 'healthy') 
            ? 'healthy' : 'degraded';

        return [
            'status' => $overallStatus,
            'apis' => $apis
        ];
    }
}

Detailed Health Endpoints

// Specific component health checks
Route::get('/health/database', [HealthCheckController::class, 'database']);
Route::get('/health/redis', [HealthCheckController::class, 'redis']);
Route::get('/health/queue', [HealthCheckController::class, 'queue']);
Route::get('/health/storage', [HealthCheckController::class, 'storage']);

Alerting System

Alert Configuration

Prometheus Alert Rules:

# alerts.yml
groups:
  - name: timesheet_application
    rules:
      - alert: HighResponseTime
        expr: avg(api_response_time) > 2000
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High API response time detected"
          description: "Average response time is {{ $value }}ms"

      - alert: DatabaseConnectionFailure
        expr: database_connection_failures > 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Database connection failures detected"
          description: "{{ $value }} database connection failures in the last minute"

      - alert: QueueBacklog
        expr: queue_size > 1000
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Large queue backlog detected"
          description: "Queue size is {{ $value }} jobs"

      - alert: HighErrorRate
        expr: rate(http_requests_total{status=~"5.."}[5m]) > 0.1
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "High error rate detected"
          description: "Error rate is {{ $value }} requests per second"

Custom Alert Handlers

<?php

namespace App\Services;

use Illuminate\Support\Facades\Http;
use Illuminate\Support\Facades\Log;

class AlertService
{
    public function sendAlert($level, $title, $message, $context = [])
    {
        $alert = [
            'level' => $level,
            'title' => $title,
            'message' => $message,
            'context' => $context,
            'timestamp' => now()->toISOString(),
            'application' => 'timesheet',
            'environment' => config('app.env')
        ];

        // Log the alert
        Log::channel('alerts')->log($level, $title, $alert);

        // Send to notification channels based on severity
        switch ($level) {
            case 'critical':
                $this->sendToAllChannels($alert);
                break;
            case 'warning':
                $this->sendToSlack($alert);
                $this->sendToEmail($alert);
                break;
            case 'info':
                $this->sendToSlack($alert);
                break;
        }
    }

    private function sendToSlack($alert)
    {
        $webhook = config('monitoring.slack_webhook');

        if (!$webhook) return;

        $payload = [
            'text' => $alert['title'],
            'attachments' => [
                [
                    'color' => $this->getSlackColor($alert['level']),
                    'fields' => [
                        [
                            'title' => 'Level',
                            'value' => strtoupper($alert['level']),
                            'short' => true
                        ],
                        [
                            'title' => 'Environment',
                            'value' => $alert['environment'],
                            'short' => true
                        ],
                        [
                            'title' => 'Message',
                            'value' => $alert['message'],
                            'short' => false
                        ]
                    ],
                    'ts' => now()->timestamp
                ]
            ]
        ];

        Http::post($webhook, $payload);
    }

    private function sendToEmail($alert)
    {
        $recipients = config('monitoring.alert_emails');

        foreach ($recipients as $email) {
            Mail::to($email)->send(new AlertNotificationMail($alert));
        }
    }

    private function sendToPagerDuty($alert)
    {
        $integrationKey = config('monitoring.pagerduty_integration_key');

        if (!$integrationKey) return;

        $payload = [
            'routing_key' => $integrationKey,
            'event_action' => 'trigger',
            'payload' => [
                'summary' => $alert['title'],
                'severity' => $this->mapToPagerDutySeverity($alert['level']),
                'source' => 'timesheet-application',
                'custom_details' => $alert['context']
            ]
        ];

        Http::post('https://events.pagerduty.com/v2/enqueue', $payload);
    }
}

Log Management

Centralized Logging

Log Configuration:

// config/logging.php
'channels' => [
    'stack' => [
        'driver' => 'stack',
        'channels' => ['daily', 'elk'],
        'ignore_exceptions' => false,
    ],

    'elk' => [
        'driver' => 'custom',
        'via' => App\Logging\ELKLogger::class,
        'host' => env('ELK_HOST', 'localhost'),
        'port' => env('ELK_PORT', 5044),
        'index' => 'timesheet-logs',
    ],

    'alerts' => [
        'driver' => 'daily',
        'path' => storage_path('logs/alerts.log'),
        'level' => 'info',
        'days' => 30,
    ],

    'performance' => [
        'driver' => 'daily',
        'path' => storage_path('logs/performance.log'),
        'level' => 'info',
        'days' => 7,
    ],
]

Structured Logging

<?php

namespace App\Services;

use Illuminate\Support\Facades\Log;

class StructuredLogger
{
    public function logUserAction($action, $userId, $context = [])
    {
        Log::info('User action performed', [
            'action' => $action,
            'user_id' => $userId,
            'context' => $context,
            'timestamp' => now()->toISOString(),
            'session_id' => session()->getId(),
            'ip_address' => request()->ip(),
            'user_agent' => request()->userAgent()
        ]);
    }

    public function logAPIRequest($endpoint, $method, $responseTime, $statusCode)
    {
        Log::info('API request processed', [
            'endpoint' => $endpoint,
            'method' => $method,
            'response_time' => $responseTime,
            'status_code' => $statusCode,
            'timestamp' => now()->toISOString(),
            'user_id' => auth()->id()
        ]);
    }

    public function logIntegrationEvent($system, $operation, $status, $data = [])
    {
        Log::info('Integration event', [
            'system' => $system,
            'operation' => $operation,
            'status' => $status,
            'data' => $data,
            'timestamp' => now()->toISOString()
        ]);
    }
}

Maintenance Procedures

Automated Maintenance Tasks

Daily Maintenance:

// app/Console/Commands/DailyMaintenance.php
<?php

namespace App\Console\Commands;

use Illuminate\Console\Command;

class DailyMaintenance extends Command
{
    protected $signature = 'maintenance:daily';
    protected $description = 'Run daily maintenance tasks';

    public function handle()
    {
        $this->info('Starting daily maintenance...');

        // Clean up expired sessions
        $this->call('session:gc');

        // Clean up expired tokens
        $this->call('auth:clear-resets');

        // Optimize database
        $this->call('db:optimize');

        // Clear expired cache entries
        $this->call('cache:prune-stale-tags');

        // Generate daily reports
        $this->call('reports:generate-daily');

        // Check system health
        $this->call('health:check');

        $this->info('Daily maintenance completed.');
    }
}

Weekly Maintenance:

// app/Console/Commands/WeeklyMaintenance.php
<?php

namespace App\Console\Commands;

use Illuminate\Console\Command;

class WeeklyMaintenance extends Command
{
    protected $signature = 'maintenance:weekly';
    protected $description = 'Run weekly maintenance tasks';

    public function handle()
    {
        $this->info('Starting weekly maintenance...');

        // Database maintenance
        $this->call('db:analyze-tables');
        $this->call('db:optimize-tables');

        // Log rotation
        $this->call('logs:rotate');

        // Security scan
        $this->call('security:scan');

        // Performance analysis
        $this->call('performance:analyze');

        // Backup verification
        $this->call('backup:verify');

        $this->info('Weekly maintenance completed.');
    }
}

Performance Optimization

Database Optimization:

-- Weekly database optimization queries
ANALYZE TABLE users, projects, timelogs, project_users;
OPTIMIZE TABLE users, projects, timelogs, project_users;

-- Index analysis
SELECT 
    TABLE_NAME,
    INDEX_NAME,
    CARDINALITY,
    PAGES,
    FILTER_CONDITION
FROM INFORMATION_SCHEMA.STATISTICS 
WHERE TABLE_SCHEMA = 'timesheet'
ORDER BY CARDINALITY DESC;

-- Query performance analysis
SELECT 
    DIGEST_TEXT,
    COUNT_STAR,
    AVG_TIMER_WAIT/1000000000 as avg_time_ms,
    MAX_TIMER_WAIT/1000000000 as max_time_ms
FROM performance_schema.events_statements_summary_by_digest 
ORDER BY AVG_TIMER_WAIT DESC 
LIMIT 10;

Cache Optimization:

// Cache warming strategy
public function warmCache()
{
    // Warm frequently accessed data
    Cache::remember('active_projects', 3600, function () {
        return Project::where('status', 'active')->get();
    });

    Cache::remember('user_permissions', 3600, function () {
        return Permission::with('roles')->get();
    });

    Cache::remember('system_config', 3600, function () {
        return Config::all();
    });
}

This comprehensive monitoring and maintenance strategy ensures the Stratpoint Timesheet Application maintains optimal performance, reliability, and availability while providing early detection and resolution of potential issues.