From 7043429150f7f3fc77e5c869aa064aff8ca597d2 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Thu, 5 Feb 2026 00:44:11 +0300 Subject: [PATCH] Initial commit: Project foundation - Backend: Go API server with Gin framework - Frontend: React setup (placeholder) - ML Service: Python FastAPI skeleton - Docker Compose: Full stack configuration - Database: PostgreSQL schema with migrations - Documentation: Implementation plan and README Co-Authored-By: Claude Sonnet 4.5 --- .env.example | 23 + .gitignore | 20 + IMPLEMENTATION_PLAN.md | 1159 ++++++++++++++++++++++++ README.md | 93 ++ alertmanager.yml | 25 + backend/.gitignore | 22 + backend/Dockerfile | 40 + backend/configs/config.yaml | 39 + backend/configs/vmalert-rules/.gitkeep | 0 backend/go.mod | 12 + backend/internal/api/router.go | 186 ++++ backend/internal/config/config.go | 106 +++ backend/internal/models/user.go | 78 ++ backend/migrations/001_init.sql | 132 +++ docker-compose.yml | 152 ++++ frontend/.gitignore | 25 + frontend/Dockerfile | 34 + frontend/nginx.conf | 28 + ml-service/.gitignore | 26 + ml-service/Dockerfile | 27 + ml-service/app/__init__.py | 1 + ml-service/app/main.py | 122 +++ ml-service/app/models/.gitkeep | 0 ml-service/requirements.txt | 9 + 24 files changed, 2359 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 IMPLEMENTATION_PLAN.md create mode 100644 README.md create mode 100644 alertmanager.yml create mode 100644 backend/.gitignore create mode 100644 backend/Dockerfile create mode 100644 backend/configs/config.yaml create mode 100644 backend/configs/vmalert-rules/.gitkeep create mode 100644 backend/go.mod create mode 100644 backend/internal/api/router.go create mode 100644 backend/internal/config/config.go create mode 100644 backend/internal/models/user.go create mode 100644 backend/migrations/001_init.sql create mode 100644 docker-compose.yml create mode 100644 frontend/.gitignore create mode 100644 frontend/Dockerfile create mode 100644 frontend/nginx.conf create mode 100644 ml-service/.gitignore create mode 100644 ml-service/Dockerfile create mode 100644 ml-service/app/__init__.py create mode 100644 ml-service/app/main.py create mode 100644 ml-service/app/models/.gitkeep create mode 100644 ml-service/requirements.txt diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..feabe76 --- /dev/null +++ b/.env.example @@ -0,0 +1,23 @@ +# Database Configuration +DB_HOST=postgres +DB_PORT=5432 +DB_NAME=vlogs_manager +DB_USER=vlogs +DB_PASSWORD=your_secure_password_here + +# JWT Configuration +JWT_SECRET=your_jwt_secret_key_at_least_32_characters_long + +# VictoriaLogs Configuration +VLOGS_URL=http://victorialogs:9428 + +# vmalert Configuration +VMALERT_URL=http://vmalert:8880 +VMALERT_RULES_DIR=/app/rules + +# ML Service Configuration +ML_SERVICE_URL=http://ml-service:8000 + +# Application Configuration +ENV=development +LOG_LEVEL=info diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..80dce5c --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +# Environment variables +.env +.env.local + +# IDE +.vscode/ +.idea/ +*.swp +*.swo + +# OS +.DS_Store +Thumbs.db + +# Logs +*.log + +# Temporary files +*.tmp +tmp/ diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md new file mode 100644 index 0000000..6e7858b --- /dev/null +++ b/IMPLEMENTATION_PLAN.md @@ -0,0 +1,1159 @@ +# VictoriaLogs Manager - Implementation Plan + +## Project Overview + +A comprehensive web application for managing VictoriaLogs with alerting, advanced reporting, and malicious pattern detection. + +## Milestones & Phases + +### Milestone 1: Foundation & Authentication (Week 1) +**Goal**: Complete project setup and implement authentication system + +**Duration**: 5-7 days + +**Deliverables**: +- Fully configured development environment +- Working JWT authentication +- User management system +- Role-based access control + +--- + +### Milestone 2: VictoriaLogs Integration (Week 2) +**Goal**: Implement log querying and exploration features + +**Duration**: 5-7 days + +**Deliverables**: +- VictoriaLogs HTTP client +- Log query API endpoints +- Logs Explorer UI +- Live tail functionality +- Export features + +--- + +### Milestone 3: Alert Management (Week 3) +**Goal**: Complete vmalert rule management system + +**Duration**: 5-7 days + +**Deliverables**: +- Alert rule CRUD operations +- YAML file generation +- vmalert integration +- Alert Manager UI +- Active alerts viewer + +--- + +### Milestone 4: Pattern Detection (Week 4) +**Goal**: Implement regex and ML-based pattern detection + +**Duration**: 7-10 days + +**Deliverables**: +- Regex pattern engine +- Pre-built pattern library +- ML service implementation +- Pattern Detection UI +- ML training and detection endpoints + +--- + +### Milestone 5: Reports & Dashboards (Week 5) +**Goal**: Build visualization and reporting system + +**Duration**: 5-7 days + +**Deliverables**: +- Report configuration system +- Dashboard page with widgets +- ECharts integration +- Report builder UI +- Multiple chart types + +--- + +### Milestone 6: Polish & Production (Week 6) +**Goal**: Production readiness and deployment + +**Duration**: 5-7 days + +**Deliverables**: +- Security hardening +- Performance optimization +- Documentation +- Testing +- Deployment guides + +--- + +## Detailed Task Breakdown + +### Milestone 1: Foundation & Authentication + +#### Issue #1: Setup React Frontend +**Priority**: High +**Estimated Time**: 2-3 hours + +**Tasks**: +- Initialize Vite + React + TypeScript project +- Install dependencies (React Router, Zustand, Axios, Tailwind CSS) +- Configure Tailwind CSS +- Create basic folder structure +- Setup ESLint and Prettier + +**Files to Create**: +- `frontend/package.json` +- `frontend/vite.config.ts` +- `frontend/tailwind.config.js` +- `frontend/src/main.tsx` +- `frontend/src/App.tsx` + +**Acceptance Criteria**: +- [ ] Frontend dev server runs successfully +- [ ] Tailwind CSS is working +- [ ] Hot reload is functional + +--- + +#### Issue #2: Implement JWT Utilities +**Priority**: High +**Estimated Time**: 2 hours + +**Tasks**: +- Create JWT token generation function +- Create JWT token validation function +- Implement claims structure +- Add token expiration handling + +**Files to Create**: +- `backend/internal/utils/jwt.go` + +**Acceptance Criteria**: +- [ ] Can generate JWT tokens with user claims +- [ ] Can validate and parse JWT tokens +- [ ] Token expiration is enforced + +--- + +#### Issue #3: Implement Authentication Handlers +**Priority**: High +**Estimated Time**: 3-4 hours + +**Tasks**: +- Create login handler (username/password) +- Create refresh token handler +- Create "get current user" handler +- Implement password hashing with bcrypt +- Add input validation + +**Files to Create**: +- `backend/internal/api/handlers/auth.go` +- `backend/internal/services/auth_service.go` + +**Acceptance Criteria**: +- [ ] POST /api/v1/auth/login returns JWT token +- [ ] POST /api/v1/auth/refresh refreshes token +- [ ] GET /api/v1/auth/me returns current user +- [ ] Passwords are securely hashed + +--- + +#### Issue #4: Implement Authentication Middleware +**Priority**: High +**Estimated Time**: 2 hours + +**Tasks**: +- Create JWT validation middleware +- Create permission check middleware +- Create role check middleware +- Add error handling + +**Files to Create**: +- `backend/internal/api/middleware/auth.go` + +**Acceptance Criteria**: +- [ ] Middleware validates JWT tokens +- [ ] Invalid tokens return 401 +- [ ] Permission checks work correctly +- [ ] Role checks work correctly + +--- + +#### Issue #5: Create User Repository +**Priority**: High +**Estimated Time**: 2-3 hours + +**Tasks**: +- Implement user CRUD operations +- Create database queries +- Add error handling +- Implement user lookup functions + +**Files to Create**: +- `backend/internal/repository/user_repo.go` + +**Acceptance Criteria**: +- [ ] Can create new users +- [ ] Can retrieve users by ID/username/email +- [ ] Can update user information +- [ ] Can delete users +- [ ] Can list all users + +--- + +#### Issue #6: Implement User Management API +**Priority**: High +**Estimated Time**: 3 hours + +**Tasks**: +- Create user CRUD endpoints +- Add role-based access control +- Implement user service layer +- Add input validation + +**Files to Create**: +- `backend/internal/api/handlers/users.go` +- `backend/internal/services/user_service.go` + +**Acceptance Criteria**: +- [ ] GET /api/v1/users returns user list (admin only) +- [ ] POST /api/v1/users creates user (admin only) +- [ ] PUT /api/v1/users/:id updates user (admin only) +- [ ] DELETE /api/v1/users/:id deletes user (admin only) + +--- + +#### Issue #7: Create Auth Store (Frontend) +**Priority**: High +**Estimated Time**: 2 hours + +**Tasks**: +- Setup Zustand store for authentication +- Implement login/logout functions +- Add token persistence (localStorage) +- Create auth hooks + +**Files to Create**: +- `frontend/src/store/authStore.ts` +- `frontend/src/hooks/useAuth.ts` + +**Acceptance Criteria**: +- [ ] Auth state persists across page reloads +- [ ] Login updates auth state +- [ ] Logout clears auth state + +--- + +#### Issue #8: Create API Client (Frontend) +**Priority**: High +**Estimated Time**: 2 hours + +**Tasks**: +- Setup Axios instance +- Add request interceptor (JWT token) +- Add response interceptor (401 handling) +- Create API client functions + +**Files to Create**: +- `frontend/src/api/client.ts` +- `frontend/src/api/auth.ts` + +**Acceptance Criteria**: +- [ ] All requests include JWT token +- [ ] 401 responses trigger logout +- [ ] API client is properly typed + +--- + +#### Issue #9: Create Login Page +**Priority**: High +**Estimated Time**: 3 hours + +**Tasks**: +- Create login form UI +- Add form validation +- Implement login submission +- Add error handling +- Add loading states + +**Files to Create**: +- `frontend/src/pages/Login.tsx` + +**Acceptance Criteria**: +- [ ] Login form validates inputs +- [ ] Successful login redirects to dashboard +- [ ] Error messages display correctly +- [ ] Loading state shows during request + +--- + +#### Issue #10: Create Protected Route Component +**Priority**: High +**Estimated Time**: 1 hour + +**Tasks**: +- Create ProtectedRoute component +- Add authentication check +- Add role-based access +- Implement redirects + +**Files to Create**: +- `frontend/src/components/common/ProtectedRoute.tsx` + +**Acceptance Criteria**: +- [ ] Unauthenticated users redirect to login +- [ ] Unauthorized users see error message +- [ ] Authorized users see content + +--- + +### Milestone 2: VictoriaLogs Integration + +#### Issue #11: Create VictoriaLogs HTTP Client +**Priority**: High +**Estimated Time**: 4 hours + +**Tasks**: +- Create VictoriaLogs HTTP client +- Implement query method +- Implement tail method +- Implement facets method +- Implement stats query method +- Add error handling + +**Files to Create**: +- `backend/pkg/vlogs/client.go` +- `backend/pkg/vlogs/models.go` + +**Acceptance Criteria**: +- [ ] Can query VictoriaLogs +- [ ] Can tail logs in real-time +- [ ] Can fetch facets +- [ ] Can execute stats queries + +--- + +#### Issue #12: Create Log Query Service +**Priority**: High +**Estimated Time**: 3 hours + +**Tasks**: +- Implement log query proxy +- Add authentication checks +- Implement rate limiting +- Add query validation + +**Files to Create**: +- `backend/internal/services/vlogs_service.go` + +**Acceptance Criteria**: +- [ ] Service proxies requests to VictoriaLogs +- [ ] User permissions are enforced +- [ ] Rate limiting works per role + +--- + +#### Issue #13: Implement Log Query API Endpoints +**Priority**: High +**Estimated Time**: 3 hours + +**Tasks**: +- Create POST /logs/query endpoint +- Create POST /logs/tail endpoint +- Create GET /logs/facets endpoint +- Create POST /logs/stats endpoint +- Create POST /logs/export endpoint + +**Files to Create**: +- `backend/internal/api/handlers/logs.go` + +**Acceptance Criteria**: +- [ ] All endpoints return correct data +- [ ] Authentication is required +- [ ] Error handling is robust + +--- + +#### Issue #14: Create Logs API Client (Frontend) +**Priority**: High +**Estimated Time**: 2 hours + +**Tasks**: +- Create logs API functions +- Add TypeScript types +- Implement query builder helper + +**Files to Create**: +- `frontend/src/api/logs.ts` +- `frontend/src/types/logs.ts` + +**Acceptance Criteria**: +- [ ] Can query logs from frontend +- [ ] All functions are properly typed + +--- + +#### Issue #15: Create Logs Store (Frontend) +**Priority**: Medium +**Estimated Time**: 2 hours + +**Tasks**: +- Create Zustand store for logs +- Implement query state management +- Add pagination state +- Add filter state + +**Files to Create**: +- `frontend/src/store/logsStore.ts` + +**Acceptance Criteria**: +- [ ] Store manages query state +- [ ] Pagination works correctly +- [ ] Filters are applied properly + +--- + +#### Issue #16: Create Query Builder Component +**Priority**: High +**Estimated Time**: 4-5 hours + +**Tasks**: +- Create visual query builder UI +- Add field selector +- Add operator selector +- Add value input +- Implement query preview +- Add raw LogsQL editor toggle + +**Files to Create**: +- `frontend/src/components/logs/QueryBuilder.tsx` + +**Acceptance Criteria**: +- [ ] Can build queries visually +- [ ] Can switch to raw LogsQL +- [ ] Query preview is accurate + +--- + +#### Issue #17: Create Log Table Component +**Priority**: High +**Estimated Time**: 4 hours + +**Tasks**: +- Create virtualized table (react-window) +- Implement expandable rows +- Add column customization +- Add sorting +- Add loading states + +**Files to Create**: +- `frontend/src/components/logs/LogTable.tsx` +- `frontend/src/components/logs/LogDetails.tsx` + +**Acceptance Criteria**: +- [ ] Table performs well with 10,000+ rows +- [ ] Rows expand to show details +- [ ] Columns are customizable + +--- + +#### Issue #18: Create Logs Explorer Page +**Priority**: High +**Estimated Time**: 4 hours + +**Tasks**: +- Create main Logs Explorer page +- Integrate QueryBuilder +- Integrate LogTable +- Add time range selector +- Add facets sidebar +- Add export button + +**Files to Create**: +- `frontend/src/pages/LogsExplorer.tsx` +- `frontend/src/components/logs/TimeRangeSelector.tsx` +- `frontend/src/components/logs/FacetsSidebar.tsx` + +**Acceptance Criteria**: +- [ ] Can query and view logs +- [ ] Time range selector works +- [ ] Facets are clickable filters + +--- + +#### Issue #19: Implement Live Tail (WebSocket) +**Priority**: Medium +**Estimated Time**: 4 hours + +**Tasks**: +- Implement WebSocket endpoint (backend) +- Create live tail component (frontend) +- Add auto-scroll +- Add pause/resume +- Add disconnect handling + +**Files to Create**: +- `backend/internal/api/handlers/websocket.go` +- `frontend/src/components/logs/LiveTail.tsx` +- `frontend/src/hooks/useWebSocket.ts` + +**Acceptance Criteria**: +- [ ] Logs stream in real-time +- [ ] Can pause/resume streaming +- [ ] Handles disconnections gracefully + +--- + +#### Issue #20: Implement Log Export +**Priority**: Low +**Estimated Time**: 2 hours + +**Tasks**: +- Add CSV export +- Add JSON export +- Add download handling + +**Files to Modify**: +- `backend/internal/api/handlers/logs.go` +- `frontend/src/components/logs/ExportButton.tsx` + +**Acceptance Criteria**: +- [ ] Can export logs as CSV +- [ ] Can export logs as JSON +- [ ] Large exports stream properly + +--- + +### Milestone 3: Alert Management + +#### Issue #21: Create Alert Rule Models +**Priority**: High +**Estimated Time**: 1 hour + +**Tasks**: +- Define alert rule struct +- Add JSON/YAML tags +- Create validation functions + +**Files to Create**: +- `backend/internal/models/alert.go` + +**Acceptance Criteria**: +- [ ] Alert rule model matches schema +- [ ] Validation functions work + +--- + +#### Issue #22: Create Alert Repository +**Priority**: High +**Estimated Time**: 3 hours + +**Tasks**: +- Implement alert rule CRUD +- Add database queries +- Implement rule search/filter + +**Files to Create**: +- `backend/internal/repository/alert_repo.go` + +**Acceptance Criteria**: +- [ ] Can create/read/update/delete alert rules +- [ ] Can filter rules by group/status + +--- + +#### Issue #23: Implement vmalert Service +**Priority**: High +**Estimated Time**: 5 hours + +**Tasks**: +- Create vmalert service +- Implement YAML file generation +- Implement vmalert reload trigger +- Add active alerts fetching +- Add validation + +**Files to Create**: +- `backend/internal/services/vmalert_service.go` + +**Acceptance Criteria**: +- [ ] Can generate valid YAML files +- [ ] vmalert reloads successfully +- [ ] Can fetch active alerts from vmalert + +--- + +#### Issue #24: Implement Alert API Endpoints +**Priority**: High +**Estimated Time**: 3 hours + +**Tasks**: +- Create alert rule CRUD endpoints +- Add reload endpoint +- Add active alerts endpoint +- Add permissions checks + +**Files to Create**: +- `backend/internal/api/handlers/alerts.go` + +**Acceptance Criteria**: +- [ ] All CRUD operations work +- [ ] Reload triggers vmalert reload +- [ ] Active alerts are fetched + +--- + +#### Issue #25: Create Alert API Client (Frontend) +**Priority**: High +**Estimated Time**: 2 hours + +**Tasks**: +- Create alerts API functions +- Add TypeScript types + +**Files to Create**: +- `frontend/src/api/alerts.ts` +- `frontend/src/types/alerts.ts` + +**Acceptance Criteria**: +- [ ] Can manage alerts from frontend +- [ ] All functions are typed + +--- + +#### Issue #26: Create Alert Rule Form Component +**Priority**: High +**Estimated Time**: 4 hours + +**Tasks**: +- Create alert rule form +- Add field validation +- Add LogsQL query builder +- Add labels/annotations editor + +**Files to Create**: +- `frontend/src/components/alerts/AlertRuleForm.tsx` + +**Acceptance Criteria**: +- [ ] Form validates all fields +- [ ] Can create/edit rules +- [ ] LogsQL syntax is validated + +--- + +#### Issue #27: Create Alert Rule YAML Editor +**Priority**: Medium +**Estimated Time**: 3 hours + +**Tasks**: +- Integrate Monaco Editor +- Add YAML syntax highlighting +- Add YAML validation +- Add save functionality + +**Files to Create**: +- `frontend/src/components/alerts/AlertRuleEditor.tsx` + +**Acceptance Criteria**: +- [ ] YAML editor has syntax highlighting +- [ ] Invalid YAML shows errors +- [ ] Can save YAML directly + +--- + +#### Issue #28: Create Alerts Manager Page +**Priority**: High +**Estimated Time**: 4 hours + +**Tasks**: +- Create alerts list view +- Add create/edit/delete actions +- Add active alerts section +- Add alert history + +**Files to Create**: +- `frontend/src/pages/AlertsManager.tsx` +- `frontend/src/components/alerts/AlertRuleList.tsx` +- `frontend/src/components/alerts/AlertStatus.tsx` + +**Acceptance Criteria**: +- [ ] Can view all alert rules +- [ ] Can create/edit/delete rules +- [ ] Active alerts are displayed + +--- + +### Milestone 4: Pattern Detection + +#### Issue #29: Create Pattern Models +**Priority**: High +**Estimated Time**: 1 hour + +**Tasks**: +- Define pattern struct +- Add validation +- Create pre-built patterns + +**Files to Create**: +- `backend/internal/models/pattern.go` + +**Acceptance Criteria**: +- [ ] Pattern model is complete +- [ ] Pre-built patterns are defined + +--- + +#### Issue #30: Create Pattern Repository +**Priority**: High +**Estimated Time**: 2 hours + +**Tasks**: +- Implement pattern CRUD +- Add pattern search + +**Files to Create**: +- `backend/internal/repository/pattern_repo.go` + +**Acceptance Criteria**: +- [ ] Can manage patterns in database + +--- + +#### Issue #31: Implement Regex Pattern Detection +**Priority**: High +**Estimated Time**: 4 hours + +**Tasks**: +- Create pattern detection engine +- Implement regex matching +- Generate LogsQL queries +- Execute detection + +**Files to Create**: +- `backend/internal/services/pattern_service.go` + +**Acceptance Criteria**: +- [ ] Can detect patterns using regex +- [ ] Returns matched logs + +--- + +#### Issue #32: Implement ML Feature Extraction +**Priority**: High +**Estimated Time**: 4 hours + +**Tasks**: +- Create feature extractor +- Extract temporal features +- Extract log content features +- Extract numerical features + +**Files to Create**: +- `ml-service/app/utils/feature_extraction.py` + +**Acceptance Criteria**: +- [ ] Extracts relevant features from logs +- [ ] Features are properly normalized + +--- + +#### Issue #33: Implement ML Anomaly Detector +**Priority**: High +**Estimated Time**: 5 hours + +**Tasks**: +- Implement Isolation Forest model +- Add training functionality +- Add prediction functionality +- Add model persistence + +**Files to Create**: +- `ml-service/app/models/anomaly_detector.py` + +**Acceptance Criteria**: +- [ ] Model trains successfully +- [ ] Model detects anomalies +- [ ] Model can be saved/loaded + +--- + +#### Issue #34: Implement ML Service Endpoints +**Priority**: High +**Estimated Time**: 3 hours + +**Tasks**: +- Complete /api/ml/train endpoint +- Complete /api/ml/detect endpoint +- Add VictoriaLogs integration + +**Files to Modify**: +- `ml-service/app/main.py` +- `ml-service/app/api/endpoints.py` + +**Acceptance Criteria**: +- [ ] Training endpoint works +- [ ] Detection endpoint works +- [ ] Fetches logs from VictoriaLogs + +--- + +#### Issue #35: Create ML Service Client (Backend) +**Priority**: High +**Estimated Time**: 2 hours + +**Tasks**: +- Create ML service HTTP client +- Implement train/detect methods + +**Files to Create**: +- `backend/internal/services/ml_service.go` + +**Acceptance Criteria**: +- [ ] Can communicate with ML service +- [ ] Handles timeouts properly + +--- + +#### Issue #36: Create Pattern Detection Page +**Priority**: High +**Estimated Time**: 5 hours + +**Tasks**: +- Create pattern library UI +- Add regex pattern form +- Add ML detection trigger +- Display detection results + +**Files to Create**: +- `frontend/src/pages/PatternDetection.tsx` +- `frontend/src/components/patterns/PatternLibrary.tsx` +- `frontend/src/components/patterns/RegexPatternForm.tsx` +- `frontend/src/components/patterns/MLAnomalyViewer.tsx` + +**Acceptance Criteria**: +- [ ] Can browse pattern library +- [ ] Can create custom patterns +- [ ] Can trigger ML detection +- [ ] Results are displayed clearly + +--- + +### Milestone 5: Reports & Dashboards + +#### Issue #37: Create Report Models +**Priority**: High +**Estimated Time**: 1 hour + +**Tasks**: +- Define report struct +- Define widget configs + +**Files to Create**: +- `backend/internal/models/report.go` + +**Acceptance Criteria**: +- [ ] Report model is complete + +--- + +#### Issue #38: Create Report Repository +**Priority**: High +**Estimated Time**: 2 hours + +**Tasks**: +- Implement report CRUD + +**Files to Create**: +- `backend/internal/repository/report_repo.go` + +**Acceptance Criteria**: +- [ ] Can manage reports + +--- + +#### Issue #39: Implement Report Execution Engine +**Priority**: High +**Estimated Time**: 4 hours + +**Tasks**: +- Create report execution service +- Execute queries for widgets +- Aggregate data + +**Files to Create**: +- `backend/internal/services/report_service.go` + +**Acceptance Criteria**: +- [ ] Can execute report queries +- [ ] Returns formatted data for charts + +--- + +#### Issue #40: Integrate ECharts +**Priority**: High +**Estimated Time**: 2 hours + +**Tasks**: +- Install echarts-for-react +- Create chart wrapper components + +**Files to Create**: +- `frontend/src/components/charts/BaseChart.tsx` + +**Acceptance Criteria**: +- [ ] ECharts renders correctly +- [ ] Charts are responsive + +--- + +#### Issue #41: Create Chart Components +**Priority**: High +**Estimated Time**: 4 hours + +**Tasks**: +- Create TimeSeriesChart +- Create PieChart +- Create BarChart +- Create HeatmapChart + +**Files to Create**: +- `frontend/src/components/charts/TimeSeriesChart.tsx` +- `frontend/src/components/charts/PieChart.tsx` +- `frontend/src/components/charts/BarChart.tsx` +- `frontend/src/components/charts/HeatmapChart.tsx` + +**Acceptance Criteria**: +- [ ] All chart types render correctly +- [ ] Charts are interactive + +--- + +#### Issue #42: Create Dashboard Page +**Priority**: High +**Estimated Time**: 5 hours + +**Tasks**: +- Create dashboard layout +- Add summary cards +- Add multiple chart widgets +- Make layout responsive + +**Files to Create**: +- `frontend/src/pages/Dashboard.tsx` +- `frontend/src/components/dashboard/SummaryCard.tsx` +- `frontend/src/components/dashboard/DashboardGrid.tsx` + +**Acceptance Criteria**: +- [ ] Dashboard displays key metrics +- [ ] Multiple widgets are shown +- [ ] Layout is responsive + +--- + +#### Issue #43: Create Report Builder UI +**Priority**: Medium +**Estimated Time**: 5 hours + +**Tasks**: +- Create report builder interface +- Add widget configuration +- Add query builder for widgets +- Add save/load functionality + +**Files to Create**: +- `frontend/src/components/reports/ReportBuilder.tsx` +- `frontend/src/components/reports/WidgetConfig.tsx` + +**Acceptance Criteria**: +- [ ] Can create custom reports +- [ ] Can configure widgets +- [ ] Reports are saved + +--- + +### Milestone 6: Polish & Production + +#### Issue #44: Implement Rate Limiting +**Priority**: High +**Estimated Time**: 3 hours + +**Tasks**: +- Add rate limiting middleware +- Configure limits per role +- Add rate limit headers + +**Files to Create**: +- `backend/internal/api/middleware/ratelimit.go` + +**Acceptance Criteria**: +- [ ] Rate limiting works per role +- [ ] Returns 429 when exceeded + +--- + +#### Issue #45: Add Input Validation +**Priority**: High +**Estimated Time**: 3 hours + +**Tasks**: +- Add query validation +- Add pattern validation +- Add length limits +- Add SQL injection prevention + +**Files to Create**: +- `backend/internal/utils/validator.go` + +**Acceptance Criteria**: +- [ ] Dangerous inputs are rejected +- [ ] Validation messages are clear + +--- + +#### Issue #46: Add Audit Logging +**Priority**: High +**Estimated Time**: 3 hours + +**Tasks**: +- Create audit logging middleware +- Log sensitive operations +- Add audit log viewer (admin) + +**Files to Create**: +- `backend/internal/api/middleware/audit.go` +- `backend/internal/services/audit_service.go` + +**Acceptance Criteria**: +- [ ] All sensitive operations are logged +- [ ] Admins can view audit logs + +--- + +#### Issue #47: Add Loading States and Error Handling +**Priority**: Medium +**Estimated Time**: 4 hours + +**Tasks**: +- Add loading spinners +- Add error toast notifications +- Improve error messages +- Add retry logic + +**Files to Create**: +- `frontend/src/components/common/LoadingSpinner.tsx` +- `frontend/src/components/common/Toast.tsx` +- `frontend/src/hooks/useToast.ts` + +**Acceptance Criteria**: +- [ ] Loading states show during requests +- [ ] Errors display user-friendly messages + +--- + +#### Issue #48: Optimize Performance +**Priority**: Medium +**Estimated Time**: 4 hours + +**Tasks**: +- Add Redis caching (optional) +- Optimize database queries +- Add frontend code splitting +- Optimize bundle size + +**Acceptance Criteria**: +- [ ] Page load time < 2s +- [ ] Bundle size < 500KB + +--- + +#### Issue #49: Write API Documentation +**Priority**: Medium +**Estimated Time**: 4 hours + +**Tasks**: +- Document all API endpoints +- Add request/response examples +- Create Postman collection + +**Files to Create**: +- `docs/api.md` +- `docs/postman_collection.json` + +**Acceptance Criteria**: +- [ ] All endpoints are documented +- [ ] Examples are provided + +--- + +#### Issue #50: Write User Guide +**Priority**: Low +**Estimated Time**: 3 hours + +**Tasks**: +- Write getting started guide +- Document features +- Add screenshots + +**Files to Create**: +- `docs/user-guide.md` +- `docs/screenshots/` + +**Acceptance Criteria**: +- [ ] User guide is complete +- [ ] Screenshots are included + +--- + +#### Issue #51: Write Deployment Guide +**Priority**: Medium +**Estimated Time**: 3 hours + +**Tasks**: +- Document Docker deployment +- Document production configuration +- Add troubleshooting section + +**Files to Create**: +- `docs/deployment.md` + +**Acceptance Criteria**: +- [ ] Deployment guide is complete +- [ ] Production checklist is provided + +--- + +#### Issue #52: Add Unit Tests +**Priority**: Medium +**Estimated Time**: 6 hours + +**Tasks**: +- Write backend unit tests (critical paths) +- Write frontend unit tests (critical components) +- Add test CI pipeline + +**Files to Create**: +- `backend/*_test.go` +- `frontend/src/**/*.test.tsx` +- `.github/workflows/test.yml` or `.gitea/workflows/test.yml` + +**Acceptance Criteria**: +- [ ] Critical paths have tests +- [ ] Tests pass in CI + +--- + +## Summary + +**Total Estimated Time**: 160-200 hours (4-5 weeks) + +**Total Issues**: 52 + +**Total Milestones**: 6 + +**Priority Breakdown**: +- High Priority: 38 issues +- Medium Priority: 11 issues +- Low Priority: 3 issues diff --git a/README.md b/README.md new file mode 100644 index 0000000..b9377e9 --- /dev/null +++ b/README.md @@ -0,0 +1,93 @@ +# VictoriaLogs Manager + +A comprehensive web application for managing VictoriaLogs with alerting, advanced reporting, and malicious pattern detection. + +## Features + +- **Log Exploration**: Query and search logs using LogsQL with visual query builder +- **Alert Management**: Create and manage vmalert rules with YAML editor +- **Pattern Detection**: Find malicious patterns using regex and ML-based anomaly detection +- **Reports & Dashboards**: Fancy visualizations and custom reports +- **Role-Based Access**: Admin, Editor, Analyst, and Viewer roles + +## Tech Stack + +- **Backend**: Go (Gin framework) +- **Frontend**: React + TypeScript + Vite +- **Database**: PostgreSQL +- **ML Service**: Python (FastAPI + scikit-learn) +- **Deployment**: Docker Compose + +## Project Structure + +``` +victorialogs-manager/ +├── backend/ # Go API server +├── frontend/ # React SPA +├── ml-service/ # Python ML service +└── docker-compose.yml +``` + +## Getting Started + +### Prerequisites + +- Go 1.21+ +- Node.js 18+ +- Python 3.11+ +- Docker & Docker Compose + +### Quick Start with Docker + +```bash +# Copy environment file +cp .env.example .env + +# Edit .env with your configuration +nano .env + +# Start all services +docker-compose up -d +``` + +The application will be available at: +- Frontend: http://localhost:3000 +- Backend API: http://localhost:8080 +- VictoriaLogs: http://localhost:9428 + +### Development Setup + +#### Backend +```bash +cd backend +go mod download +go run cmd/server/main.go +``` + +#### Frontend +```bash +cd frontend +npm install +npm run dev +``` + +#### ML Service +```bash +cd ml-service +pip install -r requirements.txt +uvicorn app.main:app --reload +``` + +## Configuration + +See `.env.example` for all available configuration options. + +## Documentation + +- [API Documentation](./docs/api.md) (coming soon) +- [User Guide](./docs/user-guide.md) (coming soon) +- [Deployment Guide](./docs/deployment.md) (coming soon) + +## License + +MIT diff --git a/alertmanager.yml b/alertmanager.yml new file mode 100644 index 0000000..8f13832 --- /dev/null +++ b/alertmanager.yml @@ -0,0 +1,25 @@ +global: + resolve_timeout: 5m + +route: + group_by: ['alertname', 'cluster', 'service'] + group_wait: 10s + group_interval: 10s + repeat_interval: 12h + receiver: 'default' + +receivers: + - name: 'default' + # Configure your notification channels here + # Examples: email, slack, webhook, etc. + # For now, this is a basic configuration + webhook_configs: + - url: 'http://backend:8080/api/v1/alerts/webhook' + send_resolved: true + +inhibit_rules: + - source_match: + severity: 'critical' + target_match: + severity: 'warning' + equal: ['alertname', 'cluster', 'service'] diff --git a/backend/.gitignore b/backend/.gitignore new file mode 100644 index 0000000..0eeae58 --- /dev/null +++ b/backend/.gitignore @@ -0,0 +1,22 @@ +# Go +*.exe +*.exe~ +*.dll +*.so +*.dylib +*.test +*.out +go.work + +# Binary +/bin/ +/build/ +server + +# Vendor +vendor/ + +# vmalert rules (generated) +configs/vmalert-rules/*.yml +configs/vmalert-rules/*.yaml +!configs/vmalert-rules/.gitkeep diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 0000000..6f9e6f7 --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,40 @@ +# Build stage +FROM golang:1.21-alpine AS builder + +# Install build dependencies +RUN apk add --no-cache git + +# Set working directory +WORKDIR /app + +# Copy go mod files +COPY go.mod go.sum* ./ + +# Download dependencies +RUN go mod download + +# Copy source code +COPY . . + +# Build the application +RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o server ./cmd/server + +# Final stage +FROM alpine:latest + +# Install ca-certificates for HTTPS +RUN apk --no-cache add ca-certificates + +WORKDIR /root/ + +# Copy binary from builder +COPY --from=builder /app/server . + +# Copy configuration +COPY --from=builder /app/configs ./configs + +# Expose port +EXPOSE 8080 + +# Run the application +CMD ["./server"] diff --git a/backend/configs/config.yaml b/backend/configs/config.yaml new file mode 100644 index 0000000..cf76cb0 --- /dev/null +++ b/backend/configs/config.yaml @@ -0,0 +1,39 @@ +server: + port: 8080 + read_timeout: 30s + write_timeout: 30s + +database: + host: ${DB_HOST:postgres} + port: ${DB_PORT:5432} + name: ${DB_NAME:vlogs_manager} + user: ${DB_USER:vlogs} + password: ${DB_PASSWORD} + max_connections: 25 + +victorialogs: + url: ${VLOGS_URL:http://victorialogs:9428} + timeout: 30s + max_query_size: 10000 + +vmalert: + url: ${VMALERT_URL:http://vmalert:8880} + rules_dir: ${VMALERT_RULES_DIR:/app/rules} + reload_endpoint: "/-/reload" + +ml_service: + url: ${ML_SERVICE_URL:http://ml-service:8000} + timeout: 60s + +auth: + jwt_secret: ${JWT_SECRET} + access_token_duration: 15m + refresh_token_duration: 168h + +rate_limiting: + enabled: true + requests_per_minute: + admin: 1000 + editor: 500 + analyst: 300 + viewer: 200 diff --git a/backend/configs/vmalert-rules/.gitkeep b/backend/configs/vmalert-rules/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/backend/go.mod b/backend/go.mod new file mode 100644 index 0000000..12a0dc2 --- /dev/null +++ b/backend/go.mod @@ -0,0 +1,12 @@ +module github.com/yourusername/victorialogs-manager + +go 1.21 + +require ( + github.com/gin-gonic/gin v1.9.1 + github.com/golang-jwt/jwt/v5 v5.2.0 + github.com/lib/pq v1.10.9 + golang.org/x/crypto v0.18.0 + github.com/joho/godotenv v1.5.1 + gopkg.in/yaml.v3 v3.0.1 +) diff --git a/backend/internal/api/router.go b/backend/internal/api/router.go new file mode 100644 index 0000000..52202ad --- /dev/null +++ b/backend/internal/api/router.go @@ -0,0 +1,186 @@ +package api + +import ( + "database/sql" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/yourusername/victorialogs-manager/internal/config" +) + +// NewRouter creates and configures the main router +func NewRouter(cfg *config.Config, db *sql.DB) *gin.Engine { + // Set Gin mode + gin.SetMode(gin.ReleaseMode) + + router := gin.Default() + + // CORS middleware + router.Use(corsMiddleware()) + + // Health check endpoint + router.GET("/health", func(c *gin.Context) { + c.JSON(http.StatusOK, gin.H{ + "status": "ok", + "service": "victorialogs-manager", + }) + }) + + // API v1 routes + v1 := router.Group("/api/v1") + { + // Authentication routes (public) + auth := v1.Group("/auth") + { + auth.POST("/login", func(c *gin.Context) { + // TODO: Implement login handler + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + auth.POST("/refresh", func(c *gin.Context) { + // TODO: Implement refresh handler + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + auth.GET("/me", func(c *gin.Context) { + // TODO: Implement me handler (requires auth) + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + } + + // Protected routes (require authentication) + // TODO: Add auth middleware here + + // User management routes + users := v1.Group("/users") + { + users.GET("", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + users.POST("", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + users.GET("/:id", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + users.PUT("/:id", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + users.DELETE("/:id", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + } + + // Log querying routes + logs := v1.Group("/logs") + { + logs.POST("/query", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + logs.POST("/tail", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + logs.GET("/facets", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + logs.POST("/stats", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + logs.POST("/export", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + } + + // Alert management routes + alerts := v1.Group("/alerts") + { + alerts.GET("/rules", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + alerts.POST("/rules", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + alerts.GET("/rules/:id", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + alerts.PUT("/rules/:id", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + alerts.DELETE("/rules/:id", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + alerts.POST("/reload", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + alerts.GET("/active", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + } + + // Pattern detection routes + patterns := v1.Group("/patterns") + { + patterns.GET("", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + patterns.POST("", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + patterns.PUT("/:id", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + patterns.DELETE("/:id", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + patterns.POST("/detect", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + patterns.POST("/ml/train", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + patterns.POST("/ml/detect", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + } + + // Report routes + reports := v1.Group("/reports") + { + reports.GET("", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + reports.POST("", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + reports.GET("/:id", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + reports.PUT("/:id", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + reports.DELETE("/:id", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + reports.POST("/:id/execute", func(c *gin.Context) { + c.JSON(http.StatusNotImplemented, gin.H{"message": "not implemented yet"}) + }) + } + } + + return router +} + +// corsMiddleware handles CORS headers +func corsMiddleware() gin.HandlerFunc { + return func(c *gin.Context) { + c.Writer.Header().Set("Access-Control-Allow-Origin", "*") + c.Writer.Header().Set("Access-Control-Allow-Credentials", "true") + c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With") + c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT, DELETE") + + if c.Request.Method == "OPTIONS" { + c.AbortWithStatus(204) + return + } + + c.Next() + } +} diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go new file mode 100644 index 0000000..6221803 --- /dev/null +++ b/backend/internal/config/config.go @@ -0,0 +1,106 @@ +package config + +import ( + "fmt" + "os" + "time" + + "gopkg.in/yaml.v3" +) + +type Config struct { + Server ServerConfig `yaml:"server"` + Database DatabaseConfig `yaml:"database"` + VictoriaLogs VictoriaLogsConfig `yaml:"victorialogs"` + VMAlert VMAlertConfig `yaml:"vmalert"` + MLService MLServiceConfig `yaml:"ml_service"` + Auth AuthConfig `yaml:"auth"` + RateLimiting RateLimitingConfig `yaml:"rate_limiting"` +} + +type ServerConfig struct { + Port int `yaml:"port"` + ReadTimeout time.Duration `yaml:"read_timeout"` + WriteTimeout time.Duration `yaml:"write_timeout"` +} + +type DatabaseConfig struct { + Host string `yaml:"host"` + Port int `yaml:"port"` + Name string `yaml:"name"` + User string `yaml:"user"` + Password string `yaml:"password"` + MaxConnections int `yaml:"max_connections"` +} + +type VictoriaLogsConfig struct { + URL string `yaml:"url"` + Timeout time.Duration `yaml:"timeout"` + MaxQuerySize int `yaml:"max_query_size"` +} + +type VMAlertConfig struct { + URL string `yaml:"url"` + RulesDir string `yaml:"rules_dir"` + ReloadEndpoint string `yaml:"reload_endpoint"` +} + +type MLServiceConfig struct { + URL string `yaml:"url"` + Timeout time.Duration `yaml:"timeout"` +} + +type AuthConfig struct { + JWTSecret string `yaml:"jwt_secret"` + AccessTokenDuration time.Duration `yaml:"access_token_duration"` + RefreshTokenDuration time.Duration `yaml:"refresh_token_duration"` +} + +type RateLimitingConfig struct { + Enabled bool `yaml:"enabled"` + RequestsPerMinute map[string]int `yaml:"requests_per_minute"` +} + +// Load loads configuration from YAML file and environment variables +func Load(configPath string) (*Config, error) { + data, err := os.ReadFile(configPath) + if err != nil { + return nil, fmt.Errorf("failed to read config file: %w", err) + } + + // Expand environment variables in config + expandedData := []byte(os.ExpandEnv(string(data))) + + var cfg Config + if err := yaml.Unmarshal(expandedData, &cfg); err != nil { + return nil, fmt.Errorf("failed to parse config: %w", err) + } + + // Validate required fields + if err := cfg.validate(); err != nil { + return nil, fmt.Errorf("config validation failed: %w", err) + } + + return &cfg, nil +} + +func (c *Config) validate() error { + if c.Database.Password == "" { + return fmt.Errorf("database password is required") + } + if c.Auth.JWTSecret == "" { + return fmt.Errorf("JWT secret is required") + } + if len(c.Auth.JWTSecret) < 32 { + return fmt.Errorf("JWT secret must be at least 32 characters") + } + return nil +} + +// GetDSN returns the PostgreSQL connection string +func (c *DatabaseConfig) GetDSN() string { + return fmt.Sprintf( + "host=%s port=%d user=%s password=%s dbname=%s sslmode=disable", + c.Host, c.Port, c.User, c.Password, c.Name, + ) +} diff --git a/backend/internal/models/user.go b/backend/internal/models/user.go new file mode 100644 index 0000000..1039933 --- /dev/null +++ b/backend/internal/models/user.go @@ -0,0 +1,78 @@ +package models + +import "time" + +// Role represents user roles +type Role string + +const ( + RoleAdmin Role = "admin" + RoleEditor Role = "editor" + RoleAnalyst Role = "analyst" + RoleViewer Role = "viewer" +) + +// Permission represents a specific permission +type Permission string + +const ( + PermViewLogs Permission = "logs:view" + PermExportLogs Permission = "logs:export" + PermManageAlerts Permission = "alerts:manage" + PermViewAlerts Permission = "alerts:view" + PermManagePatterns Permission = "patterns:manage" + PermRunML Permission = "patterns:ml" + PermManageReports Permission = "reports:manage" + PermManageUsers Permission = "users:manage" +) + +// RolePermissions maps roles to their permissions +var RolePermissions = map[Role][]Permission{ + RoleAdmin: { + PermViewLogs, PermExportLogs, PermManageAlerts, + PermViewAlerts, PermManagePatterns, PermRunML, + PermManageReports, PermManageUsers, + }, + RoleEditor: { + PermViewLogs, PermExportLogs, PermManageAlerts, + PermViewAlerts, PermManagePatterns, PermManageReports, + }, + RoleAnalyst: { + PermViewLogs, PermViewAlerts, PermManagePatterns, PermRunML, + }, + RoleViewer: { + PermViewLogs, PermViewAlerts, + }, +} + +// User represents a user in the system +type User struct { + ID string `json:"id" db:"id"` + Username string `json:"username" db:"username"` + Email string `json:"email" db:"email"` + PasswordHash string `json:"-" db:"password_hash"` + Role Role `json:"role" db:"role"` + IsActive bool `json:"is_active" db:"is_active"` + CreatedAt time.Time `json:"created_at" db:"created_at"` + UpdatedAt time.Time `json:"updated_at" db:"updated_at"` +} + +// HasPermission checks if the user has a specific permission +func (u *User) HasPermission(perm Permission) bool { + permissions, ok := RolePermissions[u.Role] + if !ok { + return false + } + + for _, p := range permissions { + if p == perm { + return true + } + } + return false +} + +// GetPermissions returns all permissions for the user's role +func (u *User) GetPermissions() []Permission { + return RolePermissions[u.Role] +} diff --git a/backend/migrations/001_init.sql b/backend/migrations/001_init.sql new file mode 100644 index 0000000..e7a6e40 --- /dev/null +++ b/backend/migrations/001_init.sql @@ -0,0 +1,132 @@ +-- Create users table +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + username VARCHAR(255) UNIQUE NOT NULL, + email VARCHAR(255) UNIQUE NOT NULL, + password_hash VARCHAR(255) NOT NULL, + role VARCHAR(50) NOT NULL CHECK (role IN ('admin', 'editor', 'analyst', 'viewer')), + is_active BOOLEAN DEFAULT true, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP +); + +-- Create index on username and email for faster lookups +CREATE INDEX idx_users_username ON users(username); +CREATE INDEX idx_users_email ON users(email); +CREATE INDEX idx_users_role ON users(role); + +-- Create alert_rules table +CREATE TABLE IF NOT EXISTS alert_rules ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name VARCHAR(255) NOT NULL, + group_name VARCHAR(255) NOT NULL, + type VARCHAR(50) NOT NULL CHECK (type IN ('alert', 'record')), + expression TEXT NOT NULL, + duration VARCHAR(50), + labels JSONB DEFAULT '{}', + annotations JSONB DEFAULT '{}', + severity VARCHAR(50), + enabled BOOLEAN DEFAULT true, + yaml_content TEXT NOT NULL, + created_by UUID REFERENCES users(id) ON DELETE SET NULL, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP +); + +-- Create indexes for alert_rules +CREATE INDEX idx_alert_rules_group_name ON alert_rules(group_name); +CREATE INDEX idx_alert_rules_enabled ON alert_rules(enabled); +CREATE INDEX idx_alert_rules_created_by ON alert_rules(created_by); + +-- Create patterns table +CREATE TABLE IF NOT EXISTS patterns ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name VARCHAR(255) NOT NULL, + type VARCHAR(50) NOT NULL CHECK (type IN ('regex', 'ml')), + description TEXT, + config JSONB NOT NULL, + severity VARCHAR(50), + tags TEXT[], + enabled BOOLEAN DEFAULT true, + created_by UUID REFERENCES users(id) ON DELETE SET NULL, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP +); + +-- Create indexes for patterns +CREATE INDEX idx_patterns_type ON patterns(type); +CREATE INDEX idx_patterns_enabled ON patterns(enabled); +CREATE INDEX idx_patterns_created_by ON patterns(created_by); + +-- Create reports table +CREATE TABLE IF NOT EXISTS reports ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name VARCHAR(255) NOT NULL, + description TEXT, + config JSONB NOT NULL, + created_by UUID REFERENCES users(id) ON DELETE SET NULL, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP +); + +-- Create index for reports +CREATE INDEX idx_reports_created_by ON reports(created_by); + +-- Create audit_logs table for tracking sensitive operations +CREATE TABLE IF NOT EXISTS audit_logs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID REFERENCES users(id) ON DELETE SET NULL, + action VARCHAR(255) NOT NULL, + resource_type VARCHAR(100), + resource_id UUID, + details JSONB, + ip_address INET, + user_agent TEXT, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP +); + +-- Create indexes for audit_logs +CREATE INDEX idx_audit_logs_user_id ON audit_logs(user_id); +CREATE INDEX idx_audit_logs_action ON audit_logs(action); +CREATE INDEX idx_audit_logs_created_at ON audit_logs(created_at); + +-- Create function to update updated_at timestamp +CREATE OR REPLACE FUNCTION update_updated_at_column() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = CURRENT_TIMESTAMP; + RETURN NEW; +END; +$$ language 'plpgsql'; + +-- Create triggers for updated_at +CREATE TRIGGER update_users_updated_at BEFORE UPDATE ON users + FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); + +CREATE TRIGGER update_alert_rules_updated_at BEFORE UPDATE ON alert_rules + FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); + +CREATE TRIGGER update_patterns_updated_at BEFORE UPDATE ON patterns + FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); + +CREATE TRIGGER update_reports_updated_at BEFORE UPDATE ON reports + FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); + +-- Insert default admin user (password: admin123 - CHANGE IN PRODUCTION!) +-- Password hash is bcrypt hash of "admin123" +INSERT INTO users (username, email, password_hash, role) +VALUES ( + 'admin', + 'admin@example.com', + '$2a$12$LQv3c1yqBWVHxkd0LHAkCOYz6TtxMQJqhN8/LewY5lW.HKCFcVpSK', + 'admin' +) ON CONFLICT (username) DO NOTHING; + +-- Insert sample viewer user (password: viewer123) +INSERT INTO users (username, email, password_hash, role) +VALUES ( + 'viewer', + 'viewer@example.com', + '$2a$12$3pYKzLw9z1FqN8QdKxI.J.nVZY5TqJxM7Y.qQHkZKZhzJ8qQnPZ8W', + 'viewer' +) ON CONFLICT (username) DO NOTHING; diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..60682c4 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,152 @@ +version: '3.8' + +services: + # VictoriaLogs - Log database + victorialogs: + image: victoriametrics/victoria-logs:latest + container_name: victorialogs + ports: + - "9428:9428" + volumes: + - vlogs-data:/victoria-logs-data + command: + - "-storageDataPath=/victoria-logs-data" + - "-retentionPeriod=30d" + networks: + - vlogs-network + restart: unless-stopped + + # vmalert - Alerting engine + vmalert: + image: victoriametrics/vmalert:latest + container_name: vmalert + ports: + - "8880:8880" + volumes: + - vmalert-rules:/etc/vmalert/rules + command: + - "-datasource.url=http://victorialogs:9428" + - "-rule=/etc/vmalert/rules/*.yml" + - "-notifier.url=http://alertmanager:9093" + depends_on: + - victorialogs + - alertmanager + networks: + - vlogs-network + restart: unless-stopped + + # Alertmanager - Alert notifications + alertmanager: + image: prom/alertmanager:latest + container_name: alertmanager + ports: + - "9093:9093" + volumes: + - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml + - alertmanager-data:/alertmanager + command: + - "--config.file=/etc/alertmanager/alertmanager.yml" + - "--storage.path=/alertmanager" + networks: + - vlogs-network + restart: unless-stopped + + # PostgreSQL - Application database + postgres: + image: postgres:15-alpine + container_name: postgres + environment: + POSTGRES_DB: ${DB_NAME:-vlogs_manager} + POSTGRES_USER: ${DB_USER:-vlogs} + POSTGRES_PASSWORD: ${DB_PASSWORD} + volumes: + - postgres-data:/var/lib/postgresql/data + - ./backend/migrations:/docker-entrypoint-initdb.d + ports: + - "5432:5432" + networks: + - vlogs-network + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-vlogs}"] + interval: 10s + timeout: 5s + retries: 5 + + # Go Backend API + backend: + build: + context: ./backend + dockerfile: Dockerfile + container_name: backend + ports: + - "8080:8080" + environment: + DB_HOST: postgres + DB_PORT: 5432 + DB_NAME: ${DB_NAME:-vlogs_manager} + DB_USER: ${DB_USER:-vlogs} + DB_PASSWORD: ${DB_PASSWORD} + VLOGS_URL: http://victorialogs:9428 + VMALERT_URL: http://vmalert:8880 + ML_SERVICE_URL: http://ml-service:8000 + JWT_SECRET: ${JWT_SECRET} + VMALERT_RULES_DIR: /app/rules + volumes: + - vmalert-rules:/app/rules + depends_on: + postgres: + condition: service_healthy + victorialogs: + condition: service_started + vmalert: + condition: service_started + ml-service: + condition: service_started + networks: + - vlogs-network + restart: unless-stopped + + # Python ML Service + ml-service: + build: + context: ./ml-service + dockerfile: Dockerfile + container_name: ml-service + ports: + - "8000:8000" + environment: + VLOGS_URL: http://victorialogs:9428 + MODEL_DIR: /app/models + volumes: + - ml-models:/app/models + networks: + - vlogs-network + restart: unless-stopped + + # React Frontend + frontend: + build: + context: ./frontend + dockerfile: Dockerfile + args: + VITE_API_BASE_URL: http://localhost:8080/api/v1 + container_name: frontend + ports: + - "3000:80" + depends_on: + - backend + networks: + - vlogs-network + restart: unless-stopped + +volumes: + vlogs-data: + postgres-data: + vmalert-rules: + ml-models: + alertmanager-data: + +networks: + vlogs-network: + driver: bridge diff --git a/frontend/.gitignore b/frontend/.gitignore new file mode 100644 index 0000000..005078b --- /dev/null +++ b/frontend/.gitignore @@ -0,0 +1,25 @@ +# Dependencies +node_modules/ +.pnp +.pnp.js + +# Build +dist/ +build/ +.vite/ + +# Cache +.cache/ +*.tsbuildinfo + +# Environment +.env.local +.env.development.local +.env.test.local +.env.production.local + +# Debug logs +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* diff --git a/frontend/Dockerfile b/frontend/Dockerfile new file mode 100644 index 0000000..a123bd2 --- /dev/null +++ b/frontend/Dockerfile @@ -0,0 +1,34 @@ +# Build stage +FROM node:18-alpine AS builder + +WORKDIR /app + +# Copy package files +COPY package*.json ./ + +# Install dependencies +RUN npm ci + +# Copy source code +COPY . . + +# Build argument for API URL +ARG VITE_API_BASE_URL +ENV VITE_API_BASE_URL=$VITE_API_BASE_URL + +# Build the application +RUN npm run build + +# Production stage +FROM nginx:alpine + +# Copy built assets from builder +COPY --from=builder /app/dist /usr/share/nginx/html + +# Copy nginx configuration +COPY nginx.conf /etc/nginx/conf.d/default.conf + +# Expose port +EXPOSE 80 + +CMD ["nginx", "-g", "daemon off;"] diff --git a/frontend/nginx.conf b/frontend/nginx.conf new file mode 100644 index 0000000..29a5747 --- /dev/null +++ b/frontend/nginx.conf @@ -0,0 +1,28 @@ +server { + listen 80; + server_name localhost; + root /usr/share/nginx/html; + index index.html; + + # Enable gzip compression + gzip on; + gzip_vary on; + gzip_min_length 1024; + gzip_types text/plain text/css text/xml text/javascript application/javascript application/xml+rss application/json; + + # SPA routing - serve index.html for all routes + location / { + try_files $uri $uri/ /index.html; + } + + # Cache static assets + location /assets/ { + expires 1y; + add_header Cache-Control "public, immutable"; + } + + # Security headers + add_header X-Frame-Options "SAMEORIGIN" always; + add_header X-Content-Type-Options "nosniff" always; + add_header X-XSS-Protection "1; mode=block" always; +} diff --git a/ml-service/.gitignore b/ml-service/.gitignore new file mode 100644 index 0000000..46905d7 --- /dev/null +++ b/ml-service/.gitignore @@ -0,0 +1,26 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +*.egg-info/ +dist/ +build/ + +# Virtual environment +venv/ +env/ +ENV/ + +# ML models +models/*.pkl +models/*.joblib +!models/.gitkeep + +# Jupyter +.ipynb_checkpoints/ + +# Cache +.pytest_cache/ +.mypy_cache/ diff --git a/ml-service/Dockerfile b/ml-service/Dockerfile new file mode 100644 index 0000000..ac030ee --- /dev/null +++ b/ml-service/Dockerfile @@ -0,0 +1,27 @@ +FROM python:3.11-slim + +# Set working directory +WORKDIR /app + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + gcc \ + && rm -rf /var/lib/apt/lists/* + +# Copy requirements file +COPY requirements.txt . + +# Install Python dependencies +RUN pip install --no-cache-dir -r requirements.txt + +# Copy application code +COPY app/ ./app/ + +# Create models directory +RUN mkdir -p models + +# Expose port +EXPOSE 8000 + +# Run the application +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/ml-service/app/__init__.py b/ml-service/app/__init__.py new file mode 100644 index 0000000..6686187 --- /dev/null +++ b/ml-service/app/__init__.py @@ -0,0 +1 @@ +# ML Service Package diff --git a/ml-service/app/main.py b/ml-service/app/main.py new file mode 100644 index 0000000..fe78dbe --- /dev/null +++ b/ml-service/app/main.py @@ -0,0 +1,122 @@ +from fastapi import FastAPI, HTTPException +from fastapi.middleware.cors import CORSMiddleware +from pydantic import BaseModel +from typing import List, Dict, Any, Optional +import logging + +# Configure logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +app = FastAPI( + title="VictoriaLogs ML Service", + description="Machine Learning service for anomaly detection in logs", + version="1.0.0" +) + +# CORS middleware +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +# Request/Response models +class TrainRequest(BaseModel): + model_name: str + query: str + start_time: str + end_time: str + contamination: float = 0.1 + +class DetectRequest(BaseModel): + model_name: str + logs: List[Dict[str, Any]] + +class TrainResponse(BaseModel): + status: str + model_name: str + model_path: str + message: str + +class DetectResponse(BaseModel): + status: str + anomalies: List[Dict[str, Any]] + anomaly_count: int + +@app.get("/") +async def root(): + return { + "service": "VictoriaLogs ML Service", + "status": "running", + "version": "1.0.0" + } + +@app.get("/health") +async def health_check(): + return {"status": "healthy"} + +@app.post("/api/ml/train", response_model=TrainResponse) +async def train_model(request: TrainRequest): + """ + Train an anomaly detection model on historical logs + """ + logger.info(f"Training model: {request.model_name}") + + # TODO: Implement model training + # 1. Fetch logs from VictoriaLogs + # 2. Extract features + # 3. Train Isolation Forest model + # 4. Save model to disk + + return TrainResponse( + status="success", + model_name=request.model_name, + model_path=f"/app/models/{request.model_name}.pkl", + message="Model training not yet implemented" + ) + +@app.post("/api/ml/detect", response_model=DetectResponse) +async def detect_anomalies(request: DetectRequest): + """ + Detect anomalies in a batch of logs using a trained model + """ + logger.info(f"Detecting anomalies with model: {request.model_name}") + + # TODO: Implement anomaly detection + # 1. Load trained model + # 2. Extract features from logs + # 3. Predict anomalies + # 4. Return flagged logs with scores + + return DetectResponse( + status="success", + anomalies=[], + anomaly_count=0 + ) + +@app.get("/api/ml/models") +async def list_models(): + """ + List all trained models + """ + # TODO: List model files from disk + return { + "models": [], + "count": 0 + } + +@app.delete("/api/ml/models/{model_name}") +async def delete_model(model_name: str): + """ + Delete a trained model + """ + logger.info(f"Deleting model: {model_name}") + + # TODO: Delete model file + return { + "status": "success", + "message": f"Model {model_name} deleted" + } diff --git a/ml-service/app/models/.gitkeep b/ml-service/app/models/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/ml-service/requirements.txt b/ml-service/requirements.txt new file mode 100644 index 0000000..806d2b9 --- /dev/null +++ b/ml-service/requirements.txt @@ -0,0 +1,9 @@ +fastapi==0.109.0 +uvicorn[standard]==0.27.0 +pydantic==2.5.3 +scikit-learn==1.4.0 +pandas==2.2.0 +numpy==1.26.3 +httpx==0.26.0 +python-multipart==0.0.6 +joblib==1.3.2