diff --git a/README.md b/README.md index a5948bcedd6889b5056eee829c22fbbd81852b00..257eb41fefd4bd5f88529e4aec5d7a8d81094246 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Ce dépôt contient une petite infrastructure Docker Swarm capable de bruteforce ### Auto-scaling -Un **scaler** tourne dans le processus du backend. Il lit périodiquement la taille de la file Redis (`jobs:pending`) et ajuste le nombre de réplicas du service worker Swarm entre un min et un max configurables. Seuils avec hysteresis : scale up si les jobs en attente dépassent un seuil, scale down s’ils tombent en dessous d’un seuil plus bas. +Un **scaler** tourne dans le processus du backend. Il lit périodiquement la file Redis (`jobs:pending`) et le nombre de workers, puis ajuste le nombre de réplicas du service worker Swarm entre un min et un max configurables. Les seuils sont basés sur la charge (jobs par worker) : scale up si (jobs en attente / workers) dépasse un seuil, scale down si ça tombe en dessous d’un seuil plus bas (et qu’aucun job n’est en cours). Variables d’environnement (optionnelles, sur le service `api_backend`) : @@ -27,8 +27,8 @@ Variables d’environnement (optionnelles, sur le service `api_backend`) : | `SCALER_INTERVAL_MS` | `10000` | Période entre deux évaluations (ms). | | `SCALER_MIN_REPLICAS` | `1` | Nombre minimum de workers. | | `SCALER_MAX_REPLICAS` | `10` | Nombre maximum de workers. | -| `SCALER_SCALE_UP_WHEN_JOBS_ABOVE` | `5` | Scale up d’un réplica si jobs en attente ≥ cette valeur. | -| `SCALER_SCALE_DOWN_WHEN_JOBS_BELOW` | `1` | Scale down d’un réplica si jobs en attente ≤ cette valeur. | +| `SCALER_SCALE_UP_WHEN_JOBS_PER_WORKER_ABOVE` | `4` | Scale up si (jobs en attente / workers) ≥ cette valeur. | +| `SCALER_SCALE_DOWN_WHEN_JOBS_PER_WORKER_BELOW` | `1` | Scale down si (jobs en attente / workers) ≤ cette valeur (et aucun job en cours). | ### Démarrage rapide (esquisse) diff --git a/backend/src/clients.js b/backend/src/clients.js new file mode 100644 index 0000000000000000000000000000000000000000..715e0721efc6b47f10115216abe24dfbb5cee6a9 --- /dev/null +++ b/backend/src/clients.js @@ -0,0 +1,10 @@ +/** + * Shared Redis and Docker client instances. + * Single connection per process instead of one per route/scaler. + */ +import Redis from "ioredis"; +import Docker from "dockerode"; +import { REDIS_URL, DOCKER_SOCKET } from "./config.js"; + +export const redis = new Redis(REDIS_URL); +export const docker = new Docker({ socketPath: DOCKER_SOCKET }); diff --git a/backend/src/config.js b/backend/src/config.js new file mode 100644 index 0000000000000000000000000000000000000000..f4c6dacd8f73c8fd286e1b9735a84f820d3bf61e --- /dev/null +++ b/backend/src/config.js @@ -0,0 +1,32 @@ +/** + * Centralized configuration for the backend. + * All env-based constants and Redis key names live here. + */ + +// Redis +export const REDIS_URL = process.env.REDIS_URL || "redis://redis:6379"; + +export const REDIS_KEYS = { + JOBS_PENDING: "jobs:pending", + JOBS_RESULTS: "jobs:results", + JOBS_STATUS: "jobs:status", + JOBS_IN_PROGRESS: "jobs:in_progress", +}; + +// Docker +export const DOCKER_SOCKET = process.env.DOCKER_SOCKET || "/var/run/docker.sock"; +export const WORKER_SERVICE_NAME = process.env.WORKER_SERVICE_NAME || "md5_hash_worker"; + +// Server +export const PORT = Number(process.env.PORT) || 8080; + +// Scaler +export const SCALER_ENABLED = process.env.SCALER_ENABLED === "true"; +export const SCALER_INTERVAL_MS = Number(process.env.SCALER_INTERVAL_MS) || 10000; +export const SCALER_MIN_REPLICAS = Number(process.env.SCALER_MIN_REPLICAS) || 1; +export const SCALER_MAX_REPLICAS = Number(process.env.SCALER_MAX_REPLICAS) || 10; +// Load-based: scale by jobs per worker (pending / current replicas) +export const SCALER_SCALE_UP_WHEN_JOBS_PER_WORKER_ABOVE = + Number(process.env.SCALER_SCALE_UP_WHEN_JOBS_PER_WORKER_ABOVE) || 4; +export const SCALER_SCALE_DOWN_WHEN_JOBS_PER_WORKER_BELOW = + Number(process.env.SCALER_SCALE_DOWN_WHEN_JOBS_PER_WORKER_BELOW) || 1; diff --git a/backend/src/index.js b/backend/src/index.js index 0d9982b9603b815ab6e90b326c8c8a125a7fd379..b27b7238d39875d0496e6075662f6847c077c2f9 100644 --- a/backend/src/index.js +++ b/backend/src/index.js @@ -3,11 +3,21 @@ import cors from "cors"; import hashRouter from "./routes/hash.js"; import clusterRouter from "./routes/cluster.js"; import { startScaler } from "./services/scaler.js"; +import { PORT, SCALER_ENABLED } from "./config.js"; const app = express(); -const port = process.env.PORT || 8080; -app.use(cors({origin: "*"})); +/** + * @todo: + * use helmet for server wide input sanitization + */ + + +/**@note */ +// this is too permessive, but it is not a real world project so....we do not care as much! +// ideally, the best thing to do is to pin point the exact adresse of the frontend server, +// but this also means that we have to pass it as an env var for the backend server as well, (we can't hardcode it) +app.use(cors({origin: "*"})); app.use(express.json()); app.get("/health", (req, res) => { @@ -17,9 +27,9 @@ app.get("/health", (req, res) => { app.use("/hash", hashRouter); app.use("/cluster", clusterRouter); -app.listen(port, () => { - console.log(`Backend API listening on port ${port}`); - if (process.env.SCALER_ENABLED === "true") { +app.listen(PORT, () => { + console.log(`Backend API listening on port ${PORT}`); + if (SCALER_ENABLED) { startScaler(); } }); diff --git a/backend/src/routes/cluster.js b/backend/src/routes/cluster.js index e1a012ec33192cdf3f2bece8edf55d3ddbdb2175..a188d7eb12b5b5671cef05fd28d0ce29eca1afc0 100644 --- a/backend/src/routes/cluster.js +++ b/backend/src/routes/cluster.js @@ -1,43 +1,51 @@ import express from "express"; -import Docker from "dockerode"; -import Redis from "ioredis"; -import { scaleToReplicas } from "../services/clusterService.js"; +import { + WORKER_SERVICE_NAME, + REDIS_KEYS, + SCALER_ENABLED, + SCALER_INTERVAL_MS, + SCALER_MIN_REPLICAS, + SCALER_MAX_REPLICAS, + SCALER_SCALE_UP_WHEN_JOBS_PER_WORKER_ABOVE, + SCALER_SCALE_DOWN_WHEN_JOBS_PER_WORKER_BELOW, +} from "../config.js"; +import { docker, redis } from "../clients.js"; const router = express.Router(); -const dockerSocketPath = process.env.DOCKER_SOCKET || "/var/run/docker.sock"; -const docker = new Docker({ socketPath: dockerSocketPath }); - -const redisUrl = process.env.REDIS_URL || "redis://redis:6379"; -const redis = new Redis(redisUrl); - -const workerServiceName = process.env.WORKER_SERVICE_NAME || "md5_hash_worker"; - // GET /cluster/state - état simple du cluster router.get("/state", async (req, res) => { try { const [services, pendingCount] = await Promise.all([ docker.listServices(), - redis.llen("jobs:pending"), + redis.llen(REDIS_KEYS.JOBS_PENDING), ]); - const workerService = services.find( - (s) => s.Spec && s.Spec.Name === workerServiceName + const workerServiceInfo = services.find( + (s) => s.Spec && s.Spec.Name === WORKER_SERVICE_NAME ); let workerReplicas = 0; if ( - workerService && - workerService.Spec && - workerService.Spec.Mode && - workerService.Spec.Mode.Replicated + workerServiceInfo && + workerServiceInfo.Spec && + workerServiceInfo.Spec.Mode && + workerServiceInfo.Spec.Mode.Replicated ) { - workerReplicas = workerService.Spec.Mode.Replicated.Replicas || 0; + workerReplicas = workerServiceInfo.Spec.Mode.Replicated.Replicas || 0; } res.json({ pendingJobs: pendingCount, workerReplicas, + scaler: { + enabled: SCALER_ENABLED, + intervalMs: SCALER_INTERVAL_MS, + minReplicas: SCALER_MIN_REPLICAS, + maxReplicas: SCALER_MAX_REPLICAS, + scaleUpWhenJobsPerWorkerAbove: SCALER_SCALE_UP_WHEN_JOBS_PER_WORKER_ABOVE, + scaleDownWhenJobsPerWorkerBelow: SCALER_SCALE_DOWN_WHEN_JOBS_PER_WORKER_BELOW, + }, }); } catch (err) { console.error("Error getting cluster state", err); @@ -45,27 +53,6 @@ router.get("/state", async (req, res) => { } }); -// POST /cluster/scale - scale up/down des workers -router.post("/scale", async (req, res) => { - const desired = Number(req.body.replicas); - if (Number.isNaN(desired) || desired < 0) { - return res.status(400).json({ error: "invalid replicas count" }); - } - - const result = await scaleToReplicas(docker, workerServiceName, desired); - - if (result.ok) { - return res.json({ workerReplicas: result.workerReplicas }); - } - if (result.reason === "service_not_found") { - return res.status(404).json({ error: "worker service not found" }); - } - if (result.reason === "not_swarm") { - return res.status(503).json({ error: "Docker is not in Swarm mode" }); - } - console.error("Error scaling workers", result.error); - res.status(500).json({ error: "failed to scale workers" }); -}); export default router; diff --git a/backend/src/routes/hash.js b/backend/src/routes/hash.js index b329679c20aec296a68b1952931b53c959536628..d203a59f1cecda1c8b07b5308b575ca295fb99b6 100644 --- a/backend/src/routes/hash.js +++ b/backend/src/routes/hash.js @@ -1,13 +1,11 @@ import express from "express"; -import Redis from "ioredis"; import { v4 as uuidv4 } from "uuid"; +import { REDIS_KEYS } from "../config.js"; +import { redis } from "../clients.js"; const router = express.Router(); -const redisUrl = process.env.REDIS_URL || "redis://redis:6379"; -const redis = new Redis(redisUrl); - -// POST /hash/manual - envoyer un hash MD5 à bruteforcer +// POST /hash/manual - queing the hash to be bruteforced by the worker router.post("/manual", async (req, res) => { const { hash } = req.body; if (!hash) { @@ -17,21 +15,29 @@ router.post("/manual", async (req, res) => { const jobId = uuidv4(); const job = { id: jobId, hash, createdAt: Date.now() }; - await redis.lpush("jobs:pending", JSON.stringify(job)); - await redis.hset("jobs:status", jobId, JSON.stringify({ status: "queued" })); + /**@note */ + // Consider using Promise.then rather then two awaits, this + // will leave the nodejs runtime schedule them when it is + // appropriate regarding the current process load. Double + // awaits forces the event loop to consider these over other + // stuff, it is okey for critical stuff, but this is not ! + await redis.lpush(REDIS_KEYS.JOBS_PENDING, JSON.stringify(job)); + await redis.hset(REDIS_KEYS.JOBS_STATUS, jobId, JSON.stringify({ status: "queued" })); - res.status(202).json({ id: jobId }); + return res.status(202).json({ id: jobId }); }); -// GET /hash/:id - récupérer le résultat d’un bruteforce +// GET /hash/:id - get the status of a specific job router.get("/:id", async (req, res) => { const { id } = req.params; - const raw = await redis.hget("jobs:results", id); + const raw = await redis.hget(REDIS_KEYS.JOBS_RESULTS, id); + if (!raw) { - return res.status(404).json({ error: "result not found" }); + const jobStatus = await redis.hget(REDIS_KEYS.JOBS_STATUS, id); + return res.status(200).json(JSON.parse(jobStatus)); } - res.json(JSON.parse(raw)); + return res.json(JSON.parse(raw)); }); export default router; diff --git a/backend/src/services/clusterService.js b/backend/src/services/clusterService.js index 1c454b16a824938fe12ae2c623750560be6efc87..aa09dde84cdf33b507045b27f1e87ab8ee9ecbc3 100644 --- a/backend/src/services/clusterService.js +++ b/backend/src/services/clusterService.js @@ -6,25 +6,16 @@ * @param {number} desiredReplicas - Desired replica count * @returns {Promise<{ ok: boolean, workerReplicas?: number, reason?: string, error?: string }>} */ -export async function scaleToReplicas(docker, workerServiceName, desiredReplicas) { +export async function scaleToReplicas(docker, workerServiceInfo, desiredReplicas) { try { - const services = await docker.listServices(); - const serviceInfo = services.find( - (s) => s.Spec && s.Spec.Name === workerServiceName - ); - - if (!serviceInfo) { - return { ok: false, reason: "service_not_found" }; - } - - const service = docker.getService(serviceInfo.ID); - const spec = { ...serviceInfo.Spec }; + const workerService = docker.getService(workerServiceInfo.ID); + const spec = { ...workerServiceInfo.Spec }; spec.Mode = spec.Mode || {}; spec.Mode.Replicated = spec.Mode.Replicated || {}; spec.Mode.Replicated.Replicas = desiredReplicas; - await service.update({ - version: serviceInfo.Version.Index, + await workerService.update({ + version: workerServiceInfo.Version.Index, ...spec, }); diff --git a/backend/src/services/scaler.js b/backend/src/services/scaler.js index 5d1ca16862c059f7da5b76ac2b00e1dfa22dbadc..126a69f6186621d8e03c6f9d20409ede381865e7 100644 --- a/backend/src/services/scaler.js +++ b/backend/src/services/scaler.js @@ -1,22 +1,24 @@ -import Docker from "dockerode"; -import Redis from "ioredis"; import { scaleToReplicas } from "./clusterService.js"; - -const dockerSocketPath = process.env.DOCKER_SOCKET || "/var/run/docker.sock"; -const redisUrl = process.env.REDIS_URL || "redis://redis:6379"; -const workerServiceName = process.env.WORKER_SERVICE_NAME || "md5_hash_worker"; - -const SCALER_INTERVAL_MS = Number(process.env.SCALER_INTERVAL_MS) || 10000; -const SCALER_MIN_REPLICAS = Number(process.env.SCALER_MIN_REPLICAS) || 1; -const SCALER_MAX_REPLICAS = Number(process.env.SCALER_MAX_REPLICAS) || 10; -const SCALER_SCALE_UP_WHEN_JOBS_ABOVE = Number(process.env.SCALER_SCALE_UP_WHEN_JOBS_ABOVE) || 5; -const SCALER_SCALE_DOWN_WHEN_JOBS_BELOW = Number(process.env.SCALER_SCALE_DOWN_WHEN_JOBS_BELOW) || 3; +import { docker, redis } from "../clients.js"; +import { + WORKER_SERVICE_NAME, + REDIS_KEYS, + SCALER_INTERVAL_MS, + SCALER_MIN_REPLICAS, + SCALER_MAX_REPLICAS, + SCALER_SCALE_UP_WHEN_JOBS_PER_WORKER_ABOVE, + SCALER_SCALE_DOWN_WHEN_JOBS_PER_WORKER_BELOW, +} from "../config.js"; let intervalId = null; +// this will evaluate the current load, and decide if we need to scale up or down async function evaluate(docker, redis) { try { - const pendingCount = await redis.llen("jobs:pending"); + const [pendingCount, inProgressCount] = await Promise.all([ + redis.llen(REDIS_KEYS.JOBS_PENDING), + redis.scard(REDIS_KEYS.JOBS_IN_PROGRESS), + ]); let services; try { @@ -29,29 +31,42 @@ async function evaluate(docker, redis) { throw err; } - const workerService = services.find( - (s) => s.Spec && s.Spec.Name === workerServiceName + const workerServiceInfo = services.find( + (s) => s.Spec && s.Spec.Name === WORKER_SERVICE_NAME ); + if (!workerServiceInfo) { + console.warn("Scaler: scale failed, service_not_found."); + return; + } + let current = 0; - if (workerService?.Spec?.Mode?.Replicated != null) { - current = workerService.Spec.Mode.Replicated.Replicas ?? 0; + if (workerServiceInfo?.Spec?.Mode?.Replicated != null) { + current = workerServiceInfo.Spec.Mode.Replicated.Replicas ?? 0; } + // set lower bound to 1 to avoid div/0 + const effectiveWorkers = Math.max(1, current); + const jobsPerWorker = pendingCount / effectiveWorkers; + let desired = -1; - if (pendingCount >= SCALER_SCALE_UP_WHEN_JOBS_ABOVE) { + if (jobsPerWorker >= SCALER_SCALE_UP_WHEN_JOBS_PER_WORKER_ABOVE) { desired = Math.min(SCALER_MAX_REPLICAS, current + 1); - } else if (pendingCount <= SCALER_SCALE_DOWN_WHEN_JOBS_BELOW) { - desired = Math.max(SCALER_MIN_REPLICAS, current - 1); + } else if (jobsPerWorker <= SCALER_SCALE_DOWN_WHEN_JOBS_PER_WORKER_BELOW) { + if (inProgressCount === 0) { + desired = Math.max(SCALER_MIN_REPLICAS, current - 1); + } else { + console.log(`Scaler: skipping scale-down (jobs in progress: ${inProgressCount}, pending: ${pendingCount})`); + } } - if(desired === -1) return; // value did not change, so no todo + if (desired === -1) return; - const result = await scaleToReplicas(docker, workerServiceName, desired); + const result = await scaleToReplicas(docker, workerServiceInfo, desired); if (result.ok) { - console.log(`Scaler: scaled workers from ${current} to ${desired} (pending jobs: ${pendingCount})`); + console.log(`Scaler: scaled workers from ${current} to ${desired} (pending: ${pendingCount}, jobs/worker: ${jobsPerWorker.toFixed(1)})`); } else { console.warn("Scaler: scale failed", result.reason, result.error || ""); } @@ -65,10 +80,7 @@ export function startScaler() { console.log("Scaler: already started"); return; } - - const redis = new Redis(redisUrl); - const docker = new Docker({ socketPath: dockerSocketPath }); - + intervalId = setInterval(() => evaluate(docker, redis), SCALER_INTERVAL_MS); console.log(`Scaler: started (interval ${SCALER_INTERVAL_MS}ms, min=${SCALER_MIN_REPLICAS}, max=${SCALER_MAX_REPLICAS})`); diff --git a/cmds.txt b/cmds.txt new file mode 100644 index 0000000000000000000000000000000000000000..55f61b3829f4eb22c94ac644ca6778fd70accb1b --- /dev/null +++ b/cmds.txt @@ -0,0 +1,13 @@ + + docker swarm leave --force + docker swarm init --advertise-addr 10.67.138.118 + + docker build -t md5-swarm-backend:latest ./backend + docker build -t md5-swarm-worker:latest ./worker + docker build -t md5-swarm-frontend:latest ./frontend + + docker stack deploy -c infra/stack.yml md5 + docker stack services md5 + + + docker service logs service_id diff --git a/flow.txt b/flow.txt new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 421c5f375296a55cb564e3e7811e7d6c16a0e3a7..b9a4c6a79eb0eec3e850853058636ec13eff73c8 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -8,6 +8,7 @@ "name": "md5-swarm-frontend", "version": "0.1.0", "dependencies": { + "crypto-js": "^4.2.0", "react": "^18.3.1", "react-dom": "^18.3.1" }, @@ -1292,6 +1293,12 @@ "dev": true, "license": "MIT" }, + "node_modules/crypto-js": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.2.0.tgz", + "integrity": "sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==", + "license": "MIT" + }, "node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", diff --git a/frontend/package.json b/frontend/package.json index 1e7346a5cfe24aba0875ecc8f56de9c102a7d067..178c676cbff48e0c46ef6be28a08cb6296df6fa5 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -9,6 +9,7 @@ "test": "echo \"no frontend tests yet\"" }, "dependencies": { + "crypto-js": "^4.2.0", "react": "^18.3.1", "react-dom": "^18.3.1" }, diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 1a1d947e2ba18c159c48ef29aeb65f94c9d73fdd..844956635f608ca68d158d45f0b873105774273f 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -1,13 +1,28 @@ import React, { useEffect, useState } from "react"; +import CryptoJS from "crypto-js"; const MODES = { - gentle: 5000, - normal: 2000, + veryGentle: 5000, // nice for quick debugging + gentle: 3000, + normal: 1500, + fast: 1000, aggressive: 500, + veryAggressive: 250, +}; + +const MODE_KEYS = ["veryGentle", "gentle", "normal", "fast", "aggressive", "veryAggressive"]; +const intervalLabel = (ms) => (ms >= 1000 ? `${ms / 1000}s` : `${ms}ms`); +const MODE_LABELS = { + veryGentle: `Très gentil (${intervalLabel(MODES.veryGentle)})`, + gentle: `Gentil (${intervalLabel(MODES.gentle)})`, + normal: `Normal (${intervalLabel(MODES.normal)})`, + fast: `Rapide (${intervalLabel(MODES.fast)})`, + aggressive: `Agressif (${intervalLabel(MODES.aggressive)})`, + veryAggressive: `Très agressif (${intervalLabel(MODES.veryAggressive)})`, }; async function apiRequest(path, options = {}) { - const res = await fetch(`/api${path}`, { + const res = await fetch(`${import.meta.env.VITE_API_URL || "http://localhost:8080"}${path}`, { headers: { "Content-Type": "application/json" }, ...options, }); @@ -20,12 +35,14 @@ async function apiRequest(path, options = {}) { function App() { const [hashInput, setHashInput] = useState(""); - const [mode, setMode] = useState("gentle"); + + const [modeStep, setModeStep] = useState(2); // default: normal (2s) + const mode = MODE_KEYS[modeStep]; + const [autoRunning, setAutoRunning] = useState(false); const [jobs, setJobs] = useState([]); const [clusterState, setClusterState] = useState(null); - // Envoi manuel const sendManual = async () => { if (!hashInput) return; try { @@ -41,52 +58,75 @@ function App() { } }; - // Polling des résultats pour les jobs connus useEffect(() => { - let cancelled = false; - async function poll() { - if (jobs.length === 0) return; - try { - const updated = await Promise.all( - jobs.map(async (job) => { + const poll = async () => { + // We use a functional update to peek at the latest jobs without + // needing jobs in the dependency array + setJobs(currentJobs => { + const pendingJobs = currentJobs.filter(j => !j.result); + + // If nothing to poll, return existing state to avoid re-render + if (pendingJobs.length === 0) return currentJobs; + + // Trigger the API calls + Promise.all( + currentJobs.map(async (job) => { + + // if the job obj already has a result key + // then skip if (job.result) return job; + + // otherwise, request the backend try { const res = await apiRequest(`/hash/${job.id}`); - return { ...job, result: res }; + return res.status === "queued" ? job : { ...job, result: res }; } catch { return job; } }) - ); - if (!cancelled) setJobs(updated); + ).then(updatedResults => { + // Only update if something actually changed to prevent infinite loops + setJobs(updatedResults); + }); + + return currentJobs; // Return current state while waiting for Promise + }); + }; + + const pollId = setInterval(poll, 2000); + return () => clearInterval(pollId); + }, []); + + useEffect(() => { + // Monitoring cluster + async function pollCluster() { + try { + const state = await apiRequest("/cluster/state"); + setClusterState(state); } catch (e) { - console.error("poll error", e); + console.error("cluster state error", e); } } - const id = setInterval(poll, 2000); + + pollCluster(); + let pollClusterId = setInterval(pollCluster, 5000); + return () => { - cancelled = true; - clearInterval(id); - }; - }, [jobs]); + clearInterval(pollClusterId) + } + }, []) + // Envoi auto de hash aléatoires selon le mode useEffect(() => { + if (!autoRunning) return; - + const delay = MODES[mode] || MODES.normal; - + const interval = setInterval(async () => { - - const randomText = Math.random().toString(36).slice(2, 8); - const encoder = new TextEncoder(); - const data = encoder.encode(randomText); - const digest = await window.crypto.subtle.digest("MD5", data).catch(() => null); - - if (!digest) return; - - const hashArray = Array.from(new Uint8Array(digest)); - const hash = hashArray.map((b) => b.toString(16).padStart(2, "0")).join(""); + const randomText = Math.random().toString(36).slice(2, 6); + const hash = CryptoJS.MD5(randomText).toString(CryptoJS.enc.Hex); try { const res = await apiRequest("/hash/manual", { @@ -94,39 +134,40 @@ function App() { body: JSON.stringify({ hash }), }); setJobs((prev) => [...prev, { id: res.id, hash }]); - } catch (e) { console.error("auto send error", e); } - }, delay); - + return () => clearInterval(interval); - + }, [autoRunning, mode]); - // Monitoring cluster - useEffect(() => { - let cancelled = false; - - async function pollCluster() { - try { - const state = await apiRequest("/cluster/state"); - if (!cancelled) setClusterState(state); - } catch (e) { - console.error("cluster state error", e); - } + const downloadBlob = (blob, filename) => { + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = filename; + a.click(); + URL.revokeObjectURL(url); + }; + + const handleExport = () => { + if (jobs.length === 0) { + alert("Aucun job à exporter."); + return; } - pollCluster(); - - const id = setInterval(pollCluster, 5000); - - return () => { - cancelled = true; - clearInterval(id); - }; - - }, []); + const inputsContent = jobs.map((j) => j.hash).join("\n"); + const outputsContent = jobs + .map((j) => { + if (!j.result) return "PENDING"; + return j.result.found ? j.result.plaintext : "NOT_FOUND"; + }) + .join("\n"); + const ts = new Date().toISOString().slice(0, 19).replace(/[-:T]/g, ""); + downloadBlob(new Blob([inputsContent], { type: "text/plain" }), `export_inputs_${ts}.txt`); + downloadBlob(new Blob([outputsContent], { type: "text/plain" }), `export_outputs_${ts}.txt`); + }; return (
@@ -150,40 +191,65 @@ function App() {

Mode automatique

-
-
- -
- -
- +
+
+ + + {MODE_LABELS[mode]} + +
+ + Télécharge deux fichiers : hashes (inputs) et résultats (outputs), un élément par ligne, même ordre. + +
+ )} {jobs.length === 0 ? (

Aucun job pour le moment.

) : ( diff --git a/frontend/vite.config.mts b/frontend/vite.config.mts index 503b25ec62855c58c76afa5ce54cca57b98233ba..869e853ec098431a35441ca1ae5d4f924f82e314 100644 --- a/frontend/vite.config.mts +++ b/frontend/vite.config.mts @@ -4,14 +4,7 @@ import react from "@vitejs/plugin-react"; export default defineConfig({ plugins: [react()], server: { - port: 5173, - proxy: { - "/api": { - target: "http://localhost:8080", - changeOrigin: true, - rewrite: (path) => path.replace(/^\/api/, ""), - }, - }, + port: 5173 }, }); diff --git a/infra/stack.yml b/infra/stack.yml index 6b807929753af300ae995bc0c0962224b59e4c9b..bd1a3ef29baab26c9dd1d250d8bf5b836d38effe 100644 --- a/infra/stack.yml +++ b/infra/stack.yml @@ -5,6 +5,9 @@ services: image: md5-swarm-backend:latest build: context: ../backend + depends_on: + - redis + - hash_worker ports: - "8080:8080" environment: @@ -13,11 +16,11 @@ services: - WORKER_SERVICE_NAME=md5_hash_worker # Scaler (auto-scale workers from queue size) - SCALER_ENABLED=true - - SCALER_INTERVAL_MS=10000 + - SCALER_INTERVAL_MS=5000 - SCALER_MIN_REPLICAS=1 - SCALER_MAX_REPLICAS=10 - - SCALER_SCALE_UP_WHEN_JOBS_ABOVE=5 - - SCALER_SCALE_DOWN_WHEN_JOBS_BELOW=3 + - SCALER_SCALE_UP_WHEN_JOBS_PER_WORKER_ABOVE=4 + - SCALER_SCALE_DOWN_WHEN_JOBS_PER_WORKER_BELOW=1 volumes: - /var/run/docker.sock:/var/run/docker.sock networks: @@ -27,6 +30,8 @@ services: image: md5-swarm-worker:latest build: context: ../worker + depends_on: + - redis deploy: mode: replicated replicas: 2 @@ -44,6 +49,8 @@ services: image: md5-swarm-frontend:latest build: context: ../frontend + depends_on: + - api_backend ports: - "5173:80" environment: diff --git a/todo.txt b/todo.txt new file mode 100644 index 0000000000000000000000000000000000000000..9c4867b6f1a84e21c1813040e8599c73897917ac --- /dev/null +++ b/todo.txt @@ -0,0 +1,12 @@ +* make the UI a bit nicer, add timers and loaders, (ex; add the text "refreshing the cluster state +in 2seconds" to the view) + +* add an export button to export the inputs and the outputs of automatic hash generation and submitting, make the input and + output files be seperate this way the teacher can inject the inputs to his program and check if the outputs match + +* implement the CI using gitlab tool chain. + +* review how we use async/await, and if we do really need those awaits everywhere ? + +* review the code, simplify and simplify! + diff --git a/worker/src/index.js b/worker/src/index.js index a9d7f2248881a89ec4a69ea3e53597ced18d0c7c..2987da15cf704694ff811bcfd40c4a74a8311849 100644 --- a/worker/src/index.js +++ b/worker/src/index.js @@ -10,11 +10,31 @@ const maxLength = Number(process.env.MAX_LENGTH || 4); async function bruteforce(hash) { const start = Date.now(); + + // this is a generator, which is a little bit advanced, + // but the idea is simple, we basically go over all + // possible words in an on-demand like fashion, this + // allows us to have a lazy computation workflow that + // allows us to satisfy our need without pre-computing + // all the possible words before hand which requires + // much more memory to store them all! + + // If you draw the execution path of this generator + // function, you'll see that the recursive calls generate + // the words in a tree like fashion. function* generateStrings(maxLen, prefix = "") { + // base case + if (prefix.length === maxLen){ + yield prefix; + return; + }; + + // yield current state/word if (prefix.length > 0) { yield prefix; } - if (prefix.length === maxLen) return; + + // generate next words for (const ch of charset) { yield* generateStrings(maxLen, prefix + ch); } @@ -32,17 +52,34 @@ async function bruteforce(hash) { return { found: false, plaintext: null, elapsedMs }; } + +// this is our main function of the woker async function processJobs() { + let job; + while (true) { + + job = null; + try { + + // poll a pending job from the queue const res = await redis.brpop("jobs:pending", 0); if (!res) continue; const [, payload] = res; - const job = JSON.parse(payload); + job = JSON.parse(payload); + // register its id in the in_progress queue + // this is important insights for the scaler! + await redis.sadd("jobs:in_progress", job.id); + + + // solve the job hash const result = await bruteforce(job.hash); + + // save the results await redis.hset( "jobs:results", job.id, @@ -53,16 +90,56 @@ async function processJobs() { completedAt: Date.now(), }) ); + + // save the status await redis.hset( "jobs:status", job.id, JSON.stringify({ status: "done" }) ); + // unregister the job id from the in_progress + // so now worker goes idle, which is important + // info for the scaler + await redis.srem("jobs:in_progress", job.id); + console.log(`Job ${job.id} processed, found=${result.found}`); + + } catch (err) { console.error("Worker error:", err); - await new Promise((resolve) => setTimeout(resolve, 1000)); + + + // check if the error occured after the job polling + // if that is true, then job should not be undefined + if (job?.id) { + + // register the results after error + await redis.hset( + "jobs:results", + job.id, + JSON.stringify({ + id: job.id, + hash: job.hash, + found: false, + plaintext: null, + error: true, + completedAt: Date.now(), + }) + ); + + // register the status after error + await redis.hset( + "jobs:status", + job.id, + JSON.stringify({ status: "failed" }) + ); + + // do not forget to remove if from the in_progress queue + await redis.srem("jobs:in_progress", job.id); + } + + console.log(`Job ${job?.id ?? "?"} failed (error)`); } } }