Commit 104339db authored by Dillenn Terumalai's avatar Dillenn Terumalai
Browse files

fix: Fix some update issues and improved event system to monitor cluster status

parent 6570fee2
Pipeline #2356 passed with stage
in 6 seconds
......@@ -64,13 +64,31 @@ Laravel cluster includes a metrics dashboard which provides information regardin
*/
protected function schedule(Schedule $schedule)
{
$schedule->command('laravel-cluster:refresh')->everyFiveMinutes();
$schedule->command('laravel-cluster:refresh')->everyMinute();
}
```
### Monitor cluster status
You can easily monitor the cluster status by configure Laravel cluster's `monitor` Artisan command to run every 5 minutes via your application's scheduler:
```php
/**
* Define the application's command schedule.
*
* @param \Illuminate\Console\Scheduling\Schedule $schedule
* @return void
*/
protected function schedule(Schedule $schedule)
{
$schedule->command('laravel-cluster:monitor')->everyFiveMinutes();
}
```
### Notifications
If you would like to be notified when the cluster is unreachable, you may use the LaravelCluster::routeMailNotificationsTo method. You may call this method from the boot method of your application's App\Providers\LaravelClusterServiceProvider:
If you would like to be notified when the cluster is unreachable/down, you may use the LaravelCluster::routeMailNotificationsTo method. You may call this method from the boot method of your application's App\Providers\LaravelClusterServiceProvider:
```php
/**
......
......@@ -5,13 +5,15 @@ namespace Dterumal\LaravelCluster\Connectors;
use Dterumal\LaravelCluster\Contracts\ClusterInterface;
use Dterumal\LaravelCluster\Contracts\JobRepository;
use Dterumal\LaravelCluster\Events\ClusterDownDetected;
use Dterumal\LaravelCluster\Events\ClusterUpDetected;
use Dterumal\LaravelCluster\Events\JobCancelled;
use Dterumal\LaravelCluster\Events\JobCompleted;
use Dterumal\LaravelCluster\Events\JobFailed;
use Dterumal\LaravelCluster\Events\JobQueued;
use Dterumal\LaravelCluster\Storage\JobModel;
use Illuminate\Support\Arr;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Str;
use Symfony\Component\Process\Exception\ProcessFailedException;
use Symfony\Component\Process\Process;
......@@ -62,7 +64,6 @@ class SlurmConnector implements ClusterInterface
public function mapEvents(Collection $collection)
{
return $collection->map(function ($job) {
Log::info('Job STATE on refresh is ' . $job['state']);
switch ($job['state']) {
case 'COMPLETED':
$event = 'completed_at';
......@@ -95,7 +96,7 @@ class SlurmConnector implements ClusterInterface
{
$collection->each(function ($job) {
$clusterJob = JobModel::findOrFail($job['id']);
$clusterJob->queued_at = $job['queued_at'];
event(new JobQueued($clusterJob, $job['queued_at']));
$jobClass = $clusterJob->job;
if ($job['event'] === 'completed_at') {
event(new JobCompleted($clusterJob, $job['datetime']));
......@@ -107,8 +108,6 @@ class SlurmConnector implements ClusterInterface
event(new JobCancelled($clusterJob, $job['datetime']));
$jobClass->onCancel($clusterJob);
}
$clusterJob->update(Arr::except($job, ['event']));
});
}
......@@ -261,4 +260,20 @@ EOF",
app(JobRepository::class)->store($job, $payload);
}
/**
* @inheritDoc
*/
public function monitorStatus(): void {
$output = $this->execute([
'scontrol',
'ping'
], fn() => '');
if(Str::of($output)->contains('UP')) {
event(new ClusterUpDetected());
} else {
event(new ClusterDownDetected());
}
}
}
<?php
namespace Dterumal\LaravelCluster\Console;
use Dterumal\LaravelCluster\Contracts\ClusterInterface;
use Illuminate\Console\Command;
class MonitorClusterStatusCommand extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'laravel-cluster:monitor';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Monitor the cluster status and notify by mail if down';
/**
* Execute the console command.
*
* @return void
*/
public function handle(): void
{
app(ClusterInterface::class)->monitorStatus();
}
}
......@@ -13,6 +13,13 @@ interface ClusterInterface
*/
public function status(): bool;
/**
* Monitor the cluster status
*
* @return void
*/
public function monitorStatus(): void;
/**
* Returns all the nodes
*
......
......@@ -25,6 +25,10 @@ trait EventMap
Listeners\MarkJobAsCompleted::class
],
Events\JobQueued::class => [
Listeners\MarkJobAsQueued::class
],
Events\ClusterUpDetected::class => [
Listeners\ResetClusterStatus::class
],
......
<?php
namespace Dterumal\LaravelCluster\Events;
use Dterumal\LaravelCluster\Storage\JobModel;
class JobQueued
{
/**
* The queue job instance.
*
* @var JobModel
*/
public JobModel $job;
/**
* The datetime of the event.
*
* @var string
*/
public string $datetime;
/**
* Create a new event instance.
*
* @param JobModel $job
* @param string $datetime
*/
public function __construct(JobModel $job, string $datetime)
{
$this->job = $job;
$this->datetime = $datetime;
}
}
......@@ -114,7 +114,8 @@ class LaravelClusterServiceProvider extends LaravelClusterApplicationServiceProv
Console\ClearCommand::class,
Console\InstallCommand::class,
Console\PublishCommand::class,
Console\RefreshClusterJobsCommand::class
Console\RefreshClusterJobsCommand::class,
Console\MonitorClusterStatusCommand::class
]);
}
}
......
<?php
namespace Dterumal\LaravelCluster\Listeners;
use Dterumal\LaravelCluster\Contracts\JobRepository;
use Dterumal\LaravelCluster\Events\JobQueued;
class MarkJobAsQueued
{
/**
* The job repository implementation.
*
* @var \Dterumal\LaravelCluster\Contracts\JobRepository
*/
public JobRepository $jobs;
/**
* Create a new listener instance.
*
* @param \Dterumal\LaravelCluster\Contracts\JobRepository $jobs
* @return void
*/
public function __construct(JobRepository $jobs)
{
$this->jobs = $jobs;
}
/**
* Handle the event.
*
* @param \Dterumal\LaravelCluster\Events\JobQueued $event
* @return void
*/
public function handle(JobQueued $event): void
{
$this->jobs->queued($event->job, $event->datetime);
}
}
......@@ -44,8 +44,7 @@ class DatabaseJobRepository implements JobRepository
public function queued(JobModel $clusterJob, string $datetime): void
{
$clusterJob->update([
'failed_at' => $datetime,
'queued_at' => $clusterJob->queued_at
'queued_at' => $datetime
]);
}
......@@ -59,8 +58,7 @@ class DatabaseJobRepository implements JobRepository
public function failed(JobModel $clusterJob, string $datetime): void
{
$clusterJob->update([
'failed_at' => $datetime,
'queued_at' => $clusterJob->queued_at
'failed_at' => $datetime
]);
}
......@@ -74,8 +72,7 @@ class DatabaseJobRepository implements JobRepository
public function cancelled(JobModel $clusterJob, string $datetime): void
{
$clusterJob->update([
'failed_at' => $datetime,
'queued_at' => $clusterJob->queued_at
'cancelled_at' => $datetime
]);
}
......@@ -89,8 +86,7 @@ class DatabaseJobRepository implements JobRepository
public function completed(JobModel $clusterJob, string $datetime): void
{
$clusterJob->update([
'failed_at' => $datetime,
'queued_at' => $clusterJob->queued_at
'completed_at' => $datetime
]);
}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment