Commit 4929902d authored by Dillenn Terumalai's avatar Dillenn Terumalai
Browse files

// WIP

parent f5a35d27
test:package:
image: php:8
before_script:
- apt-get update
- apt-get install -qq git curl libmcrypt-dev libjpeg-dev libpng-dev libfreetype6-dev libbz2-dev libzip-dev zip
- apt-get clean
- pecl install mcrypt
- docker-php-ext-enable mcrypt
- docker-php-ext-install zip
- curl --silent --show-error "https://getcomposer.org/installer" | php -- --install-dir=/usr/local/bin --filename=composer
script:
- composer install --no-progress
- vendor/bin/psalm
- vendor/bin/phpunit --colors=always
- vendor/bin/security-checker security:check composer.lock
only:
- merge_requests
- dev
publish:package:
image: curlimages/curl:latest
stage: build
variables:
URL: "$CI_SERVER_PROTOCOL://$CI_SERVER_HOST:$CI_SERVER_PORT/api/v4/projects/$CI_PROJECT_ID/packages/composer?job_token=$CI_JOB_TOKEN"
script:
- version=$([[ -z "$CI_COMMIT_TAG" ]] && echo "branch=$CI_COMMIT_REF_NAME" || echo "tag=$CI_COMMIT_TAG")
- insecure=$([ "$CI_SERVER_PROTOCOL" = "http" ] && echo "--insecure" || echo "")
- response=$(curl -s -w "\n%{http_code}" $insecure --data $version $URL)
- code=$(echo "$response" | tail -n 1)
- body=$(echo "$response" | head -n 1)
# Output state information
- if [ $code -eq 201 ]; then
echo "Package created - Code $code - $body";
else
echo "Could not create package - Code $code - $body";
exit 1;
fi
only:
- tags
# Contributing
Contributions are **welcome** and will be fully **credited**.
Please read and understand the contribution guide before creating an issue or pull request.
## Etiquette
This project is open source, and as such, the maintainers give their free time to build and maintain the source code
held within. They make the code freely available in the hope that it will be of use to other developers. It would be
extremely unfair for them to suffer abuse or anger for their hard work.
Please be considerate towards maintainers when raising issues or presenting pull requests. Let's show the
world that developers are civilized and selfless people.
It's the duty of the maintainer to ensure that all submissions to the project are of sufficient
quality to benefit the project. Many developers have different skillsets, strengths, and weaknesses. Respect the maintainer's decision, and do not be upset or abusive if your submission is not used.
## Viability
When requesting or submitting new features, first consider whether it might be useful to others. Open
source projects are used by many developers, who may have entirely different needs to your own. Think about
whether or not your feature is likely to be used by other users of the project.
## Procedure
Before filing an issue:
- Attempt to replicate the problem, to ensure that it wasn't a coincidental incident.
- Check to make sure your feature suggestion isn't already present within the project.
- Check the pull requests tab to ensure that the bug doesn't have a fix in progress.
- Check the pull requests tab to ensure that the feature isn't already in progress.
Before submitting a pull request:
- Check the codebase to ensure that your feature doesn't already exist.
- Check the pull requests to ensure that another person hasn't already submitted the feature or fix.
## Requirements
If the project maintainer has any additional requirements, you will find them listed here.
- **[PSR-2 Coding Standard](https://github.com/php-fig/fig-standards/blob/master/accepted/PSR-2-coding-style-guide.md)** - The easiest way to apply the conventions is to install [PHP Code Sniffer](https://pear.php.net/package/PHP_CodeSniffer).
- **Add tests!** - Your patch won't be accepted if it doesn't have tests.
- **Document any change in behaviour** - Make sure the `README.md` and any other relevant documentation are kept up-to-date.
- **Consider our release cycle** - We try to follow [SemVer v2.0.0](https://semver.org/). Randomly breaking public APIs is not an option.
- **One pull request per feature** - If you want to do more than one thing, send multiple pull requests.
- **Send coherent history** - Make sure each individual commit in your pull request is meaningful. If you had to make multiple intermediate commits while developing, please [squash them](https://www.git-scm.com/book/en/v2/Git-Tools-Rewriting-History#Changing-Multiple-Commit-Messages) before submitting.
**Happy coding**!
# Security Policy
If you discover any security related issues, please email dillenn.terumalai@sib.swiss instead of using the issue tracker.
......@@ -16,15 +16,15 @@
}
],
"require": {
"php": "^8.0",
"php": "^7.4|^8.0",
"illuminate/contracts": "^8.37",
"inertiajs/inertia-laravel": "^0.4.2",
"spatie/laravel-package-tools": "^1.4.3",
"tightenco/ziggy": "^1.3",
"ext-pdo": "*"
"tightenco/ziggy": "^1.3"
},
"require-dev": {
"brianium/paratest": "^6.2",
"enlightn/security-checker": "^1.9",
"nunomaduro/collision": "^5.3",
"orchestra/testbench": "^6.15",
"phpunit/phpunit": "^9.3",
......
......@@ -7,7 +7,7 @@ return [
| Laravel Cluster Path
|--------------------------------------------------------------------------
|
| This is the URI path where Horizon will be accessible from. Feel free
| This is the URI path where LaravelCluster will be accessible from. Feel free
| to change this path to anything you like. Note that the URI will not
| affect the paths of its internal API that aren't exposed to users.
|
......@@ -17,10 +17,10 @@ return [
/*
|--------------------------------------------------------------------------
| Horizon Route Middleware
| LaravelCluster Route Middleware
|--------------------------------------------------------------------------
|
| These middleware will get attached onto each Horizon route, giving you
| These middleware will get attached onto each LaravelCluster route, giving you
| the chance to add your own middleware to this list or change any of
| the existing middleware. Or, you can simply stick with this list.
|
......
......@@ -31,7 +31,7 @@ class JobModelFactory extends Factory
'std_out_path' => '/path/to/std/out',
'std_err_path' => '/path/to/std/err',
'export_path' => null,
'queued_at' => Carbon::now()
'created_at' => Carbon::now()
];
}
}
......@@ -24,10 +24,11 @@ class CreateClusterJobsTable extends Migration
$table->text('std_err_path');
$table->text('export_path')->nullable();
$table->unsignedTinyInteger('attempts');
$table->dateTime('queued_at');
$table->dateTime('queued_at')->nullable();
$table->dateTime('cancelled_at')->nullable();
$table->dateTime('failed_at')->nullable();
$table->dateTime('completed_at')->nullable();
$table->dateTime('created_at');
});
}
......
This diff is collapsed.
{
"/js/app.js": "/js/app.js?id=f4d6e8f4d834a138e40f"
"/js/app.js": "/js/app.js?id=237f7a475c00d8b9bc83"
}
......@@ -9,12 +9,14 @@
<v-col cols="3">
<p>ID</p>
<p>Partition</p>
<p>Created At</p>
<p>Queued At</p>
<p>Cancelled At</p>
</v-col>
<v-col>
<p>{{ job.id }}</p>
<p>{{ job.partition }}</p>
<p>{{ job.created_at }}</p>
<p>{{ job.queued_at }}</p>
<p>{{ job.cancelled_at }}</p>
</v-col>
......
......@@ -9,6 +9,7 @@
<v-col cols="3">
<p>ID</p>
<p>Partition</p>
<p>Created At</p>
<p>Queued At</p>
<p>Completed At</p>
<p>Runtime (sec.)</p>
......@@ -16,6 +17,7 @@
<v-col>
<p>{{ job.id }}</p>
<p>{{ job.partition }}</p>
<p>{{ job.created_at }}</p>
<p>{{ job.queued_at }}</p>
<p>{{ job.completed_at }}</p>
<p>{{ job.runtime }}</p>
......
......@@ -19,6 +19,7 @@
<v-col cols="3">
<p>ID</p>
<p>Partition</p>
<p>Created At</p>
<p>Queued At</p>
<p>Failed At</p>
<p>Runtime (sec.)</p>
......@@ -26,6 +27,7 @@
<v-col>
<p>{{ job.id }}</p>
<p>{{ job.partition }}</p>
<p>{{ job.created_at }}</p>
<p>{{ job.queued_at }}</p>
<p>{{ job.failed_at }}</p>
<p>{{ job.runtime }}</p>
......
......@@ -9,11 +9,13 @@
<v-col cols="3">
<p>ID</p>
<p>Partition</p>
<p>Created At</p>
<p>Queued At</p>
</v-col>
<v-col>
<p>{{ job.id }}</p>
<p>{{ job.partition }}</p>
<p>{{ job.created_at }}</p>
<p>{{ job.queued_at }}</p>
</v-col>
</v-row>
......
......@@ -13,27 +13,6 @@ const Ziggy: {
}
} = {
"url": "http:\/\/localhost", "port": null, "defaults": {}, "routes": {
"horizon.stats.index": {"uri": "horizon\/api\/stats", "methods": ["GET", "HEAD"]},
"horizon.workload.index": {"uri": "horizon\/api\/workload", "methods": ["GET", "HEAD"]},
"horizon.masters.index": {"uri": "horizon\/api\/masters", "methods": ["GET", "HEAD"]},
"horizon.monitoring.index": {"uri": "horizon\/api\/monitoring", "methods": ["GET", "HEAD"]},
"horizon.monitoring.store": {"uri": "horizon\/api\/monitoring", "methods": ["POST"]},
"horizon.monitoring-tag.paginate": {"uri": "horizon\/api\/monitoring\/{tag}", "methods": ["GET", "HEAD"]},
"horizon.monitoring-tag.destroy": {"uri": "horizon\/api\/monitoring\/{tag}", "methods": ["DELETE"]},
"horizon.jobs-metrics.index": {"uri": "horizon\/api\/metrics\/jobs", "methods": ["GET", "HEAD"]},
"horizon.jobs-metrics.show": {"uri": "horizon\/api\/metrics\/jobs\/{id}", "methods": ["GET", "HEAD"]},
"horizon.queues-metrics.index": {"uri": "horizon\/api\/metrics\/queues", "methods": ["GET", "HEAD"]},
"horizon.queues-metrics.show": {"uri": "horizon\/api\/metrics\/queues\/{id}", "methods": ["GET", "HEAD"]},
"horizon.jobs-batches.index": {"uri": "horizon\/api\/batches", "methods": ["GET", "HEAD"]},
"horizon.jobs-batches.show": {"uri": "horizon\/api\/batches\/{id}", "methods": ["GET", "HEAD"]},
"horizon.jobs-batches.retry": {"uri": "horizon\/api\/batches\/retry\/{id}", "methods": ["POST"]},
"horizon.pending-jobs.index": {"uri": "horizon\/api\/jobs\/pending", "methods": ["GET", "HEAD"]},
"horizon.completed-jobs.index": {"uri": "horizon\/api\/jobs\/completed", "methods": ["GET", "HEAD"]},
"horizon.failed-jobs.index": {"uri": "horizon\/api\/jobs\/failed", "methods": ["GET", "HEAD"]},
"horizon.failed-jobs.show": {"uri": "horizon\/api\/jobs\/failed\/{id}", "methods": ["GET", "HEAD"]},
"horizon.retry-jobs.show": {"uri": "horizon\/api\/jobs\/retry\/{id}", "methods": ["POST"]},
"horizon.jobs.show": {"uri": "horizon\/api\/jobs\/{id}", "methods": ["GET", "HEAD"]},
"horizon.index": {"uri": "horizon\/{view?}", "methods": ["GET", "HEAD"]},
"cluster.": {"uri": "cluster", "methods": ["GET", "HEAD"]},
"cluster.dashboard": {"uri": "cluster\/dashboard", "methods": ["GET", "HEAD"]},
"cluster.inspector": {"uri": "cluster\/inspector", "methods": ["GET", "HEAD"]},
......
......@@ -49,9 +49,9 @@ trait ClusterQueueable
/**
* The export path for the job.
*
* @var string|null
* @var string
*/
public ?string $exportPath;
public string $exportPath = '/data';
/**
* The memory limit for the job.
......
......@@ -5,9 +5,13 @@ namespace Dterumal\LaravelCluster\Connectors;
use Dterumal\LaravelCluster\Contracts\ClusterInterface;
use Dterumal\LaravelCluster\Contracts\JobRepository;
use Dterumal\LaravelCluster\Events\JobCancelled;
use Dterumal\LaravelCluster\Events\JobCompleted;
use Dterumal\LaravelCluster\Events\JobFailed;
use Dterumal\LaravelCluster\Storage\JobModel;
use Illuminate\Support\Arr;
use Illuminate\Support\Carbon;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Storage;
use Illuminate\Support\Str;
use Symfony\Component\Process\Exception\ProcessFailedException;
......@@ -22,7 +26,7 @@ class SlurmConnector implements ClusterInterface
$process->run();
if (!$process->isSuccessful()) {
if(!is_null($onError)) {
if (!is_null($onError)) {
return $onError;
}
......@@ -56,6 +60,57 @@ class SlurmConnector implements ClusterInterface
})->toArray();
}
public function mapEvents(Collection $collection)
{
return $collection->map(function ($job) {
switch ($job['state']) {
case 'COMPLETED':
$event = 'completed_at';
break;
case 'CANCELLED':
$event = 'cancelled_at';
break;
case 'FAILED' || 'TIMEOUT' || 'DEADLINE' || 'REVOKED':
$event = 'failed_at';
break;
default:
$event = null;
break;
}
$info = [
'id' => $job['id'],
'queued_at' => $job['queued_at'],
];
if ($event) {
$info['datetime'] = $job['ended_at'];
$info['event'] = $event;
}
return $info;
});
}
public function triggerEvents(Collection $collection)
{
$collection->each(function ($job) {
$clusterJob = JobModel::findOrFail($job['id']);
$clusterJob->queued_at = $job['queued_at'];
$jobClass = $clusterJob->job;
if ($job['event'] === 'completed_at') {
event(new JobCompleted($clusterJob, $job['datetime']));
$jobClass->onSuccess($clusterJob);
} elseif ($job['event'] === 'failed_at') {
event(new JobFailed($clusterJob, $job['datetime']));
$jobClass->onFailure($clusterJob);
} elseif ($job['event'] === 'cancelled_at') {
event(new JobCancelled($clusterJob, $job['datetime']));
$jobClass->onCancel($clusterJob);
}
$clusterJob->update(Arr::except($job, ['event']));
});
}
/**
* @inheritDoc
*/
......@@ -119,65 +174,11 @@ class SlurmConnector implements ClusterInterface
'ended_at',
'state'
]);
}
$output = collect(explode("\n", $output))
->filter()
->map(function ($line) {
return str_getcsv($line, "|");
});
$header = collect([
]);
$rows = $output->splice(1);
$resultsWithEvents = $this->mapEvents($results);
$formattedResults = $rows->map(function ($row) use ($header) {
return $header->mapWithKeys(function ($item, $key) use ($row) {
$header = Str::remove(' ', Str::lower($item));
return [$header => $row[$key]];
});
})->map(function ($job) {
switch ($job['state']) {
case 'COMPLETED':
$event = 'completed_at';
break;
case 'CANCELLED':
$event = 'cancelled_at';
break;
case 'FAILED':
$event = 'failed_at';
break;
default:
$event = null;
}
$info = [
'id' => $job['id'],
'queued_at' => $job['queued_at'],
];
if ($event) {
$info[$event] = $job['ended_at'];
$info['event'] = $event;
}
return $info;
});
$formattedResults->each(function ($job) {
$clusterJob = JobModel::find($job['id']);
$jobClass = $clusterJob->job;
if ($job['event'] === 'completed_at') {
$jobClass->onSuccess($clusterJob);
} elseif ($job['event'] === 'failed_at') {
$jobClass->onFailure($clusterJob);
} elseif ($job['event'] === 'cancelled_at') {
$jobClass->onCancel($clusterJob);
}
$clusterJob->update(Arr::except($job, ['event']));
});
$this->triggerEvents($resultsWithEvents);
}
}
/**
......@@ -185,17 +186,12 @@ class SlurmConnector implements ClusterInterface
*/
public function status(): bool
{
$process = new Process([
$output = $this->execute([
'scontrol',
'ping'
]);
$process->run();
if (!$process->isSuccessful()) {
return false;
}
], fn() => '');
return Str::of($process->getOutput())->contains('UP');
return Str::of($output)->contains('UP');
}
/**
......@@ -203,35 +199,19 @@ class SlurmConnector implements ClusterInterface
*/
public function queues(): array
{
$process = new Process([
$username = config('cluster.connections.slurm.username');
$output = $this->execute([
'squeue',
'--format=%all',
'-u'.config('cluster.connections.slurm.username')
]);
$process->run();
"-u{$username}"
], fn() => []);
if (!$process->isSuccessful()) {
if (empty($output)) {
return [];
}
$output = $process->getOutput();
$output = collect(explode("\n", $output))
->filter()
->map(function ($line) {
return str_getcsv($line, "|");
});
$header = collect($output->first());
$rows = $output->splice(1);
return $rows->map(function ($row) use ($header) {
return $header->mapWithKeys(function ($item, $key) use ($row) {
$header = Str::remove(' ', Str::lower($item));
return [$header => $row[$key]];
});
})->toArray();
return $this->formatResults($output);
}
/**
......@@ -239,87 +219,39 @@ class SlurmConnector implements ClusterInterface
*/
public function run($job): void
{
// We format the array of commands to a string with newlines between commands
$content = implode("\n", $job->handle());
$files = [];
if (!empty($job->files())) {
$files = collect($job->files())->map(function ($file) use ($job) {
return sprintf(
"scp /tmp/$file \\\$USER@\\\$SLURM_SUBMIT_HOST:%s/$file",
$job->exportPath ?? job_path('\$SLURM_JOB_ID/exports')
);
})->toArray();
}
$exports = implode("\n", $files);
$payload = sprintf("<<EOF
#!/bin/bash
#SBATCH --output=/tmp/%%x_%%j.out
#SBATCH --error=/tmp/%%x_%%j.err
#SBATCH --output=%s
#SBATCH --error=%s
#SBATCH --job-name=%s
#SBATCH --partition=%s
#SBATCH --mem=%s
#SBATCH --time=%s
%s
%s
EOF",
"{$job->exportPath}/%x_%j.out",
"{$job->exportPath}/%x_%j.err",
$job->name,
$job->partition,
$job->memory,
$job->timeout,
$content,
$exports
$content
);
$process = Process::fromShellCommandline("sbatch --parsable $payload");
$process->run();
$failed = false;
if (!$process->isSuccessful()) {
if (config('app.debug')) {
throw new ProcessFailedException($process);
}
$failed = true;
throw new ProcessFailedException($process);
}
$job->id = (int) $process->getOutput();
Storage::makeDirectory("jobs/{$job->id}/exports");
app(JobRepository::class)->store($job, $payload, $failed);
}
/**
* @inheritDoc
*/
public function transferFiles(): void
{
$files = Storage::disk('transfer')->files();
$prefixCommand = config('cluster.connections.slurm.sudo') ? ['sudo', '-u', 'clinbio'] : [];
if (!empty($files)) {
foreach ($files as $file) {
$path = transfer_path($file);
$command = array_merge($prefixCommand, [
'salloc',
'--account=clbi',
'--partition=normal',
'sbcast',
'-f',
$path,
'/tmp/'.$file
]);
$process = new Process($command);
$process->run();
if (!$process->isSuccessful()) {
throw new ProcessFailedException($process);
}
Storage::disk('transfer')->delete($file);
}
}
app(JobRepository::class)->store($job, $payload);
}
}
......@@ -21,16 +21,6 @@ class RefreshClusterJobsCommand extends Command
*/
protected $description = 'Update jobs information in the database';
/**
* Create a new command instance.
*
* @return void
*/