From fc143811cf15b607452fed1da5a72fb2b5a2585a Mon Sep 17 00:00:00 2001 From: Roly Gutierrez Date: Tue, 26 Nov 2024 20:27:26 -0400 Subject: [PATCH 1/4] FOUR-20535 Initial Setup (Laravel + Prometheus Integration) --- ProcessMaker/Facades/Metrics.php | 19 +++ .../Designer/DesignerController.php | 6 + .../Providers/MetricsServiceProvider.php | 27 +++++ ProcessMaker/Services/MetricsService.php | 110 ++++++++++++++++++ composer.json | 1 + composer.lock | 70 ++++++++++- config/app.php | 1 + routes/web.php | 8 ++ tests/Feature/MetricsFacadeTest.php | 58 +++++++++ tests/unit/MetricsServiceTest.php | 61 ++++++++++ 10 files changed, 360 insertions(+), 1 deletion(-) create mode 100644 ProcessMaker/Facades/Metrics.php create mode 100644 ProcessMaker/Providers/MetricsServiceProvider.php create mode 100644 ProcessMaker/Services/MetricsService.php create mode 100644 tests/Feature/MetricsFacadeTest.php create mode 100644 tests/unit/MetricsServiceTest.php diff --git a/ProcessMaker/Facades/Metrics.php b/ProcessMaker/Facades/Metrics.php new file mode 100644 index 0000000000..d78727a47b --- /dev/null +++ b/ProcessMaker/Facades/Metrics.php @@ -0,0 +1,19 @@ +app->singleton(MetricsService::class, function ($app) { + return new MetricsService(); + }); + } + + /** + * Bootstrap services. + */ + public function boot(): void + { + // + } +} diff --git a/ProcessMaker/Services/MetricsService.php b/ProcessMaker/Services/MetricsService.php new file mode 100644 index 0000000000..dc4d463b06 --- /dev/null +++ b/ProcessMaker/Services/MetricsService.php @@ -0,0 +1,110 @@ + env('REDIS_HOST', '127.0.0.1'), + 'port' => env('REDIS_PORT', '6379') + ]); + } + $this->registry = new CollectorRegistry($adapter); + } + + /** + * Returns the CollectorRegistry used by the MetricsService. + * + * @return \Prometheus\CollectorRegistry The CollectorRegistry used by the MetricsService. + */ + public function getMetrics() + { + return $this->registry; + } + + /** + * Retrieves a metric by its name. + * + * This method iterates through all registered metrics and returns the first metric that matches the given name. + * If no metric with the specified name is found, it returns null. + * + * @param string $name The name of the metric to retrieve. + * @return \Prometheus\MetricFamilySamples|null The metric with the specified name, or null if not found. + */ + public function getMetricByName(string $name) + { + $metrics = $this->registry->getMetricFamilySamples(); + foreach ($metrics as $metric) { + if ($metric->getName() === $name) { + return $metric; + } + } + return null; + } + + /** + * Registers a new counter metric. + * + * @param string $name The name of the counter. + * @param string $help The help text of the counter. + * @param array $labels The labels of the counter. + * @return \Prometheus\Counter The registered counter. + */ + public function registerCounter(string $name, string $help, array $labels = []) + { + return $this->registry->registerCounter('app', $name, $help, $labels); + } + + /** + * Registers a new histogram metric. + * + * @param string $name The name of the histogram. + * @param string $help The help text of the histogram. + * @param array $labels The labels of the histogram. + * @param array $buckets The buckets of the histogram. + * @return \Prometheus\Histogram The registered histogram. + */ + public function registerHistogram(string $name, string $help, array $labels = [], array $buckets = []) + { + return $this->registry->registerHistogram('app', $name, $help, $labels, $buckets); + } + + /** + * Increments a counter metric by 1. + * + * @param string $name The name of the counter. + * @param array $labelValues The values of the labels for the counter. + */ + public function incrementCounter(string $name, array $labelValues = []) + { + $counter = $this->registry->getCounter('app', $name); + $counter->inc($labelValues); + } + + /** + * Renders the metrics in the Prometheus text format. + * + * @return string The rendered metrics. + */ + public function renderMetrics() + { + $renderer = new RenderTextFormat(); + $metrics = $this->registry->getMetricFamilySamples(); + return $renderer->render($metrics); + } +} diff --git a/composer.json b/composer.json index 5180ea0eec..76f4b43082 100644 --- a/composer.json +++ b/composer.json @@ -51,6 +51,7 @@ "processmaker/laravel-i18next": "dev-master", "processmaker/nayra": "1.12.0", "processmaker/pmql": "1.12.1", + "promphp/prometheus_client_php": "^2.12", "psr/http-message": "^1.1", "psr/log": "^2.0", "psr/simple-cache": "^2.0", diff --git a/composer.lock b/composer.lock index 84ed94f024..3b77fbf257 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "4addc39de26b735c2f01cd52a3c89941", + "content-hash": "8fc1de7f18fc871e084155f3d49069cb", "packages": [ { "name": "aws/aws-crt-php", @@ -7786,6 +7786,74 @@ }, "time": "2023-12-08T03:35:17+00:00" }, + { + "name": "promphp/prometheus_client_php", + "version": "v2.12.0", + "source": { + "type": "git", + "url": "https://github.com/PromPHP/prometheus_client_php.git", + "reference": "50b70a6df4017081917e004f177a3c01cc8115db" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/PromPHP/prometheus_client_php/zipball/50b70a6df4017081917e004f177a3c01cc8115db", + "reference": "50b70a6df4017081917e004f177a3c01cc8115db", + "shasum": "" + }, + "require": { + "ext-json": "*", + "php": "^7.2|^8.0" + }, + "replace": { + "endclothing/prometheus_client_php": "*", + "jimdo/prometheus_client_php": "*", + "lkaemmerling/prometheus_client_php": "*" + }, + "require-dev": { + "guzzlehttp/guzzle": "^6.3|^7.0", + "phpstan/extension-installer": "^1.0", + "phpstan/phpstan": "^1.5.4", + "phpstan/phpstan-phpunit": "^1.1.0", + "phpstan/phpstan-strict-rules": "^1.1.0", + "phpunit/phpunit": "^9.4", + "squizlabs/php_codesniffer": "^3.6", + "symfony/polyfill-apcu": "^1.6" + }, + "suggest": { + "ext-apc": "Required if using APCu.", + "ext-pdo": "Required if using PDO.", + "ext-redis": "Required if using Redis.", + "promphp/prometheus_push_gateway_php": "An easy client for using Prometheus PushGateway.", + "symfony/polyfill-apcu": "Required if you use APCu on PHP8.0+" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0-dev" + } + }, + "autoload": { + "psr-4": { + "Prometheus\\": "src/Prometheus/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "Lukas Kämmerling", + "email": "kontakt@lukas-kaemmerling.de" + } + ], + "description": "Prometheus instrumentation library for PHP applications.", + "support": { + "issues": "https://github.com/PromPHP/prometheus_client_php/issues", + "source": "https://github.com/PromPHP/prometheus_client_php/tree/v2.12.0" + }, + "time": "2024-10-18T07:33:22+00:00" + }, { "name": "psr/cache", "version": "3.0.0", diff --git a/config/app.php b/config/app.php index a011248e48..9c1464d2a7 100644 --- a/config/app.php +++ b/config/app.php @@ -188,6 +188,7 @@ ProcessMaker\Providers\OauthMailServiceProvider::class, ProcessMaker\Providers\OpenAiServiceProvider::class, ProcessMaker\Providers\LicenseServiceProvider::class, + ProcessMaker\Providers\MetricsServiceProvider::class, ])->toArray(), 'aliases' => Facade::defaultAliases()->merge([ diff --git a/routes/web.php b/routes/web.php index e9240fc6b3..9808c4fdf6 100644 --- a/routes/web.php +++ b/routes/web.php @@ -1,6 +1,7 @@ only('index'); + +// Metrics Route +Route::get('/metrics', function () { + return response(Metrics::renderMetrics(), 200, [ + 'Content-Type' => 'text/plain; version=0.0.4', + ]); +}); \ No newline at end of file diff --git a/tests/Feature/MetricsFacadeTest.php b/tests/Feature/MetricsFacadeTest.php new file mode 100644 index 0000000000..2084b9e627 --- /dev/null +++ b/tests/Feature/MetricsFacadeTest.php @@ -0,0 +1,58 @@ +assertTrue(true); // In this point we assume that there are no errors + } + + /** + * Test to check if metrics can be rendered using the facade. + */ + public function test_facade_can_render_metrics() + { + // Register and increment a counter + Metrics::registerCounter('facade_render_test', 'Render test via facade'); + Metrics::incrementCounter('facade_render_test'); + + // Render the metrics + $output = Metrics::renderMetrics(); + + // Verify the metric in the output + $this->assertStringContainsString('facade_render_test', $output); + } +} diff --git a/tests/unit/MetricsServiceTest.php b/tests/unit/MetricsServiceTest.php new file mode 100644 index 0000000000..322c28727b --- /dev/null +++ b/tests/unit/MetricsServiceTest.php @@ -0,0 +1,61 @@ +metricsService = new MetricsService($adapter); + } + + /** + * Test to check if a counter can be registered and incremented. + * + * @return void + */ + public function test_can_register_and_increment_counter() + { + $this->metricsService->registerCounter('test_counter', 'A test counter'); + $this->metricsService->incrementCounter('test_counter'); + + $metric = $this->metricsService->getMetricByName('app_test_counter'); + $this->assertNotNull($metric); + $this->assertEquals(1, $metric->getSamples()[0]->getValue()); + } + + /** + * Test to check if metrics can be rendered. + * + * @return void + */ + public function test_can_render_metrics() + { + // Register a counter and increment it + $this->metricsService->registerCounter('render_test_counter', 'Test render metrics'); + $this->metricsService->incrementCounter('render_test_counter'); + + // Render the metrics + $output = $this->metricsService->renderMetrics(); + + // Verify that the output contains the expected metric + $this->assertStringContainsString('render_test_counter', $output); + } +} From cbeeeae8ac4d81037baa20cc00fb033e9f5ab487 Mon Sep 17 00:00:00 2001 From: Roly Gutierrez Date: Thu, 28 Nov 2024 17:02:55 -0400 Subject: [PATCH 2/4] FOUR-20535 Update composer.lock after merge release-2024-fall --- composer.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.lock b/composer.lock index 53df3d30e1..5c013c19c8 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "423bcd8f34788556a8bb495ebd3b0159", + "content-hash": "6a12ea5fbcc6771ac3957652c9793e82", "packages": [ { "name": "aws/aws-crt-php", From 5a10ad0fc33d7eff6b86c6d341eb35f61864eb2f Mon Sep 17 00:00:00 2001 From: Roly Gutierrez Date: Fri, 6 Dec 2024 17:36:22 -0400 Subject: [PATCH 3/4] FOUR-20535 Initial Setup (Laravel + Prometheus Integration) --- README.md | 223 +++++++++++++++++++++++++++++- metrics/README.md | 65 +++++++++ metrics/compose.yaml | 25 ++++ metrics/grafana/datasource.yml | 11 ++ metrics/prometheus/prometheus.yml | 40 ++++++ 5 files changed, 361 insertions(+), 3 deletions(-) create mode 100644 metrics/README.md create mode 100644 metrics/compose.yaml create mode 100644 metrics/grafana/datasource.yml create mode 100644 metrics/prometheus/prometheus.yml diff --git a/README.md b/README.md index ceab37bb8d..36931c8ad2 100644 --- a/README.md +++ b/README.md @@ -439,11 +439,228 @@ npm run dev-font ``` -# Message broker driver, possible values: rabbitmq, kafka, this is optional, if not exists or is empty, the Nayra will be work as normally with local execution -MESSAGE_BROKER_DRIVER=rabbitmq +# Install Prometheus and Grafana with Docker + +This guide explains how to install and run **Prometheus** and **Grafana** using Docker. Both tools complement each other: Prometheus collects and monitors metrics, while Grafana visualizes them with interactive dashboards. + +## Install Docker +Ensure Docker is installed on your system. Verify the installation: +```bash +docker --version +``` +If not installed, follow the [official Docker documentation](https://docs.docker.com/get-docker/). + +## Prometheus Installation + +### 1. Pull the Prometheus Docker Image +Download the official Prometheus image from Docker Hub: +```bash +docker pull prom/prometheus +``` + +### 2. Create a Prometheus Configuration File +Prometheus requires a configuration file (`prometheus.yml`) to define the metrics it will collect. Example: +```yaml +# prometheus.yml +global: + scrape_interval: 15s # Metrics collection interval + +scrape_configs: + - job_name: "prometheus" + static_configs: + - targets: ["localhost:9090"] # Monitor Prometheus itself +``` + +### 3. Run Prometheus with Docker +Use the following command to start Prometheus with the configuration file: +```bash +docker run -d \ + --name prometheus \ + -p 9090:9090 \ + -v $(pwd)/prometheus.yml:/etc/prometheus/prometheus.yml \ + prom/prometheus +``` +- `-d`: Run the container in detached mode. +- `--name`: Name of the container. +- `-p 9090:9090`: Map port 9090 of the container to port 9090 of your machine. +- `-v`: Mount the `prometheus.yml` file into the container. + +### 4. Optional: Persistent Data +To retain Prometheus data after stopping the container, use a Docker volume: +```bash +docker run -d \ + --name prometheus \ + -p 9090:9090 \ + -v $(pwd)/prometheus.yml:/etc/prometheus/prometheus.yml \ + -v prometheus_data:/prometheus \ + prom/prometheus +``` +This creates a volume named `prometheus_data` to store data. + +### 5. Access Prometheus +Open your browser and visit: +``` +http://localhost:9090 +``` +From here, you can explore collected metrics and configure custom queries. + +### 6. Container Management +- View container logs: + ```bash + docker logs prometheus + ``` +- Stop the container: + ```bash + docker stop prometheus + ``` +- Remove the container: + ```bash + docker rm prometheus + ``` + +## Grafana Installation + +### 1. Pull the Grafana Docker Image +Download the official Grafana image from Docker Hub: +```bash +docker pull grafana/grafana +``` + +### 2. Run Grafana +Run Grafana in a container: +```bash +docker run -d \ + --name grafana \ + -p 3000:3000 \ + grafana/grafana +``` +- `-d`: Run the container in detached mode. +- `--name`: Assign a name to the container. +- `-p 3000:3000`: Map port 3000 of the container to port 3000 of your machine. + +### 3. Persistent Data for Grafana +To avoid losing configurations or data after restarting the container, mount a volume: +```bash +docker run -d \ + --name grafana \ + -p 3000:3000 \ + -v grafana_data:/var/lib/grafana \ + grafana/grafana +``` +This creates a volume named `grafana_data` to store Grafana data. + +### 4. Access Grafana +Open your browser and navigate to: +``` +http://localhost:3000 +``` +Default credentials: +- **Username**: `admin` +- **Password**: `admin` + +You will be prompted to change the password on the first login. + +### 5. Integrate Grafana with Prometheus +To connect Grafana with Prometheus: +1. Open **Settings** in Grafana. +2. Select **Data Sources**. +3. Click **Add data source**. +4. Choose **Prometheus** as the data source. +5. Set the Prometheus Service URL (e.g., `http://localhost:9090`). +6. If Prometheus and Grafana run on the same machine: + - Use `http://host.docker.internal:9090` (for Docker Desktop). + - Use `http://:9090`. +7. If they share the same network (e.g., Docker Compose): + - Use the container name as the host: `http://prometheus:9090`. +8. Click **Save & Test** to verify the connection. + +### 6. Container Management +- View container logs: + ```bash + docker logs grafana + ``` +- Restart the container: + ```bash + docker restart grafana + ``` +- Stop the container: + ```bash + docker stop grafana + ``` +- Remove the container: + ```bash + docker rm grafana + ``` + +### 7. Install Optional Plugins +To install plugins in Grafana, use environment variables when starting the container: +```bash +docker run -d \ + --name grafana \ + -p 3000:3000 \ + -v grafana_data:/var/lib/grafana \ + -e "GF_INSTALL_PLUGINS=grafana-piechart-panel,grafana-clock-panel" \ + grafana/grafana +``` + +## Examples of Using Grafana in ProcessMaker + +### **Configure Prometheus With Your Application** + +Add the `/metrics` endpoint from your ProcessMaker application to the `metrics/prometheus.yml` file, make the following change: + +```yaml +scrape_configs: + - job_name: 'laravel_app' + static_configs: + - targets: ['https://processmaker.net:8000'] # host and port of application +``` + +Restart Prometheus: + +```bash +docker restart prometheus +``` + +### **Use the Facade in Your Application** + +Now you can use the `Metrics` Facade anywhere in your application to manage metrics. + +#### **Example: Incrementing a Counter** + +In a controller: + +```php +namespace App\Http\Controllers; + +use ProcessMaker\Facades\Metrics; + +class ExampleController extends Controller +{ + public function index() + { + Metrics::registerCounter('requests_total', 'Total number of requests', ['method']); + Metrics::incrementCounter('requests_total', ['GET']); + + return response()->json(['message' => 'Hello, world!']); + } +} +``` + +#### **Example: Registering and Using a Histogram** + +```php +Metrics::registerHistogram( + 'request_duration_seconds', + 'Request duration time', + ['method'], + [0.1, 0.5, 1, 5] +); +$histogram = Metrics::incrementHistogram('request_duration_seconds', ['GET'], microtime(true) - LARAVEL_START); +``` -#### License +# License Distributed under the [AGPL Version 3](https://www.gnu.org/licenses/agpl-3.0.en.html) diff --git a/metrics/README.md b/metrics/README.md new file mode 100644 index 0000000000..4a13b6f35b --- /dev/null +++ b/metrics/README.md @@ -0,0 +1,65 @@ +## Compose sample +### Prometheus & Grafana + +Project structure: +``` +. +├── compose.yaml +├── grafana +│   └── datasource.yml +├── prometheus +│   └── prometheus.yml +└── README.md +``` + +[_compose.yaml_](compose.yaml) +``` +services: + prometheus: + image: prom/prometheus + ... + ports: + - 9090:9090 + grafana: + image: grafana/grafana + ... + ports: + - 3000:3000 +``` +The compose file defines a stack with two services `prometheus` and `grafana`. +When deploying the stack, docker compose maps port the default ports for each service to the equivalent ports on the host in order to inspect easier the web interface of each service. +Make sure the ports 9090 and 3000 on the host are not already in use. + +## Deploy with docker compose + +``` +$ docker compose up -d +Creating network "prometheus-grafana_default" with the default driver +Creating volume "prometheus-grafana_prom_data" with default driver +... +Creating grafana ... done +Creating prometheus ... done +Attaching to prometheus, grafana + +``` + +## Expected result + +Listing containers must show two containers running and the port mapping as below: +``` +$ docker ps +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +dbdec637814f prom/prometheus "/bin/prometheus --c…" 8 minutes ago Up 8 minutes 0.0.0.0:9090->9090/tcp prometheus +79f667cb7dc2 grafana/grafana "/run.sh" 8 minutes ago Up 8 minutes 0.0.0.0:3000->3000/tcp grafana +``` + +Navigate to `http://localhost:3000` in your web browser and use the login credentials specified in the compose file to access Grafana. It is already configured with prometheus as the default datasource. + +![page] + +Navigate to `http://localhost:9090` in your web browser to access directly the web interface of prometheus. + +Stop and remove the containers. Use `-v` to remove the volumes if looking to erase all data. +``` +$ docker compose down -v +``` diff --git a/metrics/compose.yaml b/metrics/compose.yaml new file mode 100644 index 0000000000..24369bc3ed --- /dev/null +++ b/metrics/compose.yaml @@ -0,0 +1,25 @@ +services: + prometheus: + image: prom/prometheus + container_name: prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + ports: + - 9090:9090 + restart: unless-stopped + volumes: + - ./prometheus:/etc/prometheus + - prom_data:/prometheus + grafana: + image: grafana/grafana + container_name: grafana + ports: + - 3000:3000 + restart: unless-stopped + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=grafana + volumes: + - ./grafana:/etc/grafana/provisioning/datasources +volumes: + prom_data: diff --git a/metrics/grafana/datasource.yml b/metrics/grafana/datasource.yml new file mode 100644 index 0000000000..cbde3cccf6 --- /dev/null +++ b/metrics/grafana/datasource.yml @@ -0,0 +1,11 @@ +# Global configuration settings for Grafana +apiVersion: 1 + +# Grafana datasource configuration for Prometheus +datasources: +- name: Prometheus + type: prometheus + url: http://prometheus:9090 + isDefault: true + access: proxy + editable: true diff --git a/metrics/prometheus/prometheus.yml b/metrics/prometheus/prometheus.yml new file mode 100644 index 0000000000..b8d1b8ac4e --- /dev/null +++ b/metrics/prometheus/prometheus.yml @@ -0,0 +1,40 @@ +# Global configuration settings for Prometheus +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s + +# Scrape configurations +scrape_configs: + # For Prometheus + - job_name: prometheus + honor_timestamps: true + scrape_interval: 15s + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + static_configs: + - targets: + - localhost:9090 + + # For ProcessMaker https://127.0.0.5:8092/metrics + - job_name: processmaker + honor_timestamps: true + scrape_interval: 15s + scrape_timeout: 10s + metrics_path: /metrics + scheme: https + tls_config: + insecure_skip_verify: true # Ignore the certificate validation (for development only, remove for production). + static_configs: + - targets: + - 127.0.0.5:8092 # Replace with the address of your application. + +# Alerting configuration for Prometheus. +#alerting: +# alertmanagers: +# - static_configs: +# - targets: [] +# scheme: http +# timeout: 10s +# api_version: v1 From 6760384a0edff939e8ace61799daddd58b78a8e1 Mon Sep 17 00:00:00 2001 From: Roly Gutierrez Date: Mon, 9 Dec 2024 10:55:04 -0400 Subject: [PATCH 4/4] FOUR-20535 Security issues fixed. --- metrics/compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metrics/compose.yaml b/metrics/compose.yaml index 24369bc3ed..ed204fe6b2 100644 --- a/metrics/compose.yaml +++ b/metrics/compose.yaml @@ -18,7 +18,7 @@ services: restart: unless-stopped environment: - GF_SECURITY_ADMIN_USER=admin - - GF_SECURITY_ADMIN_PASSWORD=grafana + - GF_SECURITY_ADMIN_PASSWORD= volumes: - ./grafana:/etc/grafana/provisioning/datasources volumes: