From 67d5f42d2237ce1783d4f8c2950ff3b54d4154c2 Mon Sep 17 00:00:00 2001 From: khaliiiiiil Date: Thu, 5 Dec 2024 15:31:52 +0100 Subject: [PATCH] Commit PULUMI-DOCKER final version --- TO CONFIGURE.txt | 41 +++++++++++++++++++++++++++++ config.json | 34 +++++++++--------------- exporter.Dockerfile | 35 +++++++++++++++++-------- fluent.conf | 17 ++++++------ logs/odoo/odoo.log | 6 +++++ odoo.conf | 1 + pg_metrics_exporter.py | 59 ++++++++++++++++++++++++++++++++++++++++++ prometheus.yml | 9 +++---- queries.yaml | 15 ----------- 9 files changed, 154 insertions(+), 63 deletions(-) create mode 100644 TO CONFIGURE.txt create mode 100644 logs/odoo/odoo.log create mode 100644 pg_metrics_exporter.py delete mode 100644 queries.yaml diff --git a/TO CONFIGURE.txt b/TO CONFIGURE.txt new file mode 100644 index 0000000..66b71f5 --- /dev/null +++ b/TO CONFIGURE.txt @@ -0,0 +1,41 @@ +before running pulumi up build the images : +docker build -f odoo.Dockerfile -t odoo_custom . +docker build -f backup.Dockerfile -t exporter_custom . +docker build -f exporter.Dockerfile -t exporter_custom . + + + + +after running the pulumi up in the postgres container terminal do that : +TO CONFIGURE DATABASE pg_stat_statements : +psql -h admin -U admin -d admin +CREATE EXTENSION pg_stat_statements; +exit + +apt-get update && apt-get install nano -y +nano /var/lib/postgresql/data/postgresql.conf +shared_preload_libraries = 'pg_stat_statements' + +compute_query_id = on +pg_stat_statements.max = 10000 +pg_stat_statements.track = all + +restart + + +TO TEST pg_stat_statements : +psql -h admin -U admin -d admin +SELECT * FROM pg_stat_statements; +SELECT query, calls, total_exec_time, rows, 100.0 * shared_blks_hit / + nullif(shared_blks_hit + shared_blks_read, 0) AS hit_percent + FROM pg_stat_statements ORDER BY total_exec_time DESC LIMIT 5; + +TO TEST PROMETHEUS SCRAPING : +visit http://localhost:9090/targets + + + + + +FOR GRAFANA : DISPLAY SELECT QUERIES PER TIME : +sum(increase(postgresql_query_calls{query=~"SELECT.*"}[5m])) diff --git a/config.json b/config.json index 821df98..ee4428d 100644 --- a/config.json +++ b/config.json @@ -28,29 +28,11 @@ "ports": [{"internal": 80, "external": 5050}] }, { - "name": "postgres-exporter", - "image": "wrouesnel/postgres_exporter:latest", - "envs": { - "DATA_SOURCE_NAME": "postgresql://admin:admin@admin:5432/admin?sslmode=disable" - }, + "name": "prometheus_exporter", + "image": "exporter_custom", "network_mode": "testNetwork", - "ports": [ - { - "internal": 9187, - "external": 9187 - } - ], - "volumes": [ - { - "host_path": "./queries.yaml", - "container_path": "/etc/postgres_exporter/queries.yaml" - } - ], - "command": [ - "--extend.query-path=/etc/postgres_exporter/queries.yaml", - "--log.level=debug" - ] - }, + "ports": [{"internal": 8000, "external": 8000}] + }, { "name": "odoo", "image": "odoo_custom", @@ -67,6 +49,10 @@ { "host_path": "./odoo.conf", "container_path": "/etc/odoo/odoo.conf" + }, + { + "container_path": "/var/log/odoo", + "host_path": "./logs/odoo" } ] }, @@ -103,6 +89,10 @@ "network_mode": "testNetwork", "ports": [{"internal": 24224, "external": 24224}], "volumes": [ + { + "container_path": "/var/log/odoo", + "host_path": "./logs/odoo" + }, { "container_path": "/fluentd/etc/fluent.conf", "host_path": "./fluent.conf" diff --git a/exporter.Dockerfile b/exporter.Dockerfile index 71112a2..4d577ec 100644 --- a/exporter.Dockerfile +++ b/exporter.Dockerfile @@ -1,14 +1,27 @@ -# Base image: postgres-exporter -FROM wrouesnel/postgres_exporter:latest as exporter_custom +# Use a Python slim image as base +FROM python:3.9-slim as exporter_custom -# Add your custom queries.yaml to the image -COPY queries.yaml /etc/postgres_exporter/queries.yaml +# Set environment variables to avoid buffering +ENV PYTHONUNBUFFERED=1 -# Expose the default port -EXPOSE 9187 +# Create and set working directory +WORKDIR /app -# Command to run the exporter with the custom query path -CMD [ \ - "--extend.query-path=/etc/postgres_exporter/queries.yaml", \ - "--log.level=debug" \ -] +# Install required dependencies +RUN apt-get update && \ + apt-get install -y \ + libpq-dev && \ + pip install psycopg2-binary prometheus-client && \ + apt-get clean + +# Create a directory for logs +RUN mkdir /app/logs + +# Copy the Python script into the container +COPY pg_metrics_exporter.py /app/ + +# Set permissions for log directory (if required) +RUN chmod 755 /app/logs + +# Run the script and redirect logs to a file +CMD ["python", "/app/pg_metrics_exporter.py", ">", "/app/logs/exporter.log", "2>&1"] diff --git a/fluent.conf b/fluent.conf index 39b2e17..53d1bb1 100644 --- a/fluent.conf +++ b/fluent.conf @@ -1,14 +1,13 @@ - @type forward - port 8069 # Odoo logs + @type tail + path "/var/log/odoo/odoo.log" + pos_file "/fluentd/logs/odoo.pos" + format none + tag "odoo.log" - - @type forward - port 3000 # Grafana logs - - - + @type file - path /fluentd/logs/collected-logs + path "/fluentd/logs/collected-logs" + diff --git a/logs/odoo/odoo.log b/logs/odoo/odoo.log new file mode 100644 index 0000000..1d50903 --- /dev/null +++ b/logs/odoo/odoo.log @@ -0,0 +1,6 @@ +2024-12-05 14:21:14,148 8 INFO ? odoo: Odoo version 18.0-20241108 +2024-12-05 14:21:14,151 8 INFO ? odoo: Using configuration file at /etc/odoo/odoo.conf +2024-12-05 14:21:14,152 8 INFO ? odoo: addons paths: ['/usr/lib/python3/dist-packages/odoo/addons', '/root/.local/share/Odoo/addons/18.0'] +2024-12-05 14:21:14,154 8 INFO ? odoo: database: admin@admin:5432 +2024-12-05 14:21:14,793 8 INFO ? odoo.addons.base.models.ir_actions_report: Will use the Wkhtmltopdf binary at /usr/local/bin/wkhtmltopdf +2024-12-05 14:21:14,814 8 INFO ? odoo.addons.base.models.ir_actions_report: Will use the Wkhtmltoimage binary at /usr/local/bin/wkhtmltoimage diff --git a/odoo.conf b/odoo.conf index 02d3fab..c9b9d81 100644 --- a/odoo.conf +++ b/odoo.conf @@ -5,3 +5,4 @@ db_user = admin db_password = admin default_productivity_apps = True db_name = admin +logfile = /var/log/odoo/odoo.log diff --git a/pg_metrics_exporter.py b/pg_metrics_exporter.py new file mode 100644 index 0000000..ea4e692 --- /dev/null +++ b/pg_metrics_exporter.py @@ -0,0 +1,59 @@ +import psycopg2 +from prometheus_client import start_http_server, Gauge +import time + +# Configuration for database connection +DB_PARAMS = { + "host": "admin", + "database": "admin", + "user": "admin", + "password": "admin" +} + +# Prometheus metrics +QUERY_CALLS = Gauge('postgresql_query_calls', 'Number of calls for each query', ['query']) +QUERY_TOTAL_TIME = Gauge('postgresql_query_total_time_ms', 'Total execution time for each query in ms', ['query']) + +def fetch_metrics(): + try: + # Log connection attempt + print("Connecting to PostgreSQL database...") + + conn = psycopg2.connect(**DB_PARAMS) + cur = conn.cursor() + + # Execute query to get data + cur.execute(""" + SELECT query, calls, total_exec_time + FROM pg_stat_statements + ORDER BY total_exec_time DESC; + """) + + # Iterate through results and set Prometheus metrics + for row in cur: + query = row[0].replace("\\", "\\\\").replace('"', '\\"') # Escape special characters + calls = row[1] + total_time = row[2] * 1000 # Convert seconds to milliseconds + + QUERY_CALLS.labels(query=query).set(calls) + QUERY_TOTAL_TIME.labels(query=query).set(total_time) + + # Log the metrics being set + print(f"Metrics set for query: {query} | Calls: {calls} | Total execution time: {total_time} ms") + + cur.close() + conn.close() + except psycopg2.Error as e: + print(f"Error fetching data: {e}") + except Exception as e: + print(f"Unexpected error: {e}") + +if __name__ == '__main__': + # Start Prometheus HTTP server on port 8000 + start_http_server(8000) + print("Exporter running on http://localhost:8000/metrics") + + # Main loop to fetch metrics at regular intervals + while True: + fetch_metrics() + time.sleep(60) # Scrape every 60 seconds diff --git a/prometheus.yml b/prometheus.yml index 7f8b54e..14ffc90 100644 --- a/prometheus.yml +++ b/prometheus.yml @@ -1,8 +1,5 @@ scrape_configs: - - job_name: 'postgres-exporter' + - job_name: 'pg_stat_statements' static_configs: - - targets: ['postgres-exporter:9187'] - - - job_name: 'odoo' - static_configs: - - targets: ['localhost:8069'] # Idem pour Odoo + - targets: ['prometheus_exporter:8000'] + scrape_interval: 15s diff --git a/queries.yaml b/queries.yaml deleted file mode 100644 index 3c36386..0000000 --- a/queries.yaml +++ /dev/null @@ -1,15 +0,0 @@ -pg_stat_statements: - query: "SELECT query, calls, total_exec_time FROM pg_stat_statements" - metrics: - - name: "pg_stat_statements_total_exec_time" - help: "Total execution time of the query" - type: "counter" - value: "total_exec_time" - labels: - - "query" - - name: "pg_stat_statements_calls" - help: "Total number of calls for the query" - type: "counter" - value: "calls" - labels: - - "query"