diff --git a/TO CONFIGURE.txt b/TO CONFIGURE.txt
new file mode 100644
index 0000000..66b71f5
--- /dev/null
+++ b/TO CONFIGURE.txt
@@ -0,0 +1,41 @@
+before running pulumi up build the images :
+docker build -f odoo.Dockerfile -t odoo_custom .
+docker build -f backup.Dockerfile -t exporter_custom .
+docker build -f exporter.Dockerfile -t exporter_custom .
+
+
+
+
+after running the pulumi up in the postgres container terminal do that :
+TO CONFIGURE DATABASE pg_stat_statements :
+psql -h admin -U admin -d admin
+CREATE EXTENSION pg_stat_statements;
+exit
+
+apt-get update && apt-get install nano -y
+nano /var/lib/postgresql/data/postgresql.conf
+shared_preload_libraries = 'pg_stat_statements'
+
+compute_query_id = on
+pg_stat_statements.max = 10000
+pg_stat_statements.track = all
+
+restart
+
+
+TO TEST pg_stat_statements :
+psql -h admin -U admin -d admin
+SELECT * FROM pg_stat_statements;
+SELECT query, calls, total_exec_time, rows, 100.0 * shared_blks_hit /
+ nullif(shared_blks_hit + shared_blks_read, 0) AS hit_percent
+ FROM pg_stat_statements ORDER BY total_exec_time DESC LIMIT 5;
+
+TO TEST PROMETHEUS SCRAPING :
+visit http://localhost:9090/targets
+
+
+
+
+
+FOR GRAFANA : DISPLAY SELECT QUERIES PER TIME :
+sum(increase(postgresql_query_calls{query=~"SELECT.*"}[5m]))
diff --git a/config.json b/config.json
index 821df98..ee4428d 100644
--- a/config.json
+++ b/config.json
@@ -28,29 +28,11 @@
"ports": [{"internal": 80, "external": 5050}]
},
{
- "name": "postgres-exporter",
- "image": "wrouesnel/postgres_exporter:latest",
- "envs": {
- "DATA_SOURCE_NAME": "postgresql://admin:admin@admin:5432/admin?sslmode=disable"
- },
+ "name": "prometheus_exporter",
+ "image": "exporter_custom",
"network_mode": "testNetwork",
- "ports": [
- {
- "internal": 9187,
- "external": 9187
- }
- ],
- "volumes": [
- {
- "host_path": "./queries.yaml",
- "container_path": "/etc/postgres_exporter/queries.yaml"
- }
- ],
- "command": [
- "--extend.query-path=/etc/postgres_exporter/queries.yaml",
- "--log.level=debug"
- ]
- },
+ "ports": [{"internal": 8000, "external": 8000}]
+ },
{
"name": "odoo",
"image": "odoo_custom",
@@ -67,6 +49,10 @@
{
"host_path": "./odoo.conf",
"container_path": "/etc/odoo/odoo.conf"
+ },
+ {
+ "container_path": "/var/log/odoo",
+ "host_path": "./logs/odoo"
}
]
},
@@ -103,6 +89,10 @@
"network_mode": "testNetwork",
"ports": [{"internal": 24224, "external": 24224}],
"volumes": [
+ {
+ "container_path": "/var/log/odoo",
+ "host_path": "./logs/odoo"
+ },
{
"container_path": "/fluentd/etc/fluent.conf",
"host_path": "./fluent.conf"
diff --git a/exporter.Dockerfile b/exporter.Dockerfile
index 71112a2..4d577ec 100644
--- a/exporter.Dockerfile
+++ b/exporter.Dockerfile
@@ -1,14 +1,27 @@
-# Base image: postgres-exporter
-FROM wrouesnel/postgres_exporter:latest as exporter_custom
+# Use a Python slim image as base
+FROM python:3.9-slim as exporter_custom
-# Add your custom queries.yaml to the image
-COPY queries.yaml /etc/postgres_exporter/queries.yaml
+# Set environment variables to avoid buffering
+ENV PYTHONUNBUFFERED=1
-# Expose the default port
-EXPOSE 9187
+# Create and set working directory
+WORKDIR /app
-# Command to run the exporter with the custom query path
-CMD [ \
- "--extend.query-path=/etc/postgres_exporter/queries.yaml", \
- "--log.level=debug" \
-]
+# Install required dependencies
+RUN apt-get update && \
+ apt-get install -y \
+ libpq-dev && \
+ pip install psycopg2-binary prometheus-client && \
+ apt-get clean
+
+# Create a directory for logs
+RUN mkdir /app/logs
+
+# Copy the Python script into the container
+COPY pg_metrics_exporter.py /app/
+
+# Set permissions for log directory (if required)
+RUN chmod 755 /app/logs
+
+# Run the script and redirect logs to a file
+CMD ["python", "/app/pg_metrics_exporter.py", ">", "/app/logs/exporter.log", "2>&1"]
diff --git a/fluent.conf b/fluent.conf
index 39b2e17..53d1bb1 100644
--- a/fluent.conf
+++ b/fluent.conf
@@ -1,14 +1,13 @@
- @type forward
- port 8069 # Odoo logs
+ @type tail
+ path "/var/log/odoo/odoo.log"
+ pos_file "/fluentd/logs/odoo.pos"
+ format none
+ tag "odoo.log"
-
- @type forward
- port 3000 # Grafana logs
-
-
-
+
@type file
- path /fluentd/logs/collected-logs
+ path "/fluentd/logs/collected-logs"
+
diff --git a/logs/odoo/odoo.log b/logs/odoo/odoo.log
new file mode 100644
index 0000000..1d50903
--- /dev/null
+++ b/logs/odoo/odoo.log
@@ -0,0 +1,6 @@
+2024-12-05 14:21:14,148 8 INFO ? odoo: Odoo version 18.0-20241108
+2024-12-05 14:21:14,151 8 INFO ? odoo: Using configuration file at /etc/odoo/odoo.conf
+2024-12-05 14:21:14,152 8 INFO ? odoo: addons paths: ['/usr/lib/python3/dist-packages/odoo/addons', '/root/.local/share/Odoo/addons/18.0']
+2024-12-05 14:21:14,154 8 INFO ? odoo: database: admin@admin:5432
+2024-12-05 14:21:14,793 8 INFO ? odoo.addons.base.models.ir_actions_report: Will use the Wkhtmltopdf binary at /usr/local/bin/wkhtmltopdf
+2024-12-05 14:21:14,814 8 INFO ? odoo.addons.base.models.ir_actions_report: Will use the Wkhtmltoimage binary at /usr/local/bin/wkhtmltoimage
diff --git a/odoo.conf b/odoo.conf
index 02d3fab..c9b9d81 100644
--- a/odoo.conf
+++ b/odoo.conf
@@ -5,3 +5,4 @@ db_user = admin
db_password = admin
default_productivity_apps = True
db_name = admin
+logfile = /var/log/odoo/odoo.log
diff --git a/pg_metrics_exporter.py b/pg_metrics_exporter.py
new file mode 100644
index 0000000..ea4e692
--- /dev/null
+++ b/pg_metrics_exporter.py
@@ -0,0 +1,59 @@
+import psycopg2
+from prometheus_client import start_http_server, Gauge
+import time
+
+# Configuration for database connection
+DB_PARAMS = {
+ "host": "admin",
+ "database": "admin",
+ "user": "admin",
+ "password": "admin"
+}
+
+# Prometheus metrics
+QUERY_CALLS = Gauge('postgresql_query_calls', 'Number of calls for each query', ['query'])
+QUERY_TOTAL_TIME = Gauge('postgresql_query_total_time_ms', 'Total execution time for each query in ms', ['query'])
+
+def fetch_metrics():
+ try:
+ # Log connection attempt
+ print("Connecting to PostgreSQL database...")
+
+ conn = psycopg2.connect(**DB_PARAMS)
+ cur = conn.cursor()
+
+ # Execute query to get data
+ cur.execute("""
+ SELECT query, calls, total_exec_time
+ FROM pg_stat_statements
+ ORDER BY total_exec_time DESC;
+ """)
+
+ # Iterate through results and set Prometheus metrics
+ for row in cur:
+ query = row[0].replace("\\", "\\\\").replace('"', '\\"') # Escape special characters
+ calls = row[1]
+ total_time = row[2] * 1000 # Convert seconds to milliseconds
+
+ QUERY_CALLS.labels(query=query).set(calls)
+ QUERY_TOTAL_TIME.labels(query=query).set(total_time)
+
+ # Log the metrics being set
+ print(f"Metrics set for query: {query} | Calls: {calls} | Total execution time: {total_time} ms")
+
+ cur.close()
+ conn.close()
+ except psycopg2.Error as e:
+ print(f"Error fetching data: {e}")
+ except Exception as e:
+ print(f"Unexpected error: {e}")
+
+if __name__ == '__main__':
+ # Start Prometheus HTTP server on port 8000
+ start_http_server(8000)
+ print("Exporter running on http://localhost:8000/metrics")
+
+ # Main loop to fetch metrics at regular intervals
+ while True:
+ fetch_metrics()
+ time.sleep(60) # Scrape every 60 seconds
diff --git a/prometheus.yml b/prometheus.yml
index 7f8b54e..14ffc90 100644
--- a/prometheus.yml
+++ b/prometheus.yml
@@ -1,8 +1,5 @@
scrape_configs:
- - job_name: 'postgres-exporter'
+ - job_name: 'pg_stat_statements'
static_configs:
- - targets: ['postgres-exporter:9187']
-
- - job_name: 'odoo'
- static_configs:
- - targets: ['localhost:8069'] # Idem pour Odoo
+ - targets: ['prometheus_exporter:8000']
+ scrape_interval: 15s
diff --git a/queries.yaml b/queries.yaml
deleted file mode 100644
index 3c36386..0000000
--- a/queries.yaml
+++ /dev/null
@@ -1,15 +0,0 @@
-pg_stat_statements:
- query: "SELECT query, calls, total_exec_time FROM pg_stat_statements"
- metrics:
- - name: "pg_stat_statements_total_exec_time"
- help: "Total execution time of the query"
- type: "counter"
- value: "total_exec_time"
- labels:
- - "query"
- - name: "pg_stat_statements_calls"
- help: "Total number of calls for the query"
- type: "counter"
- value: "calls"
- labels:
- - "query"