Commit PULUMI-DOCKER final version

TO CONFIGURE.txt

@@ -0,0 +1,41 @@
+before running pulumi up build the images :
+docker build -f odoo.Dockerfile -t odoo_custom .
+docker build -f backup.Dockerfile -t exporter_custom .
+docker build -f exporter.Dockerfile -t exporter_custom .
+
+
+
+
+after running the pulumi up in the postgres container terminal do that :
+TO CONFIGURE DATABASE pg_stat_statements :
+psql -h admin -U admin -d admin
+CREATE EXTENSION pg_stat_statements;
+exit
+
+apt-get update && apt-get install nano -y
+nano /var/lib/postgresql/data/postgresql.conf
+shared_preload_libraries = 'pg_stat_statements'
+
+compute_query_id = on
+pg_stat_statements.max = 10000
+pg_stat_statements.track = all
+
+restart
+
+
+TO TEST pg_stat_statements :
+psql -h admin -U admin -d admin
+SELECT * FROM pg_stat_statements;
+SELECT query, calls, total_exec_time, rows, 100.0 * shared_blks_hit /
+               nullif(shared_blks_hit + shared_blks_read, 0) AS hit_percent
+          FROM pg_stat_statements ORDER BY total_exec_time DESC LIMIT 5;
+
+TO TEST PROMETHEUS SCRAPING :
+visit http://localhost:9090/targets
+
+
+
+
+
+FOR GRAFANA : DISPLAY SELECT QUERIES PER TIME :
+sum(increase(postgresql_query_calls{query=~"SELECT.*"}[5m]))

config.json

@@ -28,29 +28,11 @@
         "ports": [{"internal": 80, "external": 5050}]
       },
       {
-        "name": "postgres-exporter",
-        "image": "wrouesnel/postgres_exporter:latest",
-        "envs": {
-          "DATA_SOURCE_NAME": "postgresql://admin:admin@admin:5432/admin?sslmode=disable"
-        },
+        "name": "prometheus_exporter",
+        "image": "exporter_custom",
         "network_mode": "testNetwork",
-        "ports": [
-          {
-            "internal": 9187,
-            "external": 9187
-          }
-        ],
-        "volumes": [
-          {
-            "host_path": "./queries.yaml",
-            "container_path": "/etc/postgres_exporter/queries.yaml"
-          }
-        ],
-        "command": [
-          "--extend.query-path=/etc/postgres_exporter/queries.yaml",
-          "--log.level=debug"
-        ]
-      },           
+        "ports": [{"internal": 8000, "external": 8000}]
+      },    
       {
         "name": "odoo",
         "image": "odoo_custom",
@@ -67,6 +49,10 @@
           {
             "host_path": "./odoo.conf",
             "container_path": "/etc/odoo/odoo.conf"
+          },
+          {
+            "container_path": "/var/log/odoo",
+            "host_path": "./logs/odoo"
           }
         ]
       },
@@ -103,6 +89,10 @@
         "network_mode": "testNetwork",
         "ports": [{"internal": 24224, "external": 24224}],
         "volumes": [
+          {
+            "container_path": "/var/log/odoo",
+            "host_path": "./logs/odoo"
+          },
           {
             "container_path": "/fluentd/etc/fluent.conf",
             "host_path": "./fluent.conf"

exporter.Dockerfile

@@ -1,14 +1,27 @@
-# Base image: postgres-exporter
-FROM wrouesnel/postgres_exporter:latest as exporter_custom
+# Use a Python slim image as base
+FROM python:3.9-slim as exporter_custom
 
-# Add your custom queries.yaml to the image
-COPY queries.yaml /etc/postgres_exporter/queries.yaml
+# Set environment variables to avoid buffering
+ENV PYTHONUNBUFFERED=1
 
-# Expose the default port
-EXPOSE 9187
+# Create and set working directory
+WORKDIR /app
 
-# Command to run the exporter with the custom query path
-CMD [ \
-  "--extend.query-path=/etc/postgres_exporter/queries.yaml", \
-  "--log.level=debug" \
-]
+# Install required dependencies
+RUN apt-get update && \
+    apt-get install -y \
+    libpq-dev && \
+    pip install psycopg2-binary prometheus-client && \
+    apt-get clean
+
+# Create a directory for logs
+RUN mkdir /app/logs
+
+# Copy the Python script into the container
+COPY pg_metrics_exporter.py /app/
+
+# Set permissions for log directory (if required)
+RUN chmod 755 /app/logs
+
+# Run the script and redirect logs to a file
+CMD ["python", "/app/pg_metrics_exporter.py", ">", "/app/logs/exporter.log", "2>&1"]

fluent.conf

@@ -1,14 +1,13 @@
 <source>
-  @type forward
-  port 8069  # Odoo logs
+  @type tail
+  path "/var/log/odoo/odoo.log"
+  pos_file "/fluentd/logs/odoo.pos"
+  format none
+  tag "odoo.log"
 </source>
 
-<source>
-  @type forward
-  port 3000  # Grafana logs
-</source>
-
-<match *>
+<match odoo.log>
   @type file
-  path /fluentd/logs/collected-logs
+  path "/fluentd/logs/collected-logs"
 </match>
+

logs/odoo/odoo.log

@@ -0,0 +1,6 @@
+2024-12-05 14:21:14,148 8 INFO ? odoo: Odoo version 18.0-20241108 
+2024-12-05 14:21:14,151 8 INFO ? odoo: Using configuration file at /etc/odoo/odoo.conf 
+2024-12-05 14:21:14,152 8 INFO ? odoo: addons paths: ['/usr/lib/python3/dist-packages/odoo/addons', '/root/.local/share/Odoo/addons/18.0'] 
+2024-12-05 14:21:14,154 8 INFO ? odoo: database: admin@admin:5432 
+2024-12-05 14:21:14,793 8 INFO ? odoo.addons.base.models.ir_actions_report: Will use the Wkhtmltopdf binary at /usr/local/bin/wkhtmltopdf 
+2024-12-05 14:21:14,814 8 INFO ? odoo.addons.base.models.ir_actions_report: Will use the Wkhtmltoimage binary at /usr/local/bin/wkhtmltoimage 

odoo.conf

@@ -5,3 +5,4 @@ db_user = admin
 db_password = admin
 default_productivity_apps = True
 db_name = admin
+logfile = /var/log/odoo/odoo.log

pg_metrics_exporter.py

@@ -0,0 +1,59 @@
+import psycopg2
+from prometheus_client import start_http_server, Gauge
+import time
+
+# Configuration for database connection
+DB_PARAMS = {
+    "host": "admin",
+    "database": "admin",
+    "user": "admin",
+    "password": "admin"
+}
+
+# Prometheus metrics
+QUERY_CALLS = Gauge('postgresql_query_calls', 'Number of calls for each query', ['query'])
+QUERY_TOTAL_TIME = Gauge('postgresql_query_total_time_ms', 'Total execution time for each query in ms', ['query'])
+
+def fetch_metrics():
+    try:
+        # Log connection attempt
+        print("Connecting to PostgreSQL database...")
+
+        conn = psycopg2.connect(**DB_PARAMS)
+        cur = conn.cursor()
+
+        # Execute query to get data
+        cur.execute("""
+            SELECT query, calls, total_exec_time
+            FROM pg_stat_statements
+            ORDER BY total_exec_time DESC;
+        """)
+
+        # Iterate through results and set Prometheus metrics
+        for row in cur:
+            query = row[0].replace("\\", "\\\\").replace('"', '\\"')  # Escape special characters
+            calls = row[1]
+            total_time = row[2] * 1000  # Convert seconds to milliseconds
+
+            QUERY_CALLS.labels(query=query).set(calls)
+            QUERY_TOTAL_TIME.labels(query=query).set(total_time)
+
+            # Log the metrics being set
+            print(f"Metrics set for query: {query} | Calls: {calls} | Total execution time: {total_time} ms")
+
+        cur.close()
+        conn.close()
+    except psycopg2.Error as e:
+        print(f"Error fetching data: {e}")
+    except Exception as e:
+        print(f"Unexpected error: {e}")
+
+if __name__ == '__main__':
+    # Start Prometheus HTTP server on port 8000
+    start_http_server(8000)
+    print("Exporter running on http://localhost:8000/metrics")
+
+    # Main loop to fetch metrics at regular intervals
+    while True:
+        fetch_metrics()
+        time.sleep(60)  # Scrape every 60 seconds

prometheus.yml

@@ -1,8 +1,5 @@
 scrape_configs:
-  - job_name: 'postgres-exporter'
+  - job_name: 'pg_stat_statements'
     static_configs:
-      - targets: ['postgres-exporter:9187']
-
-  - job_name: 'odoo'
-    static_configs:
-      - targets: ['localhost:8069']  # Idem pour Odoo
+      - targets: ['prometheus_exporter:8000']
+    scrape_interval: 15s

queries.yaml

@@ -1,15 +0,0 @@
-pg_stat_statements:
-  query: "SELECT query, calls, total_exec_time FROM pg_stat_statements"
-  metrics:
-    - name: "pg_stat_statements_total_exec_time"
-      help: "Total execution time of the query"
-      type: "counter"
-      value: "total_exec_time"
-      labels:
-        - "query"
-    - name: "pg_stat_statements_calls"
-      help: "Total number of calls for the query"
-      type: "counter"
-      value: "calls"
-      labels:
-        - "query"