feat: Add skyline-nginx package

1. Add skyline-nginx package for managing and generating nginx
   configuration.

Change-Id: I68afd9ab9ad52fc96c13e745cb1e89a8061a53ba

libs/skyline-nginx/Makefile

@@ -0,0 +1,41 @@
+PYTHON ?= python3
+
+
+.PHONY: all
+all: install fmt lint test package
+
+
+.PHONY: venv
+venv:
+	poetry env use $(PYTHON)
+
+
+.PHONY: install
+install: venv
+	poetry run pip install -U pip setuptools
+	poetry install -vvv
+
+
+.PHONY: package
+package:
+	poetry build -f wheel
+
+
+.PHONY: fmt
+fmt:
+	poetry run isort $$(git ls-files -- **/*.py)
+	poetry run black --config ../../pyproject.toml $$(git ls-files -- **/*.py)
+	poetry run add-trailing-comma --py36-plus --exit-zero-even-if-changed $$(git ls-files -- **/*.py)
+
+
+.PHONY: lint
+lint:
+	poetry run mypy --no-incremental $$(git ls-files -- **/*.py)
+	poetry run isort --check-only --diff  $$(git ls-files -- **/*.py)
+	poetry run black --check --diff --color --config ../../pyproject.toml $$(git ls-files -- **/*.py)
+	poetry run flake8  $$(git ls-files -- **/*.py)
+
+
+.PHONY: test
+test:
+	echo TODO

libs/skyline-nginx/poetry.toml

@@ -0,0 +1,2 @@
+[virtualenvs]
+in-project = true

libs/skyline-nginx/pyproject.toml

@@ -0,0 +1,38 @@
+[tool.poetry]
+name = "skyline-nginx"
+version = "0.1.0"
+description = ""
+license = "Apache-2.0"
+authors = ["OpenStack <openstack-discuss@lists.openstack.org>"]
+
+[tool.poetry.dependencies]
+python = "^3.8"
+pydantic = "*"
+click = "*"
+jinja2 = "*"
+python-keystoneclient = "*"
+keystoneauth1 = "*"
+skyline-config = "*"
+skyline-log = "*"
+skyline-console = "*"
+skyline-apiserver = "*"
+
+[tool.poetry.dev-dependencies]
+isort = "*"
+black = "^21.5b1"
+add-trailing-comma = "*"
+flake8 = "*"
+mypy = "*"
+pytest = "*"
+pytest-xdist = {extras = ["psutil"], version = "*"}
+skyline-config = {path = "../skyline-config", develop = true}
+skyline-log = {path = "../skyline-log", develop = true}
+skyline-console = {path = "../skyline-console", develop = true}
+skyline-apiserver = {path = "../skyline-apiserver", develop = true}
+
+[tool.poetry.scripts]
+nginx-generator = 'skyline_nginx.cmd.generate_nginx:main'
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

libs/skyline-nginx/skyline_nginx/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2021 99cloud
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__version__ = "0.1.0"

libs/skyline-nginx/skyline_nginx/cmd/generate_nginx.py

@@ -0,0 +1,166 @@
+# Copyright 2021 99cloud
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import sys
+from logging import StreamHandler
+from pathlib import Path, PurePath
+from typing import Any, Dict
+from urllib.parse import urlparse
+
+import click
+import skyline_nginx
+from pydantic import BaseModel
+from jinja2 import Template
+from keystoneauth1.identity.v3 import Password, Token
+from keystoneauth1.session import Session
+from keystoneclient.client import Client as KeystoneClient
+from skyline_console import static_path
+from skyline_log import LOG, setup
+from skyline_nginx.config import CONF, configure
+
+class CommandException(Exception):
+    EXIT_CODE = 1
+
+
+class ProxyEndpoint(BaseModel):
+    part: str
+    location: str
+    url: str
+
+
+def get_system_session() -> Session:
+    auth = Password(
+        auth_url=CONF.openstack.keystone_url,
+        user_domain_name=CONF.openstack.system_user_domain,
+        username=CONF.openstack.system_user_name,
+        password=CONF.openstack.system_user_password,
+        project_name=CONF.openstack.system_project,
+        project_domain_name=CONF.openstack.system_project_domain,
+        reauthenticate=True,
+    )
+    return Session(auth=auth, verify=False, timeout=30)
+
+
+def get_proxy_endpoints() -> Dict[str, ProxyEndpoint]:
+    ks_client = KeystoneClient(
+        session=get_system_session(),
+        interface=CONF.openstack.interface_type,
+        region_name=CONF.openstack.default_region,
+    )
+    endpoints_list = ks_client.endpoints.list(interface=CONF.openstack.interface_type)
+    service_list = ks_client.services.list()
+    services = {s.id: s.type for s in service_list}
+
+    endpoints = {}
+    for endpoint in endpoints_list:
+        proxy = ProxyEndpoint(part="", location="", url="")
+        region = endpoint.region
+        service_type = services.get(endpoint.service_id)
+        service = CONF.openstack.service_mapping.get(service_type)
+        if service is None:
+            continue
+        if f"{region}-{service_type}" in endpoints:
+            raise KeyError(f"Region \"{region}\" service type \"{service_type}\" conflict in endpoints.")
+
+        proxy.part = f"# {region} {service}"
+        location = PurePath("/").joinpath(
+            CONF.openstack.nginx_prefix,
+            region.lower(),
+            service,
+        )
+        proxy.location = f"{str(location)}/"
+
+        raw_url = urlparse(endpoint.url)
+        path = ""
+        if raw_url.path:
+            raw_path = PurePath(raw_url.path)
+            if len(raw_path.parts) > 1:
+                if raw_path.match("%(tenant_id)s") or raw_path.match("$(project_id)s"):
+                    path = (
+                        "" if str(raw_path.parents[1]) == "/" else raw_path.parents[1]
+                    )
+                elif raw_path.match("v[0-9]") or raw_path.match("v[0-9][.][0-9]"):
+                    path = (
+                        "" if str(raw_path.parents[0]) == "/" else raw_path.parents[0]
+                    )
+                else:
+                    path = raw_path
+
+        proxy.url = raw_url._replace(path=f"{str(path)}/").geturl()
+        endpoints[f"{region}-{service_type}"] = proxy
+
+    return dict(sorted(endpoints.items(), key=lambda d: d[0]))
+
+
+@click.command(help="Generate nginx proxy config file.")
+@click.option(
+    "-o",
+    "--output-file",
+    "output_file_path",
+    help=(
+        "The path of the output file, this file is to generate a reverse proxy configuration "
+        "file based on the openstack endpoint and should be used in the location part of nginx."
+    ),
+)
+@click.option(
+    "--ssl-certfile",
+    "ssl_certfile",
+    help=("SSL certificate file path."),
+)
+@click.option(
+    "--ssl-keyfile",
+    "ssl_keyfile",
+    help=("SSL key file path."),
+)
+def main(output_file_path: str, ssl_certfile: str, ssl_keyfile: str) -> None:
+    try:
+        configure("skyline")
+        setup(StreamHandler(), debug=CONF.default.debug)
+
+        template_file_path = (
+            Path(skyline_nginx.__file__).parent
+            .joinpath("templates")
+            .joinpath("nginx.conf.j2")
+        )
+        content = ""
+        with template_file_path.open() as f:
+            content = f.read()
+        template = Template(content)
+
+        endpoints = get_proxy_endpoints()
+        context = {
+            "skyline_console_static_path": static_path,
+            "endpoints": [i.dict() for i in endpoints.values()],
+        }
+        if ssl_certfile:
+            context.update(ssl_certfile=ssl_certfile)
+        if ssl_keyfile:
+            context.update(ssl_certfile=ssl_keyfile)
+        result = template.render(**context)
+
+        if output_file_path:
+            with open(output_file_path, mode="w") as f:
+                f.write(result)
+        else:
+            print(result)
+
+    except CommandException as e:
+        LOG.error(e)
+        sys.exit(e.EXIT_CODE)
+
+
+if __name__ == "__main__":
+    main()

libs/skyline-nginx/skyline_nginx/config/__init__.py

@@ -0,0 +1,37 @@
+# Copyright 2021 99cloud
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import os
+
+from skyline_config import Configuration, Group
+
+from . import default, openstack
+
+CONF = Configuration()
+
+
+def configure(project: str, setup: bool = True) -> None:
+    conf_modules = (
+        (default.GROUP_NAME, default.ALL_OPTS),
+        (openstack.GROUP_NAME, openstack.ALL_OPTS),
+    )
+    groups = [Group(*item) for item in conf_modules]
+    CONF(groups)
+    if setup:
+        CONF.setup(project, os.environ.copy())
+
+
+__all__ = ("CONF", "configure")

libs/skyline-nginx/skyline_nginx/config/default.py

@@ -0,0 +1,22 @@
+# Copyright 2021 99cloud
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from skyline_apiserver.config.default import debug
+
+GROUP_NAME = __name__.split(".")[-1]
+ALL_OPTS = (debug,)
+
+__all__ = ("GROUP_NAME", "ALL_OPTS")

libs/skyline-nginx/skyline_nginx/config/openstack.py

@@ -0,0 +1,44 @@
+# Copyright 2021 99cloud
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from skyline_apiserver.config.openstack import (
+    default_region,
+    interface_type,
+    keystone_url,
+    nginx_prefix,
+    service_mapping,
+    system_project,
+    system_project_domain,
+    system_user_domain,
+    system_user_name,
+    system_user_password,
+)
+
+GROUP_NAME = __name__.split(".")[-1]
+ALL_OPTS = (
+    default_region,
+    keystone_url,
+    system_project_domain,
+    system_project,
+    system_user_domain,
+    system_user_name,
+    system_user_password,
+    interface_type,
+    nginx_prefix,
+    service_mapping,
+)
+
+__all__ = ("GROUP_NAME", "ALL_OPTS")

libs/skyline-nginx/skyline_nginx/templates/nginx.conf.j2

@@ -0,0 +1,114 @@
+worker_processes auto;
+pid /run/nginx.pid;
+include /etc/nginx/modules-enabled/*.conf;
+
+events {
+    worker_connections 1024;
+    multi_accept on;
+}
+
+http {
+
+    ##
+    # Basic Settings
+    ##
+    sendfile on;
+    tcp_nopush on;
+    tcp_nodelay on;
+    client_max_body_size 0;
+    types_hash_max_size 2048;
+    proxy_request_buffering off;
+    server_tokens off;
+
+    # server_names_hash_bucket_size 64;
+    # server_name_in_redirect off;
+
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    ##
+    # SSL Settings
+    ##
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_prefer_server_ciphers on;
+
+    # Self signed certs generated by the ssl-cert package
+    # Don't use them in a production server!
+    ssl_certificate {{ ssl_certfile | default('/etc/ssl/certs/ssl-cert-snakeoil.pem') }};
+    ssl_certificate_key {{ ssl_keyfile | default('/etc/ssl/private/ssl-cert-snakeoil.key') }};
+
+    ##
+    # Logging Settings
+    ##
+    log_format main '$remote_addr - $remote_user [$time_local] "$request_time" '
+                    '"$upstream_response_time" "$request" '
+                    '$status $body_bytes_sent "$http_referer" '
+                    '"$http_user_agent" "$http_x_forwarded_for"';
+    access_log /var/log/skyline/nginx_access.log main;
+    error_log /var/log/skyline/nginx_error.log;
+
+    ##
+    # Gzip Settings
+    ##
+    gzip on;
+    gzip_static on;
+    gzip_disable "msie6";
+
+    gzip_vary on;
+    gzip_proxied any;
+    gzip_comp_level 6;
+    gzip_buffers 16 8k;
+    # gzip_http_version 1.1;
+    gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
+
+    upstream skyline {
+        server unix:/var/lib/skyline/skyline.sock fail_timeout=0;
+    }
+
+    ##
+    # Virtual Host Configs
+    ##
+    server {
+        listen 8080 ssl http2 default_server;
+
+        root {{ skyline_console_static_path }};
+
+        # Add index.php to the list if you are using PHP
+        index index.html;
+
+        server_name _;
+
+        location / {
+            # First attempt to serve request as file, then
+            # as directory, then fall back to displaying a 404.
+            try_files $uri $uri/ /index.html;
+            expires 1d;
+            add_header Cache-Control "public";
+        }
+
+        location /api/openstack/skyline/ {
+            proxy_pass http://skyline/;
+            proxy_redirect off;
+            proxy_buffering off;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header X-Forwarded-Host $host;
+            proxy_set_header Host $http_host;
+        }
+
+        {% for endpoint in endpoints %}
+        {{ endpoint["part"] }}
+        location {{ endpoint["location"] }} {
+            proxy_pass {{ endpoint["url"] }};
+            proxy_redirect {{ endpoint["url"] }} {{ endpoint["location"] }};
+            proxy_buffering off;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header X-Forwarded-Host $host;
+            proxy_set_header Host $http_host;
+        }
+        {% endfor %}
+    }
+
+}
+

libs/skyline-nginx/tests/test_skyline_nginx.py

@@ -0,0 +1,19 @@
+# Copyright 2021 99cloud
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from skyline_log import __version__
+
+
+def test_version():
+    assert __version__ == "0.1.0"