[19.0] server_environment: some forward ports from 18.0

server_environment/README.rst

@@ -40,7 +40,7 @@ are stored in the ``server_environment_files`` companion module.
 
 The ``server_environment_files`` module is optional, the values can be
 set using an environment variable with a fallback on default values in
-the database.
+the database. you will be able to overwrite some odoo options.
 
 The configuration read from the files are visible under the
 Configuration menu. If you are not in the 'dev' environment you will not
@@ -106,6 +106,13 @@ example:
   can override or extend default values;
 - you can override or extend values in the main configuration file of
   your instance;
+- In some platforms (like odoo.sh where production config file is copied
+  to staging) it can be useful to overwrite options written in the
+  ``[options]`` section. You must allow the override by adding
+  ``server_environment_allow_overwrite_options_section = True`` to the
+  former ``odoo.cfg`` config file or through the environment variable:
+  ``export SERVER_ENVIRONMENT_ALLOW_OVERWRITE_OPTIONS_SECTION=True`` (if
+  both are set config file takes precedence).
 
 Environment variable
 --------------------

server_environment/models/server_env_mixin.py

@@ -9,6 +9,7 @@
 
 from odoo import api, fields, models
 from odoo.orm.model_classes import add_field
+from odoo.tools import mute_logger
 
 from odoo.addons.base_sparse_field.models.fields import Serialized
 
@@ -357,7 +358,9 @@ def _server_env_transform_field_to_read_from_env(self, field):
         inverse_method = _partialmethod(
             type(self)._inverse_server_env, field.name, __name__=inverse_method_name
         )
-        setattr(type(self), inverse_method_name, inverse_method)
+        # Mute message related to new safeguard (PR odoo/odoo#247151)
+        with mute_logger("odoo.tests.common"):
+            setattr(type(self), inverse_method_name, inverse_method)
         field.inverse = inverse_method_name
         field.store = False
         field.required = False

server_environment/readme/CONFIGURE.md

@@ -29,6 +29,13 @@ addon. The `server_environment_files_sample` can be used as an example:
   can override or extend default values;
 - you can override or extend values in the main configuration file of
   your instance;
+- In some platforms (like odoo.sh where production config file is copied to staging)
+  it can be useful to overwrite options written in the `[options]` section. You must
+  allow the override by adding `server_environment_allow_overwrite_options_section = True`
+  to the former `odoo.cfg` config file or through the environment variable:
+  `export SERVER_ENVIRONMENT_ALLOW_OVERWRITE_OPTIONS_SECTION=True` (if both are set
+  config file takes precedence).
+
 
 ## Environment variable
 

server_environment/readme/DESCRIPTION.md

@@ -5,8 +5,8 @@ configuration file, and the values for the various possible environments
 are stored in the `server_environment_files` companion module.
 
 The `server_environment_files` module is optional, the values can be set
-using an environment variable with a fallback on default values in the
-database.
+using an environment variable with a fallback on default values in the database. you
+will be able to overwrite some odoo options.
 
 The configuration read from the files are visible under the
 Configuration menu. If you are not in the 'dev' environment you will not

server_environment/server_env.py

@@ -10,11 +10,12 @@
 from lxml import etree
 
 from odoo import api, fields, models
+from odoo.tools import mute_logger
 from odoo.tools.config import config as system_base_config
 
 from odoo.addons.base_sparse_field.models.fields import Serialized
 
-from .system_info import get_server_environment
+from .system_info import get_server_environment, skip_subprocess
 
 _logger = logging.getLogger(__name__)
 
@@ -106,6 +107,19 @@ def _listconf(env_path):
     return files
 
 
+def _update_odoo_config_options(config_p):
+    allow_overwrite = system_base_config.get(
+        "server_environment_allow_overwrite_options_section",
+        os.environ.get("SERVER_ENVIRONMENT_ALLOW_OVERWRITE_OPTIONS_SECTION"),
+    )
+    if isinstance(allow_overwrite, str) and allow_overwrite:
+        allow_overwrite = _boolean_states.get(allow_overwrite.lower(), False)
+    if allow_overwrite and config_p.has_section("options"):
+        system_base_config.options.update(
+            {k: v for k, v in config_p["options"].items()}
+        )
+
+
 def _load_config_from_server_env_files(config_p):
     default = os.path.join(_dir, "default")
     running_env = os.path.join(_dir, system_base_config["running_env"])
@@ -118,6 +132,7 @@ def _load_config_from_server_env_files(config_p):
         config_p.read(conf_files)
     except Exception as e:
         raise Exception(f'Cannot read config files "{conf_files}":  {e}') from e
+    _update_odoo_config_options(config_p)
 
 
 def _load_config_from_rcfile(config_p):
@@ -209,11 +224,9 @@ def _add_columns(cls):
                 sparse="config",
                 readonly=True,
             )
-            setattr(
-                ServerConfiguration,
-                col_name,
-                tmp_field,
-            )
+            # Mute message related to new safeguard (PR odoo/odoo#247151)
+            with mute_logger("odoo.tests.common"):
+                setattr(ServerConfiguration, col_name, tmp_field)
             tmp_field.name = col_name
             ServerConfiguration._field_definitions.append(tmp_field)
             cls._conf_defaults[col_name] = value
@@ -241,6 +254,8 @@ def _get_env_cols(cls, sections=None):
     @classmethod
     def _get_system_cols(cls):
         """Compute system fields"""
+        if skip_subprocess():
+            return {}
         res = {}
         for col, item in get_server_environment():
             key = cls._format_key("system", col)

server_environment/static/description/index.html

@@ -382,7 +382,7 @@ <h1>server configuration environment files</h1>
 are stored in the <tt class="docutils literal">server_environment_files</tt> companion module.</p>
 <p>The <tt class="docutils literal">server_environment_files</tt> module is optional, the values can be
 set using an environment variable with a fallback on default values in
-the database.</p>
+the database. you will be able to overwrite some odoo options.</p>
 <p>The configuration read from the files are visible under the
 Configuration menu. If you are not in the ‘dev’ environment you will not
 be able to see the values contained in the defined secret keys (by
@@ -459,6 +459,13 @@ <h3><a class="toc-backref" href="#toc-entry-3">server_environment_files</a></h3>
 can override or extend default values;</li>
 <li>you can override or extend values in the main configuration file of
 your instance;</li>
+<li>In some platforms (like odoo.sh where production config file is copied
+to staging) it can be useful to overwrite options written in the
+<tt class="docutils literal">[options]</tt> section. You must allow the override by adding
+<tt class="docutils literal">server_environment_allow_overwrite_options_section = True</tt> to the
+former <tt class="docutils literal">odoo.cfg</tt> config file or through the environment variable:
+<tt class="docutils literal">export SERVER_ENVIRONMENT_ALLOW_OVERWRITE_OPTIONS_SECTION=True</tt> (if
+both are set config file takes precedence).</li>
 </ul>
 </div>
 <div class="section" id="environment-variable">

server_environment/system_info.py

@@ -3,16 +3,33 @@
 # License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html)
 
 import locale
+import logging
 import os
 import platform
 import subprocess
 from functools import lru_cache
 
+import odoo
 from odoo import release
 from odoo.tools.config import config
 
+_logger = logging.getLogger(__name__)
+
+
+def skip_subprocess():
+    # In the gevent worker (longpolling/websocket) running subprocesses can
+    # deadlock, freezing real-time updates. This system info only feeds the
+    # settings display, so skip the subprocess calls there.
+    return odoo.evented
+
 
 def _get_output(cmd):
+    # Use assert to force developers to
+    # take correct action when developing
+    # but running `python -O` removes it completely
+    assert not skip_subprocess(), (
+        "Subprocess must not be called, use skip_subprocess in a pre-check"
+    )
     bindir = config.root_path
     p = subprocess.Popen(
         cmd, shell=True, cwd=bindir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
@@ -22,6 +39,9 @@ def _get_output(cmd):
 
 @lru_cache(maxsize=1)
 def get_server_environment():
+    # Function relies mainly on subprocesses
+    if skip_subprocess():
+        return ()
     # inspired by server/bin/service/web_services.py
     try:
         rev_id = "git:{}".format(_get_output("git rev-parse HEAD"))
@@ -34,10 +54,12 @@ def get_server_environment():
     os_lang = ".".join([x for x in locale.getlocale() if x])
     if not os_lang:
         os_lang = "NOT SET"
+    lsbinfo = "not lsb compliant"
     if os.name == "posix" and platform.system() == "Linux":
-        lsbinfo = _get_output("lsb_release -a")
-    else:
-        lsbinfo = "not lsb compliant"
+        try:
+            lsbinfo = _get_output("lsb_release -a")
+        except Exception:
+            _logger.info("lsb_release not compliant")
     return (
         ("platform", platform.platform()),
         ("os.name", os.name),

server_environment/tests/common.py

@@ -48,6 +48,7 @@ def load_config(
         serv_config_class=server_env_mixin,
     ):
         original_serv_config = serv_config_class.serv_config
+        original_fields = dir(server_env.ServerConfiguration)
         try:
             with (
                 self.set_config_dir(config_dir),
@@ -61,3 +62,7 @@ def load_config(
         finally:
             serv_config_class.serv_config = original_serv_config
             server_env.serv_config = original_serv_config
+            for attr, fld in vars(server_env.ServerConfiguration).items():
+                if attr not in original_fields:
+                    server_env.ServerConfiguration._field_definitions.remove(fld)
+                    delattr(self.env["server.config"].__class__, attr)

server_environment/tests/test_server_environment.py

@@ -84,12 +84,12 @@ def test_default_hidden_password(self):
             model = self.env["server.config"]
             model._add_columns()
             self.env.registry._setup_models__(self.env.cr)
-        defaults = self._test_default(hidden_pwd=True)
+            defaults = self._test_default(hidden_pwd=True)
 
-        self.assertIn("odoo_I_admin_passwd", defaults)
-        self.assertIn("odoo_I_db_password", defaults)
-        self.assertIn("odoo_I_smtp_password", defaults)
-        self.assertIn("outgoing_mail_provider_promail_I_smtp_pass", defaults)
+            self.assertIn("odoo_I_admin_passwd", defaults)
+            self.assertIn("odoo_I_db_password", defaults)
+            self.assertIn("odoo_I_smtp_password", defaults)
+            self.assertIn("outgoing_mail_provider_promail_I_smtp_pass", defaults)
 
     def test_restore_env_managed_columns_unknown_field(self):
         """Helper gracefully skips a field that doesn't exist on the model."""
@@ -104,3 +104,61 @@ def test_restore_env_managed_columns_unknown_field(self):
     def test_restore_env_managed_columns_no_fields(self):
         """Helper is a no-op when given an empty field list."""
         restore_env_managed_columns(self.env, "res.partner", [])
+
+    @patch.dict(os.environ, {"SERVER_ENVIRONMENT_ALLOW_OVERWRITE_OPTIONS_SECTION": "0"})
+    @patch.dict(
+        odoo_config.options,
+        {
+            "running_env": "testing",
+            "server_environment_allow_overwrite_options_section": True,
+            "odoo_test_option": "fake odoo config",
+        },
+    )
+    def test_server_environment_allow_overwrite_options_section(self):
+        with self.set_config_dir("testfiles"):
+            server_env._load_config()
+            self.assertEqual(
+                odoo_config["odoo_test_option"], "Set in config file for testing env"
+            )
+
+    @patch.dict(os.environ, {"SERVER_ENVIRONMENT_ALLOW_OVERWRITE_OPTIONS_SECTION": "1"})
+    @patch.dict(
+        odoo_config.options,
+        {
+            "running_env": "testing",
+            "server_environment_allow_overwrite_options_section": False,
+            "odoo_test_option": "fake odoo config",
+        },
+    )
+    def test_server_environment_disabled_overwrite_options_section(self):
+        with self.set_config_dir("testfiles"):
+            server_env._load_config()
+            self.assertEqual(odoo_config["odoo_test_option"], "fake odoo config")
+
+    @patch.dict(os.environ, {"SERVER_ENVIRONMENT_ALLOW_OVERWRITE_OPTIONS_SECTION": "1"})
+    @patch.dict(
+        odoo_config.options,
+        {
+            "running_env": "testing",
+            "odoo_test_option": "fake odoo config",
+        },
+    )
+    def test_server_environment_allow_overwrite_options_section_by_env(self):
+        with self.set_config_dir("testfiles"):
+            server_env._load_config()
+            self.assertEqual(
+                odoo_config["odoo_test_option"], "Set in config file for testing env"
+            )
+
+    @patch.dict(os.environ, {"SERVER_ENVIRONMENT_ALLOW_OVERWRITE_OPTIONS_SECTION": "0"})
+    @patch.dict(
+        odoo_config.options,
+        {
+            "running_env": "testing",
+            "odoo_test_option": "fake odoo config",
+        },
+    )
+    def test_server_environment_disabled_overwrite_options_section_by_env(self):
+        with self.set_config_dir("testfiles"):
+            server_env._load_config()
+            self.assertEqual(odoo_config["odoo_test_option"], "fake odoo config")

server_environment/tests/testfiles/testing/base.conf

@@ -1,2 +1,5 @@
+[options]
+odoo_test_option = Set in config file for testing env
+
 [external_service.ftp]
 user = testing