From b156c2e27ef598ba22910c811621e2085dfbf822 Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Thu, 12 Mar 2015 15:46:47 +0100 Subject: [PATCH] [IMP] registry: adapt LRU sizes for registries and ormcache The registry size is now assumed to be around 10Mb, and ormcache size is proportional to the maximum number of registries. Statistics about ormcache usage is shown to the log when receiving signal SIGUSR1. --- openerp/modules/registry.py | 9 +++++---- openerp/service/server.py | 8 +++++++- openerp/tools/cache.py | 31 +++++++++++++++++++++++++------ 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/openerp/modules/registry.py b/openerp/modules/registry.py index 5c9870a488a..352c6f156af 100644 --- a/openerp/modules/registry.py +++ b/openerp/modules/registry.py @@ -305,8 +305,9 @@ class RegistryManager(object): # cannot specify the memory limit soft on windows... size = 42 else: - # On average, a clean registry take 25MB of memory + cache - avgsz = 30 * 1024 * 1024 + # A registry takes 10MB of memory on average, so we reserve + # 10Mb (registry) + 5Mb (working memory) per registry + avgsz = 15 * 1024 * 1024 size = int(config['limit_memory_soft'] / avgsz) cls._registries = LRU(size) @@ -319,8 +320,8 @@ class RegistryManager(object): """ with cls.lock(): if cls._cache is None: - # we allocate one cache entry per 32KB of memory - size = max(8192, int(config['limit_memory_soft'] / 32768)) + # we allocate 8192 cache entries per registry + size = 8192 * cls.registries.count cls._cache = LRU(size) return cls._cache diff --git a/openerp/service/server.py b/openerp/service/server.py index 0ec039e83ad..b6c57fa7325 100644 --- a/openerp/service/server.py +++ b/openerp/service/server.py @@ -38,7 +38,7 @@ import openerp from openerp.modules.registry import RegistryManager from openerp.release import nt_service_name import openerp.tools.config as config -from openerp.tools.misc import stripped_sys_argv, dumpstacks +from openerp.tools import stripped_sys_argv, dumpstacks, log_ormcache_stats _logger = logging.getLogger(__name__) @@ -296,6 +296,7 @@ class ThreadedServer(CommonServer): signal.signal(signal.SIGCHLD, self.signal_handler) signal.signal(signal.SIGHUP, self.signal_handler) signal.signal(signal.SIGQUIT, dumpstacks) + signal.signal(signal.SIGUSR1, log_ormcache_stats) elif os.name == 'nt': import win32api win32api.SetConsoleCtrlHandler(lambda sig: self.signal_handler(sig, None), 1) @@ -389,6 +390,7 @@ class GeventServer(CommonServer): if os.name == 'posix': signal.signal(signal.SIGQUIT, dumpstacks) + signal.signal(signal.SIGUSR1, log_ormcache_stats) gevent.spawn(self.watch_parent) self.httpd = WSGIServer((self.interface, self.port), self.app) @@ -510,6 +512,9 @@ class PreforkServer(CommonServer): elif sig == signal.SIGQUIT: # dump stacks on kill -3 self.dumpstacks() + elif sig == signal.SIGUSR1: + # log ormcache stats on kill -SIGUSR1 + log_ormcache_stats() elif sig == signal.SIGTTIN: # increase number of workers self.population += 1 @@ -586,6 +591,7 @@ class PreforkServer(CommonServer): signal.signal(signal.SIGTTIN, self.signal_handler) signal.signal(signal.SIGTTOU, self.signal_handler) signal.signal(signal.SIGQUIT, dumpstacks) + signal.signal(signal.SIGUSR1, log_ormcache_stats) # listen to socket self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) diff --git a/openerp/tools/cache.py b/openerp/tools/cache.py index ab387de361a..99364f8da98 100644 --- a/openerp/tools/cache.py +++ b/openerp/tools/cache.py @@ -27,7 +27,7 @@ from inspect import getargspec import lru import logging -logger = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) class ormcache(object): @@ -47,9 +47,9 @@ class ormcache(object): return lookup def stat(self): - return "lookup-stats hit=%s miss=%s err=%s ratio=%.1f" % \ + return "lookup-stats %6d hit %6d miss %6d err %4.1f ratio" % \ (self.stat_hit, self.stat_miss, self.stat_err, - (100*float(self.stat_hit))/(self.stat_miss+self.stat_hit)) + (100*float(self.stat_hit))/(self.stat_miss+self.stat_hit or 1)) def lru(self, model): return model.pool.cache, (model.pool.db_name, model._name, self.method) @@ -73,9 +73,9 @@ class ormcache(object): """ Remove *args entry from the cache or all keys if *args is undefined """ d, key0 = self.lru(model) if args: - logger.warn("ormcache.clear arguments are deprecated and ignored " - "(while clearing caches on (%s).%s)", - model._name, self.method.__name__) + _logger.warn("ormcache.clear arguments are deprecated and ignored " + "(while clearing caches on (%s).%s)", + model._name, self.method.__name__) d.clear_prefix(key0) model.pool._any_cache_cleared = True @@ -168,6 +168,25 @@ class dummy_cache(object): pass +def log_ormcache_stats(sig=None, frame=None): + """ Log statistics of ormcache usage by database, model, and method. """ + from openerp.modules.registry import RegistryManager + from collections import defaultdict + import threading + + me = threading.currentThread() + entries = defaultdict(int) + for key in RegistryManager.cache.iterkeys(): + entries[key[:3]] += 1 + for (dbname, model_name, method), count in sorted(entries.items()): + me.dbname = dbname + model = RegistryManager.get(dbname)[model_name] + func = getattr(model, method.__name__).im_func + ormcache = func.clear_cache.im_self + _logger.info("%6d entries, %s, for %s.%s", + count, ormcache.stat(), model_name, method.__name__) + + # For backward compatibility cache = ormcache