Memcache-based Buffer-Object for Global Link-Target-Caching

Products/zms/ZMSRepositoryManager.py

@@ -129,7 +129,7 @@ def get_conf_basepath(self, id=''):
       return basepath
 
 
-    def remoteFiles(self, provider):
+    def get_modelfileset_from_disk(self, provider):
       """
       Retrieve remote files from a repository provider.
 
@@ -144,19 +144,19 @@ def remoteFiles(self, provider):
       @return: A list of remote files available from the provider
       @rtype: list
 
-      @see: L{repositoryutil.remoteFiles}
+      @see: L{repositoryutil.get_modelfileset_from_disk}
       @see: L{get_conf_basepath}
       """
-      standard.writeLog(self,"[remoteFiles]: provider=%s"%str(provider))
+      standard.writeLog(self,"[get_modelfileset_from_disk]: provider=%s"%str(provider))
       basepath = self.get_conf_basepath(provider.id)
-      return repositoryutil.remoteFiles(self, basepath)
+      return repositoryutil.get_modelfileset_from_disk(self, basepath)
 
 
-    def readRepository(self, provider):
+    def get_models_from_disk(self, provider):
       """
       Read repository data from a provider.
 
-      Implements the 'readRepository' interface. This method retrieves repository
+      Implements the 'get_models_from_disk' interface. This method retrieves repository
       information from the specified provider by reading the repository structure
       from the configured base path.
 
@@ -167,11 +167,11 @@ def readRepository(self, provider):
       @return: Repository data read from the provider's base path.
       @rtype: dict
 
-      @see: repositoryutil.readRepository()
+      @see: repositoryutil.get_models_from_disk()
       """
-      standard.writeLog(self,"[readRepository]: provider=%s"%str(provider))
+      standard.writeLog(self,"[get_models_from_disk]: provider=%s"%str(provider))
       basepath = self.get_conf_basepath(provider.id)
-      return repositoryutil.readRepository(self, basepath)
+      return repositoryutil.get_models_from_disk(self, basepath)
 
 
     def commitChanges(self, ids):
@@ -187,7 +187,7 @@ def commitChanges(self, ids):
         for id in list(set([x.split(':')[1] for x in ids if x.split(':')[0]==provider_id])):
           try:
             # Read local-files from provider.
-            files = repositoryutil.localFiles(self, provider, [id])
+            files = repositoryutil.get_modelfileset_from_zodb(self, provider, [id])
             # Recreate folder.
             if os.path.exists(basepath):
               for name in os.listdir(basepath):
@@ -251,7 +251,7 @@ def updateChanges(self, ids, override=False):
         provider = [x for x in providers if x.id == provider_id][0]
         # Read repositories for provider.
         if provider_id not in repositories:
-          repositories[provider_id] = self.readRepository(provider)
+          repositories[provider_id] = self.get_models_from_disk(provider)
         repository = repositories[provider_id]
         # Update.
         try:

Products/zms/_cachemanager.py

@@ -24,13 +24,92 @@
 Organization: ZMS Publishing
 """
 # Imports.
+from AccessControl import ClassSecurityInfo
 from Products.zms import standard
 from zope.globalrequest import getRequest
 
 class Buff(object):
   """Lightweight attribute container used for request-local buffering."""
   pass
 
+class SharedBuff(object):
+    """
+    Shared buffer helpers using a multi-user cache (RAM or Memcache).
+    This allows data persistence across different requests and users.
+    """
+
+    security = ClassSecurityInfo()
+
+    security.declarePublic('get_cache_manager')
+    def get_cache_manager(self, cache_id='zms_cache'):
+        """
+        Fetch the configured Zope Cache Manager.
+        Expects a configuration property 'ZMS.cache.path' pointing to the 
+        manager (e.g., '/zms/my_ram_cache').
+        """
+        cache_path = self.getConfProperty('ZMS.cache.path', None)
+        if cache_path:
+            try:
+                manager = self.unrestrictedTraverse(cache_path)
+                return manager.ZCacheManager_getCache()
+            except:
+                pass
+        return None
+
+    security.declarePublic('fetchSharedBuff')
+    def fetchSharedBuff(self, key):
+        """Fetch a value from the shared global cache."""
+        doc_element = self.getAbsoluteHome().content # or self.getPortalMaster() or self.getDocumentElement()
+        cache = doc_element.get_cache_manager()
+        if cache:
+            # Note: keywords/view_name can be used for namespacing if needed.
+            return cache.ZCache_get(doc_element, view_name='shared', keywords={'key': key})
+        return None
+
+    security.declarePublic('storeSharedBuff')
+    def storeSharedBuff(self, key, value):
+        """Store a value in the shared global cache."""
+        doc_element = self.getAbsoluteHome().content # or self.getPortalMaster() or self.getDocumentElement()
+        cache = doc_element.get_cache_manager()
+        if cache:
+            # We use the DocumentElement (ZMS site root) as the context 
+            # to ensure the cache is shared globally across the entire site.
+            cache.ZCache_set(doc_element, value, view_name='shared', keywords={'key': key})
+        return value
+
+    security.declarePublic('getSharedBuffJSON')
+    def getSharedBuffJSON(self):
+        """
+        Return the contents of the shared cache as a JSON-formatted string.
+        Leverages the internal tracking mechanism of mcdutils using the 
+        DocumentElement as the global context.
+        """
+        import json
+        cache = self.get_cache_manager()
+        data = {}
+        if cache:
+            # Always look at the DocumentElement's path for the global cache
+            doc_element = self.getAbsoluteHome().content # or self.getPortalMaster() or self.getDocumentElement()
+            path = '/'.join(doc_element.getPhysicalPath())
+            tracked_keys = cache.proxy.get(path)
+
+            if tracked_keys:
+                for k in tracked_keys.keys():
+                    val = cache.proxy.get(k)
+                    data[str(k)] = str(val) if val is not None else "EXPIRED/NONE"
+
+            if not data:
+                data = {
+                    'info': 'Global cache is empty.',
+                    'path_searched': path,
+                    'manager': str(cache)
+                }
+        else:
+            data = {'error': 'No cache manager found.'}
+
+        return json.dumps(data, indent=2)
+
+
 class ReqBuff(object):
     """Request-scoped buffer helpers for expensive values computed during one request."""
 

Products/zms/_confmanager.py

@@ -230,7 +230,7 @@ def getConfXmlFile(self, file):
           filename = file[file.find(':')+1:]
           basepath = repositoryutil.get_system_conf_basepath()
           path = os.path.join(basepath, filename)
-          r = repositoryutil.readRepository(self, path)
+          r = repositoryutil.get_models_from_disk(self, path)
           container_id = filename.split('/')[0]
           container = zopeutil.getObject(self,container_id)
           if container is not None:
@@ -355,7 +355,7 @@ def getConfFiles(self, pattern=None, REQUEST=None, RESPONSE=None):
           path = os.path.join(basepath, filename)
           if os.path.isdir(path):
               if pattern is None or filename.startswith(pattern[1:-1]):
-                  r = repositoryutil.readRepository(self, path, deep=False)
+                  r = repositoryutil.get_models_from_disk(self, path, deep=False)
                   for k in r:
                       v = r[k]
                       # Get qualified name.

Products/zms/_zreferableitem.py

@@ -8,6 +8,8 @@
 Organization: ZMS Publishing
 """
 # Imports.
+import time
+
 from Products.PageTemplates.PageTemplateFile import PageTemplateFile
 import base64
 import re
@@ -551,6 +553,11 @@ def getLinkObj(self, url, REQUEST=None):
     @rtype: C{object} or C{None}
     """
     request = getattr(self, 'REQUEST', getRequest())
+    request.set('counter_getlinkobj', float(request.get('counter_getlinkobj', 0)) + 1)
+    # ///////////////////////////////////////////////////
+    # MEASURING PERFORMANCE:
+    start_getlinkobj = time.time()
+    # ///////////////////////////////////////////////////
     ob = None
     if isInternalLink(url):
       # Params.
@@ -565,38 +572,67 @@ def getLinkObj(self, url, REQUEST=None):
         i = max(url.find('#'),url.find(','))
         if i > 0:
           url = url[:i]
-        #-- [ReqBuff]: Fetch buffered value from Http-Request.
+        #-- [ReqBuff/SharedBuff]: Fetch buffered value.
         reqBuffId = 'getLinkObj.%s'%url
         try:
-          ob = self.getDocumentElement().fetchReqBuff(reqBuffId)
+          # Try Request-Local first.
+          ob = self.fetchReqBuff(reqBuffId)
         except:
-          if url.find('id:') >= 0:
-            catalog = self.getZMSIndex().get_catalog()
-            q = catalog({'get_uid':url})
-            for r in q:
-              path  = '%s/'%r['getPath']
+          # Try Shared Global Context if configured.
+          if hasattr(self, 'fetchSharedBuff'):
+            cached_path = self.fetchSharedBuff(reqBuffId)
+            if cached_path:
+               ob = self.unrestrictedTraverse(cached_path, None)
+
+          if ob is None:
+            # ///////////////////////////////////////////////////
+            # MEASURING PERFORMANCE:
+            # For summing up the time needed find object by URL via ZMSIndex:
+            # add the current time to the request-parameter 'total_time_consumed_by_zmsindex_requests'/int (in ms)
+            start_zmsindex_request = time.time()
+            # ///////////////////////////////////////////////////
+            if url.find('id:') >= 0:
+              catalog = self.getZMSIndex().get_catalog()
+              q = catalog({'get_uid':url})
+              for r in q:
+                path  = '%s/'%r['getPath']
+                l = [x for x in path.split('/') if x] 
+                ob = self.getRootElement()
+                if l:
+                  [l.pop(0) for x in ob.getPhysicalPath() if l[0] == x]
+                  for id in l:
+                    ob = getattr(ob,id,None)
+                break
+            elif not url.startswith('__'):
+              path = url.replace('@','/content/')
               l = [x for x in path.split('/') if x] 
-              ob = self.getRootElement()
+              ob = self.getDocumentElement()
               if l:
                 [l.pop(0) for x in ob.getPhysicalPath() if l[0] == x]
                 for id in l:
                   ob = getattr(ob,id,None)
-              break
-          elif not url.startswith('__'):
-            path = url.replace('@','/content/')
-            l = [x for x in path.split('/') if x] 
-            ob = self.getDocumentElement()
-            if l:
-              [l.pop(0) for x in ob.getPhysicalPath() if l[0] == x]
-              for id in l:
-                ob = getattr(ob,id,None)
-          #-- [ReqBuff]: Store value in buffer of Http-Request.
-          self.getDocumentElement().storeReqBuff(reqBuffId, ob)
+
+            #-- [ReqBuff/SharedBuff]: Store value.
+            self.storeReqBuff(reqBuffId, ob)
+            if ob and hasattr(self, 'storeSharedBuff'):
+               self.storeSharedBuff(reqBuffId, '/'.join(ob.getPhysicalPath()))
+            # ///////////////////////////////////////////////////
+            # MEASURING PERFORMANCE: Count and time for requests to ZMSIndex to find object by URL.
+            request.set('counter_zmsindex_requests', float(request.get('counter_zmsindex_requests', 0)) + 1)
+            request.set('time_consumed_by_zmsindex_requests_in_ms', round(float(request.get('time_consumed_by_zmsindex_requests_in_ms', 0)) + float((time.time() - start_zmsindex_request)*1000),2))
+            # ///////////////////////////////////////////////////
       # Prepare request (only if ref_params are provided)
       if ob is not None and ref_params:
         ids = self.getPhysicalPath()
         if ob.id not in ids:
           ob.set_request_context(request, ref_params)
+    # ///////////////////////////////////////////////////
+    # MEASURING PERFORMANCE: Count and time for getLinkObj calls.
+    request.set('time_consumed_by_getlinkobj_in_ms', round(float(request.get('time_consumed_by_getlinkobj_in_ms', 0)) + float((time.time() - start_getlinkobj)*1000),2))
+    time_consumed_by_getlinkobj_datalist = request.get('time_consumed_by_getlinkobj_datalist', [])
+    time_consumed_by_getlinkobj_datalist.append(round(float((time.time() - start_getlinkobj)*1000),2)) # in ms
+    request.set('time_consumed_by_getlinkobj_datalist', time_consumed_by_getlinkobj_datalist)
+    # ///////////////////////////////////////////////////
     return ob