DAOS-17823 client: implement LRU cache in shared memory

src/gurt/SConscript

@@ -1,12 +1,13 @@
-# (C) Copyright 2016-2023 Intel Corporation.
+# (C) Copyright 2016-2024 Intel Corporation.
+# (C) Copyright 2025 Hewlett Packard Enterprise Development LP
 #
 # SPDX-License-Identifier: BSD-2-Clause-Patent
 #
 """Build libgurt"""
 
 SRC = ['debug.c', 'dlog.c', 'hash.c', 'misc.c', 'heap.c', 'errno.c',
        'fault_inject.c', 'slab.c', 'telemetry.c', 'hlc.c', 'hlct.c', 'signals.c',
-       'shm_alloc.c', 'shm_dict.c', 'shm_tlsf.c', 'shm_utils.c']
+       'shm_alloc.c', 'shm_dict.c', 'shm_tlsf.c', 'shm_utils.c', 'shm_lru.c']
 
 
 def scons():
@@ -20,7 +21,7 @@ def scons():
 
     denv = env.Clone()
 
-    denv.AppendUnique(LIBS=['pthread', 'yaml', 'm', 'dl', 'rt'])
+    denv.AppendUnique(LIBS=['pthread', 'yaml', 'm', 'dl', 'rt', 'hwloc'])
     denv.require('uuid')
 
     gurt_targets = denv.SharedObject(SRC)

src/gurt/shm_alloc.c

@@ -14,6 +14,7 @@
 #include <fcntl.h>
 #include <errno.h>
 #include <pthread.h>
+#include <hwloc.h>
 
 #include <gurt/shm_utils.h>
 #include "shm_internal.h"
@@ -47,14 +48,57 @@ static uint64_t       page_size;
 
 extern __thread pid_t d_tid;
 
+#define INVALID_NUM_CORE (0xFFFFFFFF)
+static unsigned int
+get_cpu_core(void)
+{
+	hwloc_topology_t topology;
+	unsigned int     cpu_count;
+	int              depth;
+	int              rc;
+
+	/* Allocate, initialize, and perform topology detection */
+	rc = hwloc_topology_init(&topology);
+	if (rc) {
+		DS_ERROR(EINVAL, "libhwloc hwloc_topology_init() failed");
+		return INVALID_NUM_CORE;
+	}
+
+	rc = hwloc_topology_load(topology);
+	if (rc) {
+		DS_ERROR(EINVAL, "libhwloc hwloc_topology_load() failed");
+		cpu_count = INVALID_NUM_CORE;
+		goto out;
+	}
+	/* Try to get the number of CPU cores from topology */
+	depth = hwloc_get_type_depth(topology, HWLOC_OBJ_CORE);
+	if (depth == HWLOC_TYPE_DEPTH_UNKNOWN) {
+		DS_ERROR(EINVAL, "libhwloc hwloc_get_type_depth() failed");
+		cpu_count = INVALID_NUM_CORE;
+		goto out;
+	}
+
+	cpu_count = hwloc_get_nbobjs_by_depth(topology, depth);
+
+out:
+	/* Destroy topology object and return */
+	hwloc_topology_destroy(topology);
+	return cpu_count;
+}
+
 static int
 create_shm_region(uint64_t shm_size, uint64_t shm_pool_size)
 {
-	int   i;
-	int   shm_ht_fd;
-	int   shmopen_perm = 0600;
-	void *shm_addr;
-	char  daos_shm_name_buf[64];
+	int      i;
+	int      shm_ht_fd;
+	int      shmopen_perm = 0600;
+	void    *shm_addr;
+	char     daos_shm_name_buf[64];
+	uint32_t cpu_count;
+
+	cpu_count = get_cpu_core();
+	if (cpu_count == INVALID_NUM_CORE)
+		return errno;
 
 	/* the shared memory only accessible for individual user for now */
 	sprintf(daos_shm_name_buf, "%s_%d", daos_shm_name, getuid());
@@ -113,6 +157,7 @@ create_shm_region(uint64_t shm_size, uint64_t shm_pool_size)
 	d_shm_head->size          = shm_size;
 	d_shm_head->shm_pool_size = shm_pool_size;
 	d_shm_head->version       = 1;
+	d_shm_head->num_core      = cpu_count;
 	__sync_synchronize();
 	d_shm_head->magic = DSM_MAGIC;
 	/* initialization is finished now. */

src/gurt/shm_internal.h

@@ -61,6 +61,9 @@ struct d_shm_hdr {
 	uint64_t         size;
 	/* size of each shared memory allocator's pool */
 	uint64_t         shm_pool_size;
+	/* the number of physical cores on current node */
+	uint32_t         num_core;
+	uint32_t         pad;
 	/* reserved for future usage */
 	char             reserved[256];
 };
@@ -80,4 +83,84 @@ extern struct d_shm_hdr *d_shm_head;
 /* the total size of shared memory that will be allocated */
 #define SHM_SIZE_REQ (SHM_POOL_SIZE * N_SHM_FIXED_POOL + sizeof(struct d_shm_hdr))
 
+/* Node of entry in LRU cache */
+struct shm_lru_node {
+	/* key size*/
+	uint32_t    key_size;
+	/* data size*/
+	uint32_t    data_size;
+	/* store the offset to key */
+	long int    off_key;
+	/* store the offset to data */
+	long int    off_data;
+	/* the reference count of this record */
+	_Atomic int ref_count;
+	/* off_prev and off_next are used in doubly linked list for LRU */
+	int         off_prev;
+	int         off_next;
+	/* offset to the next node in hash chain in each bucket for allocated node. point to next
+	 * available node for free nodes
+	 */
+	int         off_hnext;
+	/* the index of hash bucket this record is in */
+	uint32_t    idx_bucket;
+	/* the index of sub-cache this record is in */
+	uint32_t    idx_subcache;
+};
+
+/* This implementation of shm LRU is mainly optimized for performance by using pre-allocated buffer
+ * when possible and fine grained lock
+ */
+
+typedef struct {
+	/* number of nodes allocated in this shard */
+	uint32_t      size;
+	/* Most recently used node */
+	int           off_head;
+	/* Least recently used node */
+	int           off_tail;
+	/* First available/free node */
+	int           first_av;
+	/* the offset to the array of offset of hash buckets */
+	long int      off_hashbuckets;
+	/* the offset to the array of preallocated array of nodes */
+	long int      off_nodelist;
+	/* the offset to the array of preallocated array of keys */
+	long int      off_keylist;
+	/* the offset to the array of preallocated array of data */
+	long int      off_datalist;
+
+	d_shm_mutex_t lock;
+	char          pad[8];
+} shm_lru_cache_var_t;
+
+/* LRU Cache structure */
+struct shm_lru_cache {
+	/* the number of sub-cache to use */
+	uint32_t n_subcache;
+	/* max number of nodes to hold in each shard */
+	uint32_t capacity_per_subcache;
+	/* the size of key. zero means key size is variable */
+	uint32_t key_size;
+	/* the size of data. zero means data size is variable */
+	uint32_t data_size;
+	/* 0 - dynamically allocate buffer for key, 1 - use pre-allocated buffer for key */
+	uint32_t prealloc_key;
+	/* 0 - dynamically allocate buffer for data, 1 - use pre-allocated buffer for data */
+	uint32_t prealloc_data;
+
+	/* number of bytes per sub-cache */
+	size_t   size_per_subcache;
+};
+
+/*
+ * data layout of LRU cache:
+ * 1) LRU Cache header (struct shm_lru_cache)
+ * 2) 1st sub-cache header (struct shm_lru_cache_var_t)
+ * 3) 1st sub-cache data (bucket, nodes, key, data)
+ * 4) 2nd sub-cache header (struct shm_lru_cache_var_t)
+ * 5) 2nd sub-cache data (bucket, nodes, key, data)
+ * 6) ...
+ */
+
 #endif