3 * Memory allocators debugging/test sip-router module.
5 * Copyright (C) 2010 iptelorg GmbH
7 * Permission to use, copy, modify, and distribute this software for any
8 * purpose with or without fee is hereby granted, provided that the above
9 * copyright notice and this permission notice appear in all copies.
11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
22 * 2010-03-10 initial version (andrei)
26 #include "../../sr_module.h"
27 #include "../../mem/mem.h"
28 #include "../../str.h"
29 #include "../../dprint.h"
30 #include "../../locking.h"
31 #include "../../atomic_ops.h"
32 #include "../../cfg/cfg.h"
33 #include "../../rpc.h"
34 #include "../../rand/fastrand.h"
35 #include "../../timer.h"
36 #include "../../mod_fix.h"
40 static int mt_mem_alloc_f(struct sip_msg*, char*,char*);
41 static int mt_mem_free_f(struct sip_msg*, char*,char*);
42 static int mod_init(void);
43 static void mod_destroy(void);
46 static cmd_export_t cmds[]={
47 {"mt_mem_alloc", mt_mem_alloc_f, 1, fixup_var_int_1,
48 REQUEST_ROUTE|ONREPLY_ROUTE|FAILURE_ROUTE|BRANCH_ROUTE|ONSEND_ROUTE},
49 {"mt_mem_free", mt_mem_free_f, 1, fixup_var_int_1,
50 REQUEST_ROUTE|ONREPLY_ROUTE|FAILURE_ROUTE|BRANCH_ROUTE|ONSEND_ROUTE},
56 struct cfg_group_malloc_test {
58 int realloc_p; /* realloc probability */
62 static struct cfg_group_malloc_test default_mt_cfg = {
63 0, /* check_content, off by default */
64 0 /* realloc probability, 0 by default */
67 static void * mt_cfg = &default_mt_cfg;
69 static cfg_def_t malloc_test_cfg_def[] = {
70 {"check_content", CFG_VAR_INT | CFG_ATOMIC, 0, 1, 0, 0,
71 "check if allocated memory was overwritten by filling it with "
72 "a special pattern and checking it on free."},
73 {"realloc_p", CFG_VAR_INT | CFG_ATOMIC, 0, 90, 0, 0,
74 "realloc probability in percents. During tests and mem_rnd_alloc"
75 " realloc_p percents of the allocations will be made by realloc'ing"
76 " and existing chunk. The maximum value is limited to 90, to avoid"
77 " very long mem_rnd_alloc runs (a realloc might also free memory)." },
83 static rpc_export_t mt_rpc[];
87 static param_export_t params[]={
88 {"check_content", PARAM_INT, &default_mt_cfg.check_content},
94 struct module_exports exports = {
97 mt_rpc, /* RPC methods */
99 mod_init, /* module initialization function */
100 0, /* response function*/
101 mod_destroy, /* destroy function */
102 0, /* oncancel function */
103 0 /* per-child init function */
108 #define MC_F_CHECK_CONTENTS 1
111 struct mem_chunk* next;
117 struct allocated_list {
118 struct mem_chunk* chunks;
124 struct allocated_list* alloc_lst;
127 struct rnd_time_test {
137 unsigned long reallocs;
140 struct rnd_time_test* next;
141 struct timer_ln timer;
145 struct rnd_time_test_lst {
146 struct rnd_time_test* tests;
148 volatile int last_id;
152 struct rnd_time_test_lst* rndt_lst;
154 static unsigned long mem_unleak(unsigned long size);
155 static void mem_destroy_all_tests();
157 static int mod_init(void)
159 WARN("This is a test/debugging module, don't use it in production\n");
160 /* declare configuration */
161 if (cfg_declare("malloc_test", malloc_test_cfg_def, &default_mt_cfg,
162 cfg_sizeof(malloc_test), &mt_cfg)){
163 ERR("failed to register the configuration\n");
167 alloc_lst = shm_malloc(sizeof(*alloc_lst));
170 alloc_lst->chunks = 0;
171 atomic_set_long(&alloc_lst->size, 0);
172 atomic_set_int(&alloc_lst->no, 0);
173 if (lock_init(&alloc_lst->lock) == 0)
175 rndt_lst = shm_malloc(sizeof(*rndt_lst));
179 atomic_set_int(&rndt_lst->last_id, 0);
180 if (lock_init(&rndt_lst->lock) == 0)
189 static void mod_destroy()
192 mem_destroy_all_tests();
193 lock_destroy(&rndt_lst->lock);
199 lock_destroy(&alloc_lst->lock);
207 /** record a memory chunk list entry.
208 * @param addr - address of the newly allocated memory
210 * @return 0 on success, -1 on error (no more mem).
212 static int mem_track(void* addr, unsigned long size)
214 struct mem_chunk* mc;
218 mc = shm_malloc(sizeof(*mc));
219 if (mc == 0) goto error;
223 if (cfg_get(malloc_test, mt_cfg, check_content)){
224 mc->flags |= MC_F_CHECK_CONTENTS;
226 for (r = 0; r < size/sizeof(*d); r++){
227 d[r]=~(unsigned long)&d[r];
229 for (i=0; i< size % sizeof(*d); i++){
230 ((char*)&d[r])[i]=~((unsigned long)&d[r] >> i*8);
233 lock_get(&alloc_lst->lock);
234 mc->next = alloc_lst->chunks;
235 alloc_lst->chunks = mc;
236 lock_release(&alloc_lst->lock);
237 atomic_add_long(&alloc_lst->size, size);
238 atomic_inc_int(&alloc_lst->no);
247 * Allocates memory, but keeps track of it, so that mem_unleak() can
249 * @param size - how many bytes
250 * @return 0 on success, -1 on error
252 static int mem_leak(unsigned long size)
256 d = shm_malloc(size);
258 if (mem_track(d, size) < 0){
268 /* realloc a chunk, unsafe (requires external locking) version.
269 * @return 0 on success, -1 on error
271 static int _mem_chunk_realloc_unsafe(struct mem_chunk *c, unsigned long size)
276 d = shm_realloc(c->addr, size);
278 if (cfg_get(malloc_test, mt_cfg, check_content) &&
279 c->flags & MC_F_CHECK_CONTENTS) {
280 /* re-fill the test patterns (the address might have changed
281 and they depend on it) */
282 for (r = 0; r < size/sizeof(*d); r++){
283 d[r]=~(unsigned long)&d[r];
285 for (i=0; i< size % sizeof(*d); i++){
286 ((char*)&d[r])[i]=~((unsigned long)&d[r] >> i*8);
298 static void mem_chunk_free(struct mem_chunk* c)
304 if (cfg_get(malloc_test, mt_cfg, check_content) &&
305 c->flags & MC_F_CHECK_CONTENTS) {
308 for (r = 0; r < c->size/sizeof(*d); r++){
309 if (d[r]!=~(unsigned long)&d[r])
311 d[r] = (unsigned long)&d[r]; /* fill it with something else */
313 for (i=0; i< c->size % sizeof(*d); i++){
314 if (((unsigned char*)&d[r])[i] !=
315 (unsigned char)~((unsigned long)&d[r] >> i*8))
317 ((char*)&d[r])[i] = (unsigned char)((unsigned long)&d[r] >> i*8);
320 ERR("%d errors while checking %ld bytes at %p\n", err, c->size, d);
330 * Frees previously allocated memory chunks until at least size bytes are
331 * released. Use -1 to free all,
332 * @param size - at least free size bytes.
333 * @return bytes_freed (>=0)
335 static unsigned long mem_unleak(unsigned long size)
337 struct mem_chunk** mc;
339 struct mem_chunk** min_chunk;
346 lock_get(&alloc_lst->lock);
347 if (size>=atomic_get_long(&alloc_lst->size)){
349 for (mc = &alloc_lst->chunks; *mc; ){
359 /* free at least size bytes, trying smaller chunks first */
360 for (mc = &alloc_lst->chunks; *mc && (freed < size);) {
361 if ((*mc)->size <= (size - freed)) {
369 } else if (min_chunk == 0 || (*min_chunk)->size > (*mc)->size) {
370 /* find minimum remaining chunk */
375 if (size > freed && min_chunk) {
385 lock_release(&alloc_lst->lock);
386 atomic_add_long(&alloc_lst->size, -freed);
387 atomic_add_int(&alloc_lst->no, -no);
393 /** realloc randomly size bytes.
394 * Chooses randomly a previously allocated chunk and realloc's it.
395 * @param size - size.
396 * @param diff - filled with difference, >= 0 means more bytes were alloc.,
397 * < 0 means bytes were freed.
398 * @return >= 0 on success, -1 on error/ not found
399 * (empty list is a valid error reason)
401 static int mem_rnd_realloc(unsigned long size, long* diff)
409 lock_get(&alloc_lst->lock);
410 target = fastrand_max(atomic_get_int(&alloc_lst->no));
411 for (t = alloc_lst->chunks, i=0; t; t=t->next, i++ ){
413 *diff = (long)size - (long)t->size;
414 if ((ret=_mem_chunk_realloc_unsafe(t, size)) < 0)
419 lock_release(&alloc_lst->lock);
420 atomic_add_long(&alloc_lst->size, *diff);
426 #define MIN_ulong(a, b) \
427 (unsigned long)((unsigned long)(a)<(unsigned long)(b)?(a):(b))
430 * Randomly alloc. total_size bytes, in chunks of size between
431 * min & max. max - min should be smaller then 4G.
432 * @return < 0 if there were some alloc errors, 0 on success.
434 static int mem_rnd_leak(unsigned long min, unsigned long max,
435 unsigned long total_size)
438 unsigned long crt_size, crt_min;
445 crt_min = MIN_ulong(min, size);
446 crt_size = fastrand_max(MIN_ulong(max, size) - crt_min) + crt_min;
447 p = cfg_get(malloc_test, mt_cfg, realloc_p);
448 if (p && ((fastrand_max(99) +1) <= p)){
449 if (mem_rnd_realloc(crt_size, &diff) == 0){
452 } /* else fallback to normal alloc. */
455 err += mem_leak(crt_size) < 0;
463 static ticks_t tst_timer(ticks_t ticks, struct timer_ln* tl, void* data)
465 struct rnd_time_test* tst;
468 unsigned long crt_size, crt_min, remaining;
477 if (tst->total <= tst->crt) {
478 mem_unleak(tst->crt);
482 remaining = tst->total - tst->crt;
483 crt_min = MIN_ulong(tst->min, remaining);
484 crt_size = fastrand_max(MIN_ulong(tst->max, remaining) - crt_min) +
486 p = cfg_get(malloc_test, mt_cfg, realloc_p);
487 if (p && ((fastrand_max(99) +1) <= p)) {
488 if (mem_rnd_realloc(crt_size, &diff) == 0){
494 if (mem_leak(crt_size) >= 0)
495 tst->crt += crt_size;
501 if (TICKS_GT(tst->stop_time, ticks)) {
502 next_int = fastrand_max(tst->max_intvrl - tst->min_intvrl) +
504 max_int = tst->stop_time - ticks;
507 WARN("test %d time expired, stopping"
508 " (%d s runtime, %ld calls, %d overfl, %d errors,"
510 tst->id, TICKS_TO_S(ticks - tst->start_time),
511 tst->calls, tst->overfl, tst->errs, tst->crt);
512 mem_unleak(tst->crt);
516 /* 0 means stop stop, so if next_int == 0 => stop */
517 return MIN_unsigned(next_int, max_int);
522 * start a malloc test of a test_time length:
523 * - randomly between min_intvrl and max_intvrl, alloc.
524 * a random number of bytes, between min & max.
525 * - if total_size is reached, free everything.
527 * @returns test id (>=0) on success, -1 on error.
529 static int mem_leak_time_test(unsigned long min, unsigned long max,
530 unsigned long total_size,
531 ticks_t min_intvrl, ticks_t max_intvrl,
534 struct rnd_time_test* tst;
535 struct rnd_time_test* l;
539 tst = shm_malloc(sizeof(*tst));
542 memset(tst, 0, sizeof(*tst));
543 id = tst->id = atomic_add_int(&rndt_lst->last_id, 1);
546 tst-> total = total_size;
547 tst->min_intvrl = min_intvrl;
548 tst->max_intvrl = max_intvrl;
549 tst->start_time = get_ticks_raw();
550 tst->stop_time = get_ticks_raw() + test_time;
551 first_int = fastrand_max(max_intvrl - min_intvrl) + min_intvrl;
552 timer_init(&tst->timer, tst_timer, tst, 0);
553 lock_get(&rndt_lst->lock);
554 tst->next=rndt_lst->tests;
556 lock_release(&rndt_lst->lock);
557 if (timer_add(&tst->timer, MIN_unsigned(first_int, test_time)) < 0 )
562 lock_get(&rndt_lst->lock);
563 for (l=rndt_lst->tests; l; l=l->next)
564 if (l->next == tst) {
568 lock_release(&rndt_lst->lock);
575 static int is_mem_test_stopped(struct rnd_time_test* tst)
577 return TICKS_LE(tst->stop_time, get_ticks_raw());
581 * @return 0 on success, -1 on error (test already stopped)
583 static int mem_test_stop_tst(struct rnd_time_test* tst)
585 if (!is_mem_test_stopped(tst)) {
586 if (timer_del(&tst->timer) == 0) {
587 tst->stop_time=get_ticks_raw();
596 * @return 0 on success, -1 on error (not found).
598 static int mem_test_stop(int id)
600 struct rnd_time_test* tst;
602 lock_get(&rndt_lst->lock);
603 for (tst = rndt_lst->tests; tst; tst = tst->next)
605 mem_test_stop_tst(tst);
608 lock_release(&rndt_lst->lock);
613 static void mem_destroy_all_tests()
615 struct rnd_time_test* tst;
616 struct rnd_time_test* nxt;
618 lock_get(&rndt_lst->lock);
619 for (tst = rndt_lst->tests; tst;) {
621 mem_test_stop_tst(tst);
626 lock_release(&rndt_lst->lock);
630 static int mem_test_destroy(int id)
632 struct rnd_time_test* tst;
633 struct rnd_time_test** crt_lnk;
635 lock_get(&rndt_lst->lock);
636 for (tst = 0, crt_lnk = &rndt_lst->tests; *crt_lnk;
637 crt_lnk = &(*crt_lnk)->next)
638 if ((*crt_lnk)->id == id) {
640 mem_test_stop_tst(tst);
645 lock_release(&rndt_lst->lock);
649 /* script functions: */
652 static int mt_mem_alloc_f(struct sip_msg* msg, char* sz, char* foo)
656 if (sz == 0 || get_int_fparam(&size, msg, (fparam_t*)sz) < 0)
658 return mem_leak(size)>=0?1:-1;
663 static int mt_mem_free_f(struct sip_msg* msg, char* sz, char* foo)
669 if (sz != 0 && get_int_fparam(&size, msg, (fparam_t*)sz) < 0)
671 freed=mem_unleak(size);
672 return (freed==0)?1:freed;
681 /* helper functions, parses an optional b[ytes]|k|m|g to a numeric shift value
682 (e.g. b -> 0, k -> 10, ...)
683 returns bit shift value on success, -1 on error
685 static int rpc_get_size_mod(rpc_t* rpc, void* c)
689 if (rpc->scan(c, "*s", &m) > 0) {
704 rpc->fault(c, 500, "bad param use b|k|m|g");
713 static const char* rpc_mt_alloc_doc[2] = {
714 "Allocates the specified number of bytes (debugging/test function)."
715 "Use b|k|m|g to specify the desired size unit",
719 static void rpc_mt_alloc(rpc_t* rpc, void* c)
724 if (rpc->scan(c, "d", &size) < 1) {
727 rs=rpc_get_size_mod(rpc, c);
729 /* fault already generated on rpc_get_size_mod() error */
731 if (mem_leak((unsigned long)size << rs) < 0) {
732 rpc->fault(c, 400, "memory allocation failed");
738 static const char* rpc_mt_realloc_doc[2] = {
739 "Reallocates the specified number of bytes from a pre-allocated"
740 " randomly selected memory chunk. If no pre-allocated memory"
741 " chunks exists, it will fail."
742 " Make sure mt.mem_used is non 0 or call mt.mem_alloc prior to calling"
744 " Returns the difference in bytes (<0 if bytes were freed, >0 if more"
745 " bytes were allocated)."
746 "Use b|k|m|g to specify the desired size unit",
750 static void rpc_mt_realloc(rpc_t* rpc, void* c)
756 if (rpc->scan(c, "d", &size) < 1) {
759 rs=rpc_get_size_mod(rpc, c);
761 /* fault already generated on rpc_get_size_mod() error */
763 if (mem_rnd_realloc((unsigned long)size << rs, &diff) < 0) {
764 rpc->fault(c, 400, "memory allocation failed");
766 rpc->add(c, "d", diff >> rs);
771 static const char* rpc_mt_free_doc[2] = {
772 "Frees the specified number of bytes, previously allocated by one of the"
773 " other malloc_test functions (e.g. mt.mem_alloc or the script "
774 "mt_mem_alloc). Use b|k|m|g to specify the desired size unit."
775 "Returns the number of bytes freed (can be higher or"
776 " smaller then the requested size)",
781 static void rpc_mt_free(rpc_t* rpc, void* c)
788 if (rpc->scan(c, "*d", &size) > 0) {
789 /* found size, look if a size modifier is present */
790 rs=rpc_get_size_mod(rpc, c);
792 /* fault already generated on rpc_get_size_mod() error */
795 rpc->add(c, "d", (int)(mem_unleak((unsigned long)size << rs) >> rs));
801 static const char* rpc_mt_used_doc[2] = {
802 "Returns how many memory chunks and how many bytes are currently"
803 " allocated via the mem_alloc module functions."
804 " Use b|k|m|g to specify the desired size unit.",
809 static void rpc_mt_used(rpc_t* rpc, void* c)
814 rs=rpc_get_size_mod(rpc, c);
816 /* fault already generated on rpc_get_size_mod() error */
818 rpc->add(c, "d", atomic_get_int(&alloc_lst->no));
819 rpc->add(c, "d", (int)(atomic_get_long(&alloc_lst->size) >> rs));
824 static const char* rpc_mt_rnd_alloc_doc[2] = {
825 "Takes 4 parameters: min, max, total_size and an optional unit (b|k|m|g)."
826 " It will allocate total_size memory, in pieces of random size between"
827 "min .. max (inclusive).",
832 static void rpc_mt_rnd_alloc(rpc_t* rpc, void* c)
834 int min, max, total_size;
838 if (rpc->scan(c, "ddd", &min, &max, &total_size) < 3) {
841 rs=rpc_get_size_mod(rpc, c);
843 /* fault already generated on rpc_get_size_mod() error */
845 if (min > max || min < 0 || max > total_size) {
846 rpc->fault(c, 400, "invalid parameter values");
849 if ((err=mem_rnd_leak((unsigned long)min << rs,
850 (unsigned long)max << rs,
851 (unsigned long)total_size <<rs )) < 0) {
852 rpc->fault(c, 400, "memory allocation failed (%d errors)", -err);
858 static const char* rpc_mt_test_start_doc[2] = {
859 "Takes 7 parameters: min, max, total_size, min_interval, max_interval, "
860 "test_time and an optional size unit (b|k|m|g). All the time units are ms."
861 " It will run a memory allocation test for test_time ms. At a random"
862 " interval between min_interval and max_interval ms. it will allocate a"
863 " memory chunk with random size, between min and max. Each time total_size"
864 " is reached, it will free all the memory allocated and start again."
865 "Returns the test id (integer)",
870 static void rpc_mt_test_start(rpc_t* rpc, void* c)
872 int min, max, total_size;
873 int min_intvrl, max_intvrl, total_time;
877 if (rpc->scan(c, "dddddd", &min, &max, &total_size,
878 &min_intvrl, &max_intvrl, &total_time) < 6) {
881 rs=rpc_get_size_mod(rpc, c);
883 /* fault already generated on rpc_get_size_mod() error */
885 if (min > max || min < 0 || max > total_size) {
886 rpc->fault(c, 400, "invalid size parameters values");
889 if (min_intvrl > max_intvrl || min_intvrl <= 0 || max_intvrl > total_time){
890 rpc->fault(c, 400, "invalid time intervals values");
893 if ((id=mem_leak_time_test((unsigned long)min << rs,
894 (unsigned long)max << rs,
895 (unsigned long)total_size <<rs,
896 MS_TO_TICKS(min_intvrl),
897 MS_TO_TICKS(max_intvrl),
898 MS_TO_TICKS(total_time)
900 rpc->fault(c, 400, "memory allocation failed");
902 rpc->add(c, "d", id);
908 static const char* rpc_mt_test_stop_doc[2] = {
909 "Takes 1 parameter: the test id. It will stop the corresponding test."
910 "Note: the test is stopped, but not destroyed." ,
915 static void rpc_mt_test_stop(rpc_t* rpc, void* c)
919 if (rpc->scan(c, "d", &id) < 1) {
922 if (mem_test_stop(id)<0) {
923 rpc->fault(c, 400, "test %d not found", id);
929 static const char* rpc_mt_test_destroy_doc[2] = {
930 "Takes 1 parameter: the test id. It will destroy the corresponding test.",
935 static void rpc_mt_test_destroy(rpc_t* rpc, void* c)
939 if (rpc->scan(c, "*d", &id) > 0 && id!=-1) {
940 if (mem_test_destroy(id) < 0 )
941 rpc->fault(c, 400, "test %d not found", id);
943 mem_destroy_all_tests();
949 static const char* rpc_mt_test_destroy_all_doc[2] = {
950 "It will destroy all the tests (running or stopped).",
955 static void rpc_mt_test_destroy_all(rpc_t* rpc, void* c)
957 mem_destroy_all_tests();
962 static const char* rpc_mt_test_list_doc[2] = {
963 "If a test id parameter is provided it will list the corresponding test,"
964 " else it will list all of them. Use b |k | m | g as a second parameter"
965 " for the size units (default bytes)",
970 static void rpc_mt_test_list(rpc_t* rpc, void* c)
973 struct rnd_time_test* tst;
977 if (rpc->scan(c, "*d", &id) < 1) {
980 rs=rpc_get_size_mod(rpc, c);
984 lock_get(&rndt_lst->lock);
985 for (tst = rndt_lst->tests; tst; tst=tst->next)
986 if (tst->id == id || id == -1) {
987 rpc->add(c, "{", &h);
988 rpc->struct_add(h, "ddddddddddd",
990 "run time (s) ", (int)TICKS_TO_S((
991 TICKS_LE(tst->stop_time,
993 tst->stop_time : get_ticks_raw()) -
995 "remaining (s)", TICKS_LE(tst->stop_time,
996 get_ticks_raw()) ? 0 :
997 (int)TICKS_TO_S(tst->stop_time -
999 "total calls ", (int)tst->calls,
1000 "reallocs ", (int)tst->reallocs,
1001 "errors ", (int)tst->errs,
1002 "overflows ", (int)tst->overfl,
1003 "total alloc ", (int)((tst->crt +
1004 tst->overfl * tst->total)>>rs),
1005 "min ", (int)(tst->min>>rs),
1006 "max ", (int)(tst->max>>rs),
1007 "total ", (int)(tst->total>>rs) );
1008 if (id != -1) break;
1010 lock_release(&rndt_lst->lock);
1016 static rpc_export_t mt_rpc[] = {
1017 {"mt.mem_alloc", rpc_mt_alloc, rpc_mt_alloc_doc, 0},
1018 {"mt.mem_free", rpc_mt_free, rpc_mt_free_doc, 0},
1019 {"mt.mem_realloc", rpc_mt_realloc, rpc_mt_realloc_doc, 0},
1020 {"mt.mem_used", rpc_mt_used, rpc_mt_used_doc, 0},
1021 {"mt.mem_rnd_alloc", rpc_mt_rnd_alloc, rpc_mt_rnd_alloc_doc, 0},
1022 {"mt.mem_test_start", rpc_mt_test_start, rpc_mt_test_start_doc, 0},
1023 {"mt.mem_test_stop", rpc_mt_test_stop, rpc_mt_test_stop_doc, 0},
1024 {"mt.mem_test_destroy", rpc_mt_test_destroy, rpc_mt_test_destroy_doc, 0},
1025 {"mt.mem_test_destroy_all", rpc_mt_test_destroy_all,
1026 rpc_mt_test_destroy_all_doc, 0},
1027 {"mt.mem_test_list", rpc_mt_test_list, rpc_mt_test_list_doc, 0},