summaryrefslogtreecommitdiffstats
path: root/qemu/qapi/introspect.json
blob: 3fd81fb5400a6e561f36fc71137358f0d2940122 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
# -*- Mode: Python -*-
#
# QAPI/QMP introspection
#
# Copyright (C) 2015 Red Hat, Inc.
#
# Authors:
#  Markus Armbruster <armbru@redhat.com>
#
# This work is licensed under the terms of the GNU GPL, version 2 or later.
# See the COPYING file in the top-level directory.

##
# @query-qmp-schema
#
# Command query-qmp-schema exposes the QMP wire ABI as an array of
# SchemaInfo.  This lets QMP clients figure out what commands and
# events are available in this QEMU, and their parameters and results.
#
# However, the SchemaInfo can't reflect all the rules and restrictions
# that apply to QMP.  It's interface introspection (figuring out
# what's there), not interface specification.  The specification is in
# the QAPI schema.
#
# Furthermore, while we strive to keep the QMP wire format
# backwards-compatible across qemu versions, the introspection output
# is not guaranteed to have the same stability.  For example, one
# version of qemu may list an object member as an optional
# non-variant, while another lists the same member only through the
# object's variants; or the type of a member may change from a generic
# string into a specific enum or from one specific type into an
# alternate that includes the original type alongside something else.
#
# Returns: array of @SchemaInfo, where each element describes an
# entity in the ABI: command, event, type, ...
#
# The order of the various SchemaInfo is unspecified; however, all
# names are guaranteed to be unique (no name will be duplicated with
# different meta-types).
#
# Note: the QAPI schema is also used to help define *internal*
# interfaces, by defining QAPI types.  These are not part of the QMP
# wire ABI, and therefore not returned by this command.
#
# Since: 2.5
##
{ 'command': 'query-qmp-schema',
  'returns': [ 'SchemaInfo' ],
  'gen': false }                # just to simplify qmp_query_json()

##
# @SchemaMetaType
#
# This is a @SchemaInfo's meta type, i.e. the kind of entity it
# describes.
#
# @builtin: a predefined type such as 'int' or 'bool'.
#
# @enum: an enumeration type
#
# @array: an array type
#
# @object: an object type (struct or union)
#
# @alternate: an alternate type
#
# @command: a QMP command
#
# @event: a QMP event
#
# Since: 2.5
##
{ 'enum': 'SchemaMetaType',
  'data': [ 'builtin', 'enum', 'array', 'object', 'alternate',
            'command', 'event' ] }

##
# @SchemaInfo
#
# @name: the entity's name, inherited from @base.
#        Commands and events have the name defined in the QAPI schema.
#        Unlike command and event names, type names are not part of
#        the wire ABI.  Consequently, type names are meaningless
#        strings here, although they are still guaranteed unique
#        regardless of @meta-type.
#
# All references to other SchemaInfo are by name.
#
# @meta-type: the entity's meta type, inherited from @base.
#
# Additional members depend on the value of @meta-type.
#
# Since: 2.5
##
{ 'union': 'SchemaInfo',
  'base': { 'name': 'str', 'meta-type': 'SchemaMetaType' },
  'discriminator': 'meta-type',
  'data': {
      'builtin': 'SchemaInfoBuiltin',
      'enum': 'SchemaInfoEnum',
      'array': 'SchemaInfoArray',
      'object': 'SchemaInfoObject',
      'alternate': 'SchemaInfoAlternate',
      'command': 'SchemaInfoCommand',
      'event': 'SchemaInfoEvent' } }

##
# @SchemaInfoBuiltin
#
# Additional SchemaInfo members for meta-type 'builtin'.
#
# @json-type: the JSON type used for this type on the wire.
#
# Since: 2.5
##
{ 'struct': 'SchemaInfoBuiltin',
  'data': { 'json-type': 'JSONType' } }

##
# @JSONType
#
# The four primitive and two structured types according to RFC 7159
# section 1, plus 'int' (split off 'number'), plus the obvious top
# type 'value'.
#
# Since: 2.5
##
{ 'enum': 'JSONType',
  'data': [ 'string', 'number', 'int', 'boolean', 'null',
            'object', 'array', 'value' ] }

##
# @SchemaInfoEnum
#
# Additional SchemaInfo members for meta-type 'enum'.
#
# @values: the enumeration type's values, in no particular order.
#
# Values of this type are JSON string on the wire.
#
# Since: 2.5
##
{ 'struct': 'SchemaInfoEnum',
  'data': { 'values': ['str'] } }

##
# @SchemaInfoArray
#
# Additional SchemaInfo members for meta-type 'array'.
#
# @element-type: the array type's element type.
#
# Values of this type are JSON array on the wire.
#
# Since: 2.5
##
{ 'struct': 'SchemaInfoArray',
  'data': { 'element-type': 'str' } }

##
# @SchemaInfoObject
#
# Additional SchemaInfo members for meta-type 'object'.
#
# @members: the object type's (non-variant) members, in no particular order.
#
# @tag: #optional the name of the member serving as type tag.
#       An element of @members with this name must exist.
#
# @variants: #optional variant members, i.e. additional members that
#            depend on the type tag's value.  Present exactly when
#            @tag is present.  The variants are in no particular order,
#            and may even differ from the order of the values of the
#            enum type of the @tag.
#
# Values of this type are JSON object on the wire.
#
# Since: 2.5
##
{ 'struct': 'SchemaInfoObject',
  'data': { 'members': [ 'SchemaInfoObjectMember' ],
            '*tag': 'str',
            '*variants': [ 'SchemaInfoObjectVariant' ] } }

##
# @SchemaInfoObjectMember
#
# An object member.
#
# @name: the member's name, as defined in the QAPI schema.
#
# @type: the name of the member's type.
#
# @default: #optional default when used as command parameter.
#           If absent, the parameter is mandatory.
#           If present, the value must be null.  The parameter is
#           optional, and behavior when it's missing is not specified
#           here.
#           Future extension: if present and non-null, the parameter
#           is optional, and defaults to this value.
#
# Since: 2.5
##
{ 'struct': 'SchemaInfoObjectMember',
  'data': { 'name': 'str', 'type': 'str', '*default': 'any' } }
# @default's type must be null or match @type

##
# @SchemaInfoObjectVariant
#
# The variant members for a value of the type tag.
#
# @case: a value of the type tag.
#
# @type: the name of the object type that provides the variant members
#        when the type tag has value @case.
#
# Since: 2.5
##
{ 'struct': 'SchemaInfoObjectVariant',
  'data': { 'case': 'str', 'type': 'str' } }

##
# @SchemaInfoAlternate
#
# Additional SchemaInfo members for meta-type 'alternate'.
#
# @members: the alternate type's members, in no particular order.
#           The members' wire encoding is distinct, see
#           docs/qapi-code-gen.txt section Alternate types.
#
# On the wire, this can be any of the members.
#
# Since: 2.5
##
{ 'struct': 'SchemaInfoAlternate',
  'data': { 'members': [ 'SchemaInfoAlternateMember' ] } }

##
# @SchemaInfoAlternateMember
#
# An alternate member.
#
# @type: the name of the member's type.
#
# Since: 2.5
##
{ 'struct': 'SchemaInfoAlternateMember',
  'data': { 'type': 'str' } }

##
# @SchemaInfoCommand
#
# Additional SchemaInfo members for meta-type 'command'.
#
# @arg-type: the name of the object type that provides the command's
#            parameters.
#
# @ret-type: the name of the command's result type.
#
# TODO @success-response (currently irrelevant, because it's QGA, not QMP)
#
# Since: 2.5
##
{ 'struct': 'SchemaInfoCommand',
  'data': { 'arg-type': 'str', 'ret-type': 'str' } }

##
# @SchemaInfoEvent
#
# Additional SchemaInfo members for meta-type 'event'.
#
# @arg-type: the name of the object type that provides the event's
#            parameters.
#
# Since: 2.5
##
{ 'struct': 'SchemaInfoEvent',
  'data': { 'arg-type': 'str' } }
/span> ic->rds_ibdev = rds_ibdev; atomic_inc(&rds_ibdev->refcount); } void rds_ib_remove_conn(struct rds_ib_device *rds_ibdev, struct rds_connection *conn) { struct rds_ib_connection *ic = conn->c_transport_data; /* place conn on nodev_conns_list */ spin_lock(&ib_nodev_conns_lock); spin_lock_irq(&rds_ibdev->spinlock); BUG_ON(list_empty(&ic->ib_node)); list_del(&ic->ib_node); spin_unlock_irq(&rds_ibdev->spinlock); list_add_tail(&ic->ib_node, &ib_nodev_conns); spin_unlock(&ib_nodev_conns_lock); ic->rds_ibdev = NULL; rds_ib_dev_put(rds_ibdev); } void rds_ib_destroy_nodev_conns(void) { struct rds_ib_connection *ic, *_ic; LIST_HEAD(tmp_list); /* avoid calling conn_destroy with irqs off */ spin_lock_irq(&ib_nodev_conns_lock); list_splice(&ib_nodev_conns, &tmp_list); spin_unlock_irq(&ib_nodev_conns_lock); list_for_each_entry_safe(ic, _ic, &tmp_list, ib_node) rds_conn_destroy(ic->conn); } struct rds_ib_mr_pool *rds_ib_create_mr_pool(struct rds_ib_device *rds_ibdev, int pool_type) { struct rds_ib_mr_pool *pool; pool = kzalloc(sizeof(*pool), GFP_KERNEL); if (!pool) return ERR_PTR(-ENOMEM); pool->pool_type = pool_type; init_llist_head(&pool->free_list); init_llist_head(&pool->drop_list); init_llist_head(&pool->clean_list); mutex_init(&pool->flush_lock); init_waitqueue_head(&pool->flush_wait); INIT_DELAYED_WORK(&pool->flush_worker, rds_ib_mr_pool_flush_worker); if (pool_type == RDS_IB_MR_1M_POOL) { /* +1 allows for unaligned MRs */ pool->fmr_attr.max_pages = RDS_FMR_1M_MSG_SIZE + 1; pool->max_items = RDS_FMR_1M_POOL_SIZE; } else { /* pool_type == RDS_IB_MR_8K_POOL */ pool->fmr_attr.max_pages = RDS_FMR_8K_MSG_SIZE + 1; pool->max_items = RDS_FMR_8K_POOL_SIZE; } pool->max_free_pinned = pool->max_items * pool->fmr_attr.max_pages / 4; pool->fmr_attr.max_maps = rds_ibdev->fmr_max_remaps; pool->fmr_attr.page_shift = PAGE_SHIFT; pool->max_items_soft = rds_ibdev->max_fmrs * 3 / 4; return pool; } void rds_ib_get_mr_info(struct rds_ib_device *rds_ibdev, struct rds_info_rdma_connection *iinfo) { struct rds_ib_mr_pool *pool_1m = rds_ibdev->mr_1m_pool; iinfo->rdma_mr_max = pool_1m->max_items; iinfo->rdma_mr_size = pool_1m->fmr_attr.max_pages; } void rds_ib_destroy_mr_pool(struct rds_ib_mr_pool *pool) { cancel_delayed_work_sync(&pool->flush_worker); rds_ib_flush_mr_pool(pool, 1, NULL); WARN_ON(atomic_read(&pool->item_count)); WARN_ON(atomic_read(&pool->free_pinned)); kfree(pool); } static inline struct rds_ib_mr *rds_ib_reuse_fmr(struct rds_ib_mr_pool *pool) { struct rds_ib_mr *ibmr = NULL; struct llist_node *ret; unsigned long *flag; preempt_disable(); flag = this_cpu_ptr(&clean_list_grace); set_bit(CLEAN_LIST_BUSY_BIT, flag); ret = llist_del_first(&pool->clean_list); if (ret) ibmr = llist_entry(ret, struct rds_ib_mr, llnode); clear_bit(CLEAN_LIST_BUSY_BIT, flag); preempt_enable(); return ibmr; } static inline void wait_clean_list_grace(void) { int cpu; unsigned long *flag; for_each_online_cpu(cpu) { flag = &per_cpu(clean_list_grace, cpu); while (test_bit(CLEAN_LIST_BUSY_BIT, flag)) cpu_chill(); } } static struct rds_ib_mr *rds_ib_alloc_fmr(struct rds_ib_device *rds_ibdev, int npages) { struct rds_ib_mr_pool *pool; struct rds_ib_mr *ibmr = NULL; int err = 0, iter = 0; if (npages <= RDS_FMR_8K_MSG_SIZE) pool = rds_ibdev->mr_8k_pool; else pool = rds_ibdev->mr_1m_pool; if (atomic_read(&pool->dirty_count) >= pool->max_items / 10) queue_delayed_work(rds_ib_fmr_wq, &pool->flush_worker, 10); /* Switch pools if one of the pool is reaching upper limit */ if (atomic_read(&pool->dirty_count) >= pool->max_items * 9 / 10) { if (pool->pool_type == RDS_IB_MR_8K_POOL) pool = rds_ibdev->mr_1m_pool; else pool = rds_ibdev->mr_8k_pool; } while (1) { ibmr = rds_ib_reuse_fmr(pool); if (ibmr) return ibmr; /* No clean MRs - now we have the choice of either * allocating a fresh MR up to the limit imposed by the * driver, or flush any dirty unused MRs. * We try to avoid stalling in the send path if possible, * so we allocate as long as we're allowed to. * * We're fussy with enforcing the FMR limit, though. If the driver * tells us we can't use more than N fmrs, we shouldn't start * arguing with it */ if (atomic_inc_return(&pool->item_count) <= pool->max_items) break; atomic_dec(&pool->item_count); if (++iter > 2) { if (pool->pool_type == RDS_IB_MR_8K_POOL) rds_ib_stats_inc(s_ib_rdma_mr_8k_pool_depleted); else rds_ib_stats_inc(s_ib_rdma_mr_1m_pool_depleted); return ERR_PTR(-EAGAIN); } /* We do have some empty MRs. Flush them out. */ if (pool->pool_type == RDS_IB_MR_8K_POOL) rds_ib_stats_inc(s_ib_rdma_mr_8k_pool_wait); else rds_ib_stats_inc(s_ib_rdma_mr_1m_pool_wait); rds_ib_flush_mr_pool(pool, 0, &ibmr); if (ibmr) return ibmr; } ibmr = kzalloc_node(sizeof(*ibmr), GFP_KERNEL, rdsibdev_to_node(rds_ibdev)); if (!ibmr) { err = -ENOMEM; goto out_no_cigar; } ibmr->fmr = ib_alloc_fmr(rds_ibdev->pd, (IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_READ | IB_ACCESS_REMOTE_WRITE| IB_ACCESS_REMOTE_ATOMIC), &pool->fmr_attr); if (IS_ERR(ibmr->fmr)) { err = PTR_ERR(ibmr->fmr); ibmr->fmr = NULL; printk(KERN_WARNING "RDS/IB: ib_alloc_fmr failed (err=%d)\n", err); goto out_no_cigar; } ibmr->pool = pool; if (pool->pool_type == RDS_IB_MR_8K_POOL) rds_ib_stats_inc(s_ib_rdma_mr_8k_alloc); else rds_ib_stats_inc(s_ib_rdma_mr_1m_alloc); return ibmr; out_no_cigar: if (ibmr) { if (ibmr->fmr) ib_dealloc_fmr(ibmr->fmr); kfree(ibmr); } atomic_dec(&pool->item_count); return ERR_PTR(err); } static int rds_ib_map_fmr(struct rds_ib_device *rds_ibdev, struct rds_ib_mr *ibmr, struct scatterlist *sg, unsigned int nents) { struct ib_device *dev = rds_ibdev->dev; struct scatterlist *scat = sg; u64 io_addr = 0; u64 *dma_pages; u32 len; int page_cnt, sg_dma_len; int i, j; int ret; sg_dma_len = ib_dma_map_sg(dev, sg, nents, DMA_BIDIRECTIONAL); if (unlikely(!sg_dma_len)) { printk(KERN_WARNING "RDS/IB: dma_map_sg failed!\n"); return -EBUSY; } len = 0; page_cnt = 0; for (i = 0; i < sg_dma_len; ++i) { unsigned int dma_len = ib_sg_dma_len(dev, &scat[i]); u64 dma_addr = ib_sg_dma_address(dev, &scat[i]); if (dma_addr & ~PAGE_MASK) { if (i > 0) return -EINVAL; else ++page_cnt; } if ((dma_addr + dma_len) & ~PAGE_MASK) { if (i < sg_dma_len - 1) return -EINVAL; else ++page_cnt; } len += dma_len; } page_cnt += len >> PAGE_SHIFT; if (page_cnt > ibmr->pool->fmr_attr.max_pages) return -EINVAL; dma_pages = kmalloc_node(sizeof(u64) * page_cnt, GFP_ATOMIC, rdsibdev_to_node(rds_ibdev)); if (!dma_pages) return -ENOMEM; page_cnt = 0; for (i = 0; i < sg_dma_len; ++i) { unsigned int dma_len = ib_sg_dma_len(dev, &scat[i]); u64 dma_addr = ib_sg_dma_address(dev, &scat[i]); for (j = 0; j < dma_len; j += PAGE_SIZE) dma_pages[page_cnt++] = (dma_addr & PAGE_MASK) + j; } ret = ib_map_phys_fmr(ibmr->fmr, dma_pages, page_cnt, io_addr); if (ret) goto out; /* Success - we successfully remapped the MR, so we can * safely tear down the old mapping. */ rds_ib_teardown_mr(ibmr); ibmr->sg = scat; ibmr->sg_len = nents; ibmr->sg_dma_len = sg_dma_len; ibmr->remap_count++; if (ibmr->pool->pool_type == RDS_IB_MR_8K_POOL) rds_ib_stats_inc(s_ib_rdma_mr_8k_used); else rds_ib_stats_inc(s_ib_rdma_mr_1m_used); ret = 0; out: kfree(dma_pages); return ret; } void rds_ib_sync_mr(void *trans_private, int direction) { struct rds_ib_mr *ibmr = trans_private; struct rds_ib_device *rds_ibdev = ibmr->device; switch (direction) { case DMA_FROM_DEVICE: ib_dma_sync_sg_for_cpu(rds_ibdev->dev, ibmr->sg, ibmr->sg_dma_len, DMA_BIDIRECTIONAL); break; case DMA_TO_DEVICE: ib_dma_sync_sg_for_device(rds_ibdev->dev, ibmr->sg, ibmr->sg_dma_len, DMA_BIDIRECTIONAL); break; } } static void __rds_ib_teardown_mr(struct rds_ib_mr *ibmr) { struct rds_ib_device *rds_ibdev = ibmr->device; if (ibmr->sg_dma_len) { ib_dma_unmap_sg(rds_ibdev->dev, ibmr->sg, ibmr->sg_len, DMA_BIDIRECTIONAL); ibmr->sg_dma_len = 0; } /* Release the s/g list */ if (ibmr->sg_len) { unsigned int i; for (i = 0; i < ibmr->sg_len; ++i) { struct page *page = sg_page(&ibmr->sg[i]); /* FIXME we need a way to tell a r/w MR * from a r/o MR */ WARN_ON(!page->mapping && irqs_disabled()); set_page_dirty(page); put_page(page); } kfree(ibmr->sg); ibmr->sg = NULL; ibmr->sg_len = 0; } } static void rds_ib_teardown_mr(struct rds_ib_mr *ibmr) { unsigned int pinned = ibmr->sg_len; __rds_ib_teardown_mr(ibmr); if (pinned) { struct rds_ib_mr_pool *pool = ibmr->pool; atomic_sub(pinned, &pool->free_pinned); } } static inline unsigned int rds_ib_flush_goal(struct rds_ib_mr_pool *pool, int free_all) { unsigned int item_count; item_count = atomic_read(&pool->item_count); if (free_all) return item_count; return 0; } /* * given an llist of mrs, put them all into the list_head for more processing */ static unsigned int llist_append_to_list(struct llist_head *llist, struct list_head *list) { struct rds_ib_mr *ibmr; struct llist_node *node; struct llist_node *next; unsigned int count = 0; node = llist_del_all(llist); while (node) { next = node->next; ibmr = llist_entry(node, struct rds_ib_mr, llnode); list_add_tail(&ibmr->unmap_list, list); node = next; count++; } return count; } /* * this takes a list head of mrs and turns it into linked llist nodes * of clusters. Each cluster has linked llist nodes of * MR_CLUSTER_SIZE mrs that are ready for reuse. */ static void list_to_llist_nodes(struct rds_ib_mr_pool *pool, struct list_head *list, struct llist_node **nodes_head, struct llist_node **nodes_tail) { struct rds_ib_mr *ibmr; struct llist_node *cur = NULL; struct llist_node **next = nodes_head; list_for_each_entry(ibmr, list, unmap_list) { cur = &ibmr->llnode; *next = cur; next = &cur->next; } *next = NULL; *nodes_tail = cur; } /* * Flush our pool of MRs. * At a minimum, all currently unused MRs are unmapped. * If the number of MRs allocated exceeds the limit, we also try * to free as many MRs as needed to get back to this limit. */ static int rds_ib_flush_mr_pool(struct rds_ib_mr_pool *pool, int free_all, struct rds_ib_mr **ibmr_ret) { struct rds_ib_mr *ibmr, *next; struct llist_node *clean_nodes; struct llist_node *clean_tail; LIST_HEAD(unmap_list); LIST_HEAD(fmr_list); unsigned long unpinned = 0; unsigned int nfreed = 0, dirty_to_clean = 0, free_goal; int ret = 0; if (pool->pool_type == RDS_IB_MR_8K_POOL) rds_ib_stats_inc(s_ib_rdma_mr_8k_pool_flush); else rds_ib_stats_inc(s_ib_rdma_mr_1m_pool_flush); if (ibmr_ret) { DEFINE_WAIT(wait); while (!mutex_trylock(&pool->flush_lock)) { ibmr = rds_ib_reuse_fmr(pool); if (ibmr) { *ibmr_ret = ibmr; finish_wait(&pool->flush_wait, &wait); goto out_nolock; } prepare_to_wait(&pool->flush_wait, &wait, TASK_UNINTERRUPTIBLE); if (llist_empty(&pool->clean_list)) schedule(); ibmr = rds_ib_reuse_fmr(pool); if (ibmr) { *ibmr_ret = ibmr; finish_wait(&pool->flush_wait, &wait); goto out_nolock; } } finish_wait(&pool->flush_wait, &wait); } else mutex_lock(&pool->flush_lock); if (ibmr_ret) { ibmr = rds_ib_reuse_fmr(pool); if (ibmr) { *ibmr_ret = ibmr; goto out; } } /* Get the list of all MRs to be dropped. Ordering matters - * we want to put drop_list ahead of free_list. */ dirty_to_clean = llist_append_to_list(&pool->drop_list, &unmap_list); dirty_to_clean += llist_append_to_list(&pool->free_list, &unmap_list); if (free_all) llist_append_to_list(&pool->clean_list, &unmap_list); free_goal = rds_ib_flush_goal(pool, free_all); if (list_empty(&unmap_list)) goto out; /* String all ib_mr's onto one list and hand them to ib_unmap_fmr */ list_for_each_entry(ibmr, &unmap_list, unmap_list) list_add(&ibmr->fmr->list, &fmr_list); ret = ib_unmap_fmr(&fmr_list); if (ret) printk(KERN_WARNING "RDS/IB: ib_unmap_fmr failed (err=%d)\n", ret); /* Now we can destroy the DMA mapping and unpin any pages */ list_for_each_entry_safe(ibmr, next, &unmap_list, unmap_list) { unpinned += ibmr->sg_len; __rds_ib_teardown_mr(ibmr); if (nfreed < free_goal || ibmr->remap_count >= pool->fmr_attr.max_maps) { if (ibmr->pool->pool_type == RDS_IB_MR_8K_POOL) rds_ib_stats_inc(s_ib_rdma_mr_8k_free); else rds_ib_stats_inc(s_ib_rdma_mr_1m_free); list_del(&ibmr->unmap_list); ib_dealloc_fmr(ibmr->fmr); kfree(ibmr); nfreed++; } } if (!list_empty(&unmap_list)) { /* we have to make sure that none of the things we're about * to put on the clean list would race with other cpus trying * to pull items off. The llist would explode if we managed to * remove something from the clean list and then add it back again * while another CPU was spinning on that same item in llist_del_first. * * This is pretty unlikely, but just in case wait for an llist grace period * here before adding anything back into the clean list. */ wait_clean_list_grace(); list_to_llist_nodes(pool, &unmap_list, &clean_nodes, &clean_tail); if (ibmr_ret) *ibmr_ret = llist_entry(clean_nodes, struct rds_ib_mr, llnode); /* more than one entry in llist nodes */ if (clean_nodes->next) llist_add_batch(clean_nodes->next, clean_tail, &pool->clean_list); } atomic_sub(unpinned, &pool->free_pinned); atomic_sub(dirty_to_clean, &pool->dirty_count); atomic_sub(nfreed, &pool->item_count); out: mutex_unlock(&pool->flush_lock); if (waitqueue_active(&pool->flush_wait)) wake_up(&pool->flush_wait); out_nolock: return ret; } static void rds_ib_mr_pool_flush_worker(struct work_struct *work) { struct rds_ib_mr_pool *pool = container_of(work, struct rds_ib_mr_pool, flush_worker.work); rds_ib_flush_mr_pool(pool, 0, NULL); } void rds_ib_free_mr(void *trans_private, int invalidate) { struct rds_ib_mr *ibmr = trans_private; struct rds_ib_mr_pool *pool = ibmr->pool; struct rds_ib_device *rds_ibdev = ibmr->device; rdsdebug("RDS/IB: free_mr nents %u\n", ibmr->sg_len); /* Return it to the pool's free list */ if (ibmr->remap_count >= pool->fmr_attr.max_maps) llist_add(&ibmr->llnode, &pool->drop_list); else llist_add(&ibmr->llnode, &pool->free_list); atomic_add(ibmr->sg_len, &pool->free_pinned); atomic_inc(&pool->dirty_count); /* If we've pinned too many pages, request a flush */ if (atomic_read(&pool->free_pinned) >= pool->max_free_pinned || atomic_read(&pool->dirty_count) >= pool->max_items / 5) queue_delayed_work(rds_ib_fmr_wq, &pool->flush_worker, 10); if (invalidate) { if (likely(!in_interrupt())) { rds_ib_flush_mr_pool(pool, 0, NULL); } else { /* We get here if the user created a MR marked * as use_once and invalidate at the same time. */ queue_delayed_work(rds_ib_fmr_wq, &pool->flush_worker, 10); } } rds_ib_dev_put(rds_ibdev); } void rds_ib_flush_mrs(void) { struct rds_ib_device *rds_ibdev; down_read(&rds_ib_devices_lock); list_for_each_entry(rds_ibdev, &rds_ib_devices, list) { if (rds_ibdev->mr_8k_pool) rds_ib_flush_mr_pool(rds_ibdev->mr_8k_pool, 0, NULL); if (rds_ibdev->mr_1m_pool) rds_ib_flush_mr_pool(rds_ibdev->mr_1m_pool, 0, NULL); } up_read(&rds_ib_devices_lock); } void *rds_ib_get_mr(struct scatterlist *sg, unsigned long nents, struct rds_sock *rs, u32 *key_ret) { struct rds_ib_device *rds_ibdev; struct rds_ib_mr *ibmr = NULL; int ret; rds_ibdev = rds_ib_get_device(rs->rs_bound_addr); if (!rds_ibdev) { ret = -ENODEV; goto out; } if (!rds_ibdev->mr_8k_pool || !rds_ibdev->mr_1m_pool) { ret = -ENODEV; goto out; } ibmr = rds_ib_alloc_fmr(rds_ibdev, nents); if (IS_ERR(ibmr)) { rds_ib_dev_put(rds_ibdev); return ibmr; } ret = rds_ib_map_fmr(rds_ibdev, ibmr, sg, nents); if (ret == 0) *key_ret = ibmr->fmr->rkey; else printk(KERN_WARNING "RDS/IB: map_fmr failed (errno=%d)\n", ret); ibmr->device = rds_ibdev; rds_ibdev = NULL; out: if (ret) { if (ibmr) rds_ib_free_mr(ibmr, 0); ibmr = ERR_PTR(ret); } if (rds_ibdev) rds_ib_dev_put(rds_ibdev); return ibmr; }