180 static void __table_instance_destroy(struct table_instance *ti)  in __table_instance_destroy()  argument
182 	free_buckets(ti->buckets);  in __table_instance_destroy()
183 	kfree(ti);  in __table_instance_destroy()
188 	struct table_instance *ti = kmalloc(sizeof(*ti), GFP_KERNEL);  in table_instance_alloc()  local
190 	if (!ti)  in table_instance_alloc()
193 	ti->buckets = alloc_buckets(new_size);  in table_instance_alloc()
195 	if (!ti->buckets) {  in table_instance_alloc()
196 		kfree(ti);  in table_instance_alloc()
199 	ti->n_buckets = new_size;  in table_instance_alloc()
200 	ti->node_ver = 0;  in table_instance_alloc()
201 	ti->keep_flows = false;  in table_instance_alloc()
202 	get_random_bytes(&ti->hash_seed, sizeof(u32));  in table_instance_alloc()
204 	return ti;  in table_instance_alloc()
209 	struct table_instance *ti, *ufid_ti;  in ovs_flow_tbl_init()  local
211 	ti = table_instance_alloc(TBL_MIN_BUCKETS);  in ovs_flow_tbl_init()
213 	if (!ti)  in ovs_flow_tbl_init()
220 	rcu_assign_pointer(table->ti, ti);  in ovs_flow_tbl_init()
229 	__table_instance_destroy(ti);  in ovs_flow_tbl_init()
235 	struct table_instance *ti = container_of(rcu, struct table_instance, rcu);  in flow_tbl_destroy_rcu_cb()  local
237 	__table_instance_destroy(ti);  in flow_tbl_destroy_rcu_cb()
240 static void table_instance_destroy(struct table_instance *ti,  in table_instance_destroy()  argument
246 	if (!ti)  in table_instance_destroy()
250 	if (ti->keep_flows)  in table_instance_destroy()
253 	for (i = 0; i < ti->n_buckets; i++) {  in table_instance_destroy()
255 		struct hlist_head *head = flex_array_get(ti->buckets, i);  in table_instance_destroy()
257 		int ver = ti->node_ver;  in table_instance_destroy()
270 		call_rcu(&ti->rcu, flow_tbl_destroy_rcu_cb);  in table_instance_destroy()
273 		__table_instance_destroy(ti);  in table_instance_destroy()
283 	struct table_instance *ti = rcu_dereference_raw(table->ti);  in ovs_flow_tbl_destroy()  local
286 	table_instance_destroy(ti, ufid_ti, false);  in ovs_flow_tbl_destroy()
289 struct sw_flow *ovs_flow_tbl_dump_next(struct table_instance *ti,  in ovs_flow_tbl_dump_next()  argument
297 	ver = ti->node_ver;  in ovs_flow_tbl_dump_next()
298 	while (*bucket < ti->n_buckets) {  in ovs_flow_tbl_dump_next()
300 		head = flex_array_get(ti->buckets, *bucket);  in ovs_flow_tbl_dump_next()
316 static struct hlist_head *find_bucket(struct table_instance *ti, u32 hash)  in find_bucket()  argument
318 	hash = jhash_1word(hash, ti->hash_seed);  in find_bucket()
319 	return flex_array_get(ti->buckets,  in find_bucket()
320 				(hash & (ti->n_buckets - 1)));  in find_bucket()
323 static void table_instance_insert(struct table_instance *ti,  in table_instance_insert()  argument
328 	head = find_bucket(ti, flow->flow_table.hash);  in table_instance_insert()
329 	hlist_add_head_rcu(&flow->flow_table.node[ti->node_ver], head);  in table_instance_insert()
332 static void ufid_table_instance_insert(struct table_instance *ti,  in ufid_table_instance_insert()  argument
337 	head = find_bucket(ti, flow->ufid_table.hash);  in ufid_table_instance_insert()
338 	hlist_add_head_rcu(&flow->ufid_table.node[ti->node_ver], head);  in ufid_table_instance_insert()
370 static struct table_instance *table_instance_rehash(struct table_instance *ti,  in table_instance_rehash()  argument
379 	flow_table_copy_flows(ti, new_ti, ufid);  in table_instance_rehash()
396 	old_ti = ovsl_dereference(flow_table->ti);  in ovs_flow_tbl_flush()
399 	rcu_assign_pointer(flow_table->ti, new_ti);  in ovs_flow_tbl_flush()
469 static struct sw_flow *masked_flow_lookup(struct table_instance *ti,  in masked_flow_lookup()  argument
480 	head = find_bucket(ti, hash);  in masked_flow_lookup()
481 	hlist_for_each_entry_rcu(flow, head, flow_table.node[ti->node_ver]) {  in masked_flow_lookup()
493 	struct table_instance *ti = rcu_dereference_ovsl(tbl->ti);  in ovs_flow_tbl_lookup_stats()  local
500 		flow = masked_flow_lookup(ti, key, mask);  in ovs_flow_tbl_lookup_stats()
518 	struct table_instance *ti = rcu_dereference_ovsl(tbl->ti);  in ovs_flow_tbl_lookup_exact()  local
524 		flow = masked_flow_lookup(ti, match->key, mask);  in ovs_flow_tbl_lookup_exact()
557 	struct table_instance *ti = rcu_dereference_ovsl(tbl->ufid_ti);  in ovs_flow_tbl_lookup_ufid()  local
563 	head = find_bucket(ti, hash);  in ovs_flow_tbl_lookup_ufid()
564 	hlist_for_each_entry_rcu(flow, head, ufid_table.node[ti->node_ver]) {  in ovs_flow_tbl_lookup_ufid()
583 static struct table_instance *table_instance_expand(struct table_instance *ti,  in table_instance_expand()  argument
586 	return table_instance_rehash(ti, ti->n_buckets * 2, ufid);  in table_instance_expand()
610 	struct table_instance *ti = ovsl_dereference(table->ti);  in ovs_flow_tbl_remove()  local
614 	hlist_del_rcu(&flow->flow_table.node[ti->node_ver]);  in ovs_flow_tbl_remove()
691 	struct table_instance *ti;  in flow_key_insert()  local
694 	ti = ovsl_dereference(table->ti);  in flow_key_insert()
695 	table_instance_insert(ti, flow);  in flow_key_insert()
699 	if (table->count > ti->n_buckets)  in flow_key_insert()
700 		new_ti = table_instance_expand(ti, false);  in flow_key_insert()
702 		new_ti = table_instance_rehash(ti, ti->n_buckets, false);  in flow_key_insert()
705 		rcu_assign_pointer(table->ti, new_ti);  in flow_key_insert()
706 		call_rcu(&ti->rcu, flow_tbl_destroy_rcu_cb);  in flow_key_insert()
714 	struct table_instance *ti;  in flow_ufid_insert()  local
717 	ti = ovsl_dereference(table->ufid_ti);  in flow_ufid_insert()
718 	ufid_table_instance_insert(ti, flow);  in flow_ufid_insert()
722 	if (table->ufid_count > ti->n_buckets) {  in flow_ufid_insert()
725 		new_ti = table_instance_expand(ti, true);  in flow_ufid_insert()
728 			call_rcu(&ti->rcu, flow_tbl_destroy_rcu_cb);  in flow_ufid_insert()