* [PATCH 0/3] block: fix the lock inversion caused by kobject_del under sysfs_lock
@ 2018-06-22 7:11 Jianchao Wang
2018-06-22 7:11 ` [PATCH 1/3] block: add helper interface blk_queue_registered Jianchao Wang
` (2 more replies)
0 siblings, 3 replies; 4+ messages in thread
From: Jianchao Wang @ 2018-06-22 7:11 UTC (permalink / raw)
To: axboe; +Cc: bart.vanassche, linux-block, linux-kernel
Hi Jens
Currently, the kobject_del for sysfs mq and hctx entry is invoked
under q->sysfs_lock. lock inversion will came up when someone try
to access the sysfs entries concurrently. The scenario is as below:
hold q->sysfs_lock access mq,hctx sysfs entries
kobject_del
__kernfs_remove kernfs_get_active
kernfs_drain
wait kn->active require q->sysfs_lock
To avoid this issue, kobject_del for mq and hctx sysfs entries
mustn't be inovked under sysfs_lock. These patch set is to achieve this.
1st patch introduces blk_queue_registered to test QUEUE_FLAG_REGISTERED
2nd patch remove blk_mq_register_dev which is usesless.
3rd patch fix the issue by changing the sync method, please refer to the
comment.
Jianchao Wang (3)
block: add helper interface blk_queue_registered
blk-mq: cleanup blk_mq_register_dev
block: fix the lock inversion caused by kobject_del under sysfs_lock
block/blk-mq-sysfs.c | 80 +++++++++++++++++++++++++++++++-------------------
block/blk-mq.h | 3 ++
block/blk-sysfs.c | 43 ++++++++++++++-------------
block/blk-wbt.c | 4 ---
block/elevator.c | 5 ++--
include/linux/blk-mq.h | 3 --
include/linux/blkdev.h | 4 ++-
7 files changed, 82 insertions(+), 60 deletions(-)
Thanks
Jianchao
^ permalink raw reply [flat|nested] 4+ messages in thread
* [PATCH 1/3] block: add helper interface blk_queue_registered
2018-06-22 7:11 [PATCH 0/3] block: fix the lock inversion caused by kobject_del under sysfs_lock Jianchao Wang
@ 2018-06-22 7:11 ` Jianchao Wang
2018-06-22 7:11 ` [PATCH 2/3] blk-mq: cleanup blk_mq_register_dev Jianchao Wang
2018-06-22 7:11 ` [PATCH 3/3] block: fix the lock inversion caused by kobject_del under sysfs_lock Jianchao Wang
2 siblings, 0 replies; 4+ messages in thread
From: Jianchao Wang @ 2018-06-22 7:11 UTC (permalink / raw)
To: axboe; +Cc: bart.vanassche, linux-block, linux-kernel
Add blk_queue_registered to check whether QUEUE_FLAG_REGISTERED
is set. And also replace the positions where test it with test_bit
directly.
Signed-off-by: Jianchao Wang <jianchao.w.wang@oracle.com>
---
block/blk-sysfs.c | 4 ++--
block/blk-wbt.c | 2 +-
block/elevator.c | 2 +-
include/linux/blkdev.h | 3 +++
4 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
index 94987b1..159c586 100644
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -863,7 +863,7 @@ int blk_register_queue(struct gendisk *disk)
if (WARN_ON(!q))
return -ENXIO;
- WARN_ONCE(test_bit(QUEUE_FLAG_REGISTERED, &q->queue_flags),
+ WARN_ONCE(blk_queue_registered(q),
"%s is registering an already registered queue\n",
kobject_name(&dev->kobj));
queue_flag_set_unlocked(QUEUE_FLAG_REGISTERED, q);
@@ -940,7 +940,7 @@ void blk_unregister_queue(struct gendisk *disk)
return;
/* Return early if disk->queue was never registered. */
- if (!test_bit(QUEUE_FLAG_REGISTERED, &q->queue_flags))
+ if (!blk_queue_registered(q))
return;
/*
diff --git a/block/blk-wbt.c b/block/blk-wbt.c
index 4f89b28..43ae265 100644
--- a/block/blk-wbt.c
+++ b/block/blk-wbt.c
@@ -707,7 +707,7 @@ void wbt_enable_default(struct request_queue *q)
return;
/* Queue not registered? Maybe shutting down... */
- if (!test_bit(QUEUE_FLAG_REGISTERED, &q->queue_flags))
+ if (!blk_queue_registered(q))
return;
if ((q->mq_ops && IS_ENABLED(CONFIG_BLK_WBT_MQ)) ||
diff --git a/block/elevator.c b/block/elevator.c
index fa828b5..a574841 100644
--- a/block/elevator.c
+++ b/block/elevator.c
@@ -1084,7 +1084,7 @@ static int __elevator_change(struct request_queue *q, const char *name)
struct elevator_type *e;
/* Make sure queue is not in the middle of being removed */
- if (!test_bit(QUEUE_FLAG_REGISTERED, &q->queue_flags))
+ if (!blk_queue_registered(q))
return -ENOENT;
/*
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 37511c4..d6174ed 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -740,6 +740,9 @@ bool blk_queue_flag_test_and_clear(unsigned int flag, struct request_queue *q);
#define blk_queue_scsi_passthrough(q) \
test_bit(QUEUE_FLAG_SCSI_PASSTHROUGH, &(q)->queue_flags)
+#define blk_queue_registered(q) \
+ test_bit(QUEUE_FLAG_REGISTERED, &(q)->queue_flags)
+
#define blk_noretry_request(rq) \
((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \
REQ_FAILFAST_DRIVER))
--
2.7.4
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [PATCH 2/3] blk-mq: cleanup blk_mq_register_dev
2018-06-22 7:11 [PATCH 0/3] block: fix the lock inversion caused by kobject_del under sysfs_lock Jianchao Wang
2018-06-22 7:11 ` [PATCH 1/3] block: add helper interface blk_queue_registered Jianchao Wang
@ 2018-06-22 7:11 ` Jianchao Wang
2018-06-22 7:11 ` [PATCH 3/3] block: fix the lock inversion caused by kobject_del under sysfs_lock Jianchao Wang
2 siblings, 0 replies; 4+ messages in thread
From: Jianchao Wang @ 2018-06-22 7:11 UTC (permalink / raw)
To: axboe; +Cc: bart.vanassche, linux-block, linux-kernel
nobody invokes blk_mq_register_dev, remove it and change
__blk_mq_register_dev to blk_mq_register_dev. In addition, move
the declaration of blk_mq_register/unregister_dev to block/blk-mq.h
because they are only used by block layer.
Signed-off-by: Jianchao Wang <jianchao.w.wang@oracle.com>
---
block/blk-mq-sysfs.c | 14 +-------------
block/blk-mq.h | 3 +++
block/blk-sysfs.c | 2 +-
include/linux/blk-mq.h | 3 ---
4 files changed, 5 insertions(+), 17 deletions(-)
diff --git a/block/blk-mq-sysfs.c b/block/blk-mq-sysfs.c
index aafb442..ec26745 100644
--- a/block/blk-mq-sysfs.c
+++ b/block/blk-mq-sysfs.c
@@ -295,7 +295,7 @@ void blk_mq_sysfs_init(struct request_queue *q)
}
}
-int __blk_mq_register_dev(struct device *dev, struct request_queue *q)
+int blk_mq_register_dev(struct device *dev, struct request_queue *q)
{
struct blk_mq_hw_ctx *hctx;
int ret, i;
@@ -330,18 +330,6 @@ int __blk_mq_register_dev(struct device *dev, struct request_queue *q)
return ret;
}
-int blk_mq_register_dev(struct device *dev, struct request_queue *q)
-{
- int ret;
-
- mutex_lock(&q->sysfs_lock);
- ret = __blk_mq_register_dev(dev, q);
- mutex_unlock(&q->sysfs_lock);
-
- return ret;
-}
-EXPORT_SYMBOL_GPL(blk_mq_register_dev);
-
void blk_mq_sysfs_unregister(struct request_queue *q)
{
struct blk_mq_hw_ctx *hctx;
diff --git a/block/blk-mq.h b/block/blk-mq.h
index 89231e4..7f87e03 100644
--- a/block/blk-mq.h
+++ b/block/blk-mq.h
@@ -30,6 +30,9 @@ struct blk_mq_ctx {
struct kobject kobj;
} ____cacheline_aligned_in_smp;
+int blk_mq_register_dev(struct device *dev, struct request_queue *q);
+void blk_mq_unregister_dev(struct device *dev, struct request_queue *q);
+
void blk_mq_freeze_queue(struct request_queue *q);
void blk_mq_free_queue(struct request_queue *q);
int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr);
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
index 159c586..96dcbb9 100644
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -897,7 +897,7 @@ int blk_register_queue(struct gendisk *disk)
}
if (q->mq_ops) {
- __blk_mq_register_dev(dev, q);
+ blk_mq_register_dev(dev, q);
blk_mq_debugfs_register(q);
}
diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
index 0a509ae..61b3833 100644
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -201,9 +201,6 @@ enum {
struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *);
struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
struct request_queue *q);
-int blk_mq_register_dev(struct device *, struct request_queue *);
-void blk_mq_unregister_dev(struct device *, struct request_queue *);
-
int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set);
void blk_mq_free_tag_set(struct blk_mq_tag_set *set);
--
2.7.4
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [PATCH 3/3] block: fix the lock inversion caused by kobject_del under sysfs_lock
2018-06-22 7:11 [PATCH 0/3] block: fix the lock inversion caused by kobject_del under sysfs_lock Jianchao Wang
2018-06-22 7:11 ` [PATCH 1/3] block: add helper interface blk_queue_registered Jianchao Wang
2018-06-22 7:11 ` [PATCH 2/3] blk-mq: cleanup blk_mq_register_dev Jianchao Wang
@ 2018-06-22 7:11 ` Jianchao Wang
2 siblings, 0 replies; 4+ messages in thread
From: Jianchao Wang @ 2018-06-22 7:11 UTC (permalink / raw)
To: axboe; +Cc: bart.vanassche, linux-block, linux-kernel
Currently, the kobject_del for sysfs mq and hctx entry is invoked
under q->sysfs_lock. lock inversion will came up when someone try
to access the sysfs entries concurrently. The scenario is as below:
hold q->sysfs_lock access mq,hctx sysfs entries
kobject_del
__kernfs_remove kernfs_get_active
kernfs_drain
wait kn->active require q->sysfs_lock
To fix this issue, we do as following:
- still reserve q->sysfs_lock as the sync method between queue, mq
hctx sysfs entries' show and store method.
- use QUEUE_FLAG_REGISTERED to sync the blk_register/unregister_queue
with sysfs entries.
- use QUEUE_FLAG_REGISTERED to sync the blk_mq_register/unregister_dev
with blk_mq_sysfs_register/unregister.
- change q->mq_sysfs_init_done to q->mq_sysfs_ready to sync
blk_mq_register/unregister_dev and blk_mq_sysfs_register/unregister
with mq,hctx entries.
Then we don't need sysfs_lock on kobject_del anymore.
Signed-off-by: Jianchao Wang <jianchao.w.wang@oracle.com>
---
block/blk-mq-sysfs.c | 66 +++++++++++++++++++++++++++++++++++++-------------
block/blk-sysfs.c | 39 +++++++++++++++--------------
block/blk-wbt.c | 4 ---
block/elevator.c | 3 ++-
include/linux/blkdev.h | 2 +-
5 files changed, 73 insertions(+), 41 deletions(-)
diff --git a/block/blk-mq-sysfs.c b/block/blk-mq-sysfs.c
index ec26745..0923c2c 100644
--- a/block/blk-mq-sysfs.c
+++ b/block/blk-mq-sysfs.c
@@ -55,7 +55,9 @@ static ssize_t blk_mq_sysfs_show(struct kobject *kobj, struct attribute *attr,
res = -ENOENT;
mutex_lock(&q->sysfs_lock);
- if (!blk_queue_dying(q))
+ if (!blk_queue_dying(q) &&
+ blk_queue_registered(q) &&
+ q->mq_sysfs_ready)
res = entry->show(ctx, page);
mutex_unlock(&q->sysfs_lock);
return res;
@@ -78,7 +80,9 @@ static ssize_t blk_mq_sysfs_store(struct kobject *kobj, struct attribute *attr,
res = -ENOENT;
mutex_lock(&q->sysfs_lock);
- if (!blk_queue_dying(q))
+ if (!blk_queue_dying(q) &&
+ blk_queue_registered(q) &&
+ q->mq_sysfs_ready)
res = entry->store(ctx, page, length);
mutex_unlock(&q->sysfs_lock);
return res;
@@ -101,7 +105,9 @@ static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj,
res = -ENOENT;
mutex_lock(&q->sysfs_lock);
- if (!blk_queue_dying(q))
+ if (!blk_queue_dying(q) &&
+ blk_queue_registered(q) &&
+ q->mq_sysfs_ready)
res = entry->show(hctx, page);
mutex_unlock(&q->sysfs_lock);
return res;
@@ -125,7 +131,9 @@ static ssize_t blk_mq_hw_sysfs_store(struct kobject *kobj,
res = -ENOENT;
mutex_lock(&q->sysfs_lock);
- if (!blk_queue_dying(q))
+ if (!blk_queue_dying(q) &&
+ blk_queue_registered(q) &&
+ q->mq_sysfs_ready)
res = entry->store(hctx, page, length);
mutex_unlock(&q->sysfs_lock);
return res;
@@ -253,7 +261,9 @@ void blk_mq_unregister_dev(struct device *dev, struct request_queue *q)
struct blk_mq_hw_ctx *hctx;
int i;
- lockdep_assert_held(&q->sysfs_lock);
+ mutex_lock(&q->sysfs_lock);
+ q->mq_sysfs_ready = false;
+ mutex_unlock(&q->sysfs_lock);
queue_for_each_hw_ctx(q, hctx, i)
blk_mq_unregister_hctx(hctx);
@@ -262,7 +272,6 @@ void blk_mq_unregister_dev(struct device *dev, struct request_queue *q)
kobject_del(&q->mq_kobj);
kobject_put(&dev->kobj);
- q->mq_sysfs_init_done = false;
}
void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx)
@@ -295,13 +304,22 @@ void blk_mq_sysfs_init(struct request_queue *q)
}
}
+/*
+ * blk_mq_register_dev/blk_mq_unregister_dev are only invoked by
+ * blk_register_queue/blk_unregister_queue. So we could use
+ * QUEUE_FLAG_REGISTERED to sync with blk_mq_sysfs_register/unregister.
+ * If QUEUE_FLAG_REGISTERED is set, q->mq_kobj is ok, otherwise, it is
+ * not there.
+ * blk_mq_register/unregister_dev blk_mq_sysfs_register/unregister all
+ * use q->mq_sysfs_ready to sync with mq and hctx sysfs entries' store
+ * and show method.
+ */
int blk_mq_register_dev(struct device *dev, struct request_queue *q)
{
struct blk_mq_hw_ctx *hctx;
int ret, i;
WARN_ON_ONCE(!q->kobj.parent);
- lockdep_assert_held(&q->sysfs_lock);
ret = kobject_add(&q->mq_kobj, kobject_get(&dev->kobj), "%s", "mq");
if (ret < 0)
@@ -315,7 +333,9 @@ int blk_mq_register_dev(struct device *dev, struct request_queue *q)
goto unreg;
}
- q->mq_sysfs_init_done = true;
+ mutex_lock(&q->sysfs_lock);
+ q->mq_sysfs_ready = true;
+ mutex_unlock(&q->sysfs_lock);
out:
return ret;
@@ -335,15 +355,20 @@ void blk_mq_sysfs_unregister(struct request_queue *q)
struct blk_mq_hw_ctx *hctx;
int i;
+ /*
+ * If QUEUE_FLAG_REGISTERED is not set, q->mq_kobj
+ * is not ready.
+ */
mutex_lock(&q->sysfs_lock);
- if (!q->mq_sysfs_init_done)
- goto unlock;
+ if (!blk_queue_registered(q)) {
+ mutex_unlock(&q->sysfs_lock);
+ return;
+ }
+ q->mq_sysfs_ready = false;
+ mutex_unlock(&q->sysfs_lock);
queue_for_each_hw_ctx(q, hctx, i)
blk_mq_unregister_hctx(hctx);
-
-unlock:
- mutex_unlock(&q->sysfs_lock);
}
int blk_mq_sysfs_register(struct request_queue *q)
@@ -351,9 +376,16 @@ int blk_mq_sysfs_register(struct request_queue *q)
struct blk_mq_hw_ctx *hctx;
int i, ret = 0;
+ /*
+ * If QUEUE_FLAG_REGISTERED is not set, q->mq_kobj
+ * is not ready.
+ */
mutex_lock(&q->sysfs_lock);
- if (!q->mq_sysfs_init_done)
- goto unlock;
+ if (!blk_queue_registered(q)) {
+ mutex_unlock(&q->sysfs_lock);
+ return ret;
+ }
+ mutex_unlock(&q->sysfs_lock);
queue_for_each_hw_ctx(q, hctx, i) {
ret = blk_mq_register_hctx(hctx);
@@ -361,8 +393,8 @@ int blk_mq_sysfs_register(struct request_queue *q)
break;
}
-unlock:
+ mutex_lock(&q->sysfs_lock);
+ q->mq_sysfs_ready = true;
mutex_unlock(&q->sysfs_lock);
-
return ret;
}
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
index 96dcbb9..a3ef681 100644
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -741,7 +741,8 @@ queue_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
if (!entry->show)
return -EIO;
mutex_lock(&q->sysfs_lock);
- if (blk_queue_dying(q)) {
+ if (blk_queue_dying(q) ||
+ !blk_queue_registered(q)) {
mutex_unlock(&q->sysfs_lock);
return -ENOENT;
}
@@ -763,7 +764,8 @@ queue_attr_store(struct kobject *kobj, struct attribute *attr,
q = container_of(kobj, struct request_queue, kobj);
mutex_lock(&q->sysfs_lock);
- if (blk_queue_dying(q)) {
+ if (blk_queue_dying(q) ||
+ !blk_queue_registered(q)) {
mutex_unlock(&q->sysfs_lock);
return -ENOENT;
}
@@ -866,7 +868,6 @@ int blk_register_queue(struct gendisk *disk)
WARN_ONCE(blk_queue_registered(q),
"%s is registering an already registered queue\n",
kobject_name(&dev->kobj));
- queue_flag_set_unlocked(QUEUE_FLAG_REGISTERED, q);
/*
* SCSI probing may synchronously create and destroy a lot of
@@ -887,17 +888,16 @@ int blk_register_queue(struct gendisk *disk)
if (ret)
return ret;
- /* Prevent changes through sysfs until registration is completed. */
- mutex_lock(&q->sysfs_lock);
-
ret = kobject_add(&q->kobj, kobject_get(&dev->kobj), "%s", "queue");
if (ret < 0) {
blk_trace_remove_sysfs(dev);
- goto unlock;
+ return ret;
}
if (q->mq_ops) {
- blk_mq_register_dev(dev, q);
+ ret = blk_mq_register_dev(dev, q);
+ if (ret)
+ goto err_kobj;
blk_mq_debugfs_register(q);
}
@@ -907,20 +907,25 @@ int blk_register_queue(struct gendisk *disk)
blk_throtl_register_queue(q);
+ /* Prevent changes through sysfs until registration is completed. */
+ mutex_lock(&q->sysfs_lock);
if (q->request_fn || (q->mq_ops && q->elevator)) {
ret = elv_register_queue(q);
if (ret) {
mutex_unlock(&q->sysfs_lock);
- kobject_uevent(&q->kobj, KOBJ_REMOVE);
- kobject_del(&q->kobj);
- blk_trace_remove_sysfs(dev);
- kobject_put(&dev->kobj);
- return ret;
+ goto err_kobj;
}
}
- ret = 0;
-unlock:
+
+ blk_queue_flag_set(QUEUE_FLAG_REGISTERED, q);
mutex_unlock(&q->sysfs_lock);
+ return 0;
+
+err_kobj:
+ kobject_uevent(&q->kobj, KOBJ_REMOVE);
+ kobject_del(&q->kobj);
+ blk_trace_remove_sysfs(dev);
+ kobject_put(&dev->kobj);
return ret;
}
EXPORT_SYMBOL_GPL(blk_register_queue);
@@ -949,16 +954,14 @@ void blk_unregister_queue(struct gendisk *disk)
* concurrent elv_iosched_store() calls.
*/
mutex_lock(&q->sysfs_lock);
-
blk_queue_flag_clear(QUEUE_FLAG_REGISTERED, q);
-
+ mutex_unlock(&q->sysfs_lock);
/*
* Remove the sysfs attributes before unregistering the queue data
* structures that can be modified through sysfs.
*/
if (q->mq_ops)
blk_mq_unregister_dev(disk_to_dev(disk), q);
- mutex_unlock(&q->sysfs_lock);
kobject_uevent(&q->kobj, KOBJ_REMOVE);
kobject_del(&q->kobj);
diff --git a/block/blk-wbt.c b/block/blk-wbt.c
index 43ae265..c26deca 100644
--- a/block/blk-wbt.c
+++ b/block/blk-wbt.c
@@ -706,10 +706,6 @@ void wbt_enable_default(struct request_queue *q)
if (q->rq_wb)
return;
- /* Queue not registered? Maybe shutting down... */
- if (!blk_queue_registered(q))
- return;
-
if ((q->mq_ops && IS_ENABLED(CONFIG_BLK_WBT_MQ)) ||
(q->request_fn && IS_ENABLED(CONFIG_BLK_WBT_SQ)))
wbt_init(q);
diff --git a/block/elevator.c b/block/elevator.c
index a574841..a68e3e6 100644
--- a/block/elevator.c
+++ b/block/elevator.c
@@ -869,7 +869,8 @@ void elv_unregister_queue(struct request_queue *q)
kobject_del(&e->kobj);
e->registered = 0;
/* Re-enable throttling in case elevator disabled it */
- wbt_enable_default(q);
+ if (blk_queue_registered(q))
+ wbt_enable_default(q);
}
}
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index d6174ed..df53e8f 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -666,7 +666,7 @@ struct request_queue {
struct dentry *sched_debugfs_dir;
#endif
- bool mq_sysfs_init_done;
+ bool mq_sysfs_ready;
size_t cmd_size;
void *rq_alloc_data;
--
2.7.4
^ permalink raw reply related [flat|nested] 4+ messages in thread
end of thread, other threads:[~2018-06-22 7:11 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2018-06-22 7:11 [PATCH 0/3] block: fix the lock inversion caused by kobject_del under sysfs_lock Jianchao Wang
2018-06-22 7:11 ` [PATCH 1/3] block: add helper interface blk_queue_registered Jianchao Wang
2018-06-22 7:11 ` [PATCH 2/3] blk-mq: cleanup blk_mq_register_dev Jianchao Wang
2018-06-22 7:11 ` [PATCH 3/3] block: fix the lock inversion caused by kobject_del under sysfs_lock Jianchao Wang
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).