@@ -829,7 +829,7 @@ static void bcm2835_dma_free(struct bcm2835_dmadev *od)
list_for_each_entry_safe(c, next, &od->ddev.channels,
vc.chan.device_node) {
list_del(&c->vc.chan.device_node);
- tasklet_kill(&c->vc.task);
+ dmaengine_kill_bh(&c->vc.chan);
}
dma_unmap_page_attrs(od->ddev.dev, od->zero_page, PAGE_SIZE,
@@ -1195,9 +1195,11 @@ static int axi_dmac_detect_caps(struct axi_dmac *dmac, unsigned int version)
return 0;
}
-static void axi_dmac_tasklet_kill(void *task)
+static void axi_dmac_kill_bh(void *data)
{
- tasklet_kill(task);
+ struct dma_chan *chan = data;
+
+ dmaengine_kill_bh(chan);
}
static void axi_dmac_free_dma_controller(void *of_node)
@@ -1302,8 +1304,8 @@ static int axi_dmac_probe(struct platform_device *pdev)
* Put the action in here so it get's done before unregistering the DMA
* device.
*/
- ret = devm_add_action_or_reset(&pdev->dev, axi_dmac_tasklet_kill,
- &dmac->chan.vchan.task);
+ ret = devm_add_action_or_reset(&pdev->dev, axi_dmac_kill_bh,
+ &dmac->chan.vchan.chan);
if (ret)
return ret;
@@ -1019,7 +1019,7 @@ static void jz4780_dma_remove(struct platform_device *pdev)
free_irq(jzdma->irq, jzdma);
for (i = 0; i < jzdma->soc_data->nb_channels; i++)
- tasklet_kill(&jzdma->chan[i].vchan.task);
+ dmaengine_kill_bh(&jzdma->chan[i].vchan.chan);
}
static const struct jz4780_dma_soc_data jz4740_dma_soc_data = {
@@ -1428,6 +1428,48 @@ static void dmaengine_destroy_unmap_pool(void)
}
}
+static void dma_chan_bh_entry(struct tasklet_struct *tasklet)
+{
+ struct dma_chan *chan = from_tasklet(chan, tasklet, bh_tasklet);
+ dmaengine_bh_work_fn fn = READ_ONCE(chan->bh_work_fn);
+
+ if (fn)
+ fn(chan);
+}
+
+void dmaengine_init_bh(struct dma_chan *chan, dmaengine_bh_work_fn fn)
+{
+ if (WARN_ON(!fn))
+ return;
+
+ if (WARN_ON(chan->bh_work_initialized))
+ return;
+
+ chan->bh_work_fn = fn;
+ tasklet_setup(&chan->bh_tasklet, dma_chan_bh_entry);
+ chan->bh_work_initialized = true;
+}
+EXPORT_SYMBOL_GPL(dmaengine_init_bh);
+
+bool dmaengine_schedule_bh(struct dma_chan *chan)
+{
+ if (WARN_ON(!chan->bh_work_initialized))
+ return false;
+
+ tasklet_schedule(&chan->bh_tasklet);
+ return true;
+}
+EXPORT_SYMBOL_GPL(dmaengine_schedule_bh);
+
+void dmaengine_kill_bh(struct dma_chan *chan)
+{
+ if (!chan->bh_work_initialized)
+ return;
+
+ tasklet_kill(&chan->bh_tasklet);
+}
+EXPORT_SYMBOL_GPL(dmaengine_kill_bh);
+
static int __init dmaengine_init_unmap_pool(void)
{
int i;
@@ -1663,7 +1663,7 @@ static void dw_remove(struct platform_device *pdev)
list_for_each_entry_safe(chan, _chan, &dw->dma.channels,
vc.chan.device_node) {
list_del(&chan->vc.chan.device_node);
- tasklet_kill(&chan->vc.task);
+ dmaengine_kill_bh(&chan->vc.chan);
}
}
@@ -1170,7 +1170,7 @@ int dw_edma_remove(struct dw_edma_chip *chip)
dma_async_device_unregister(&dw->dma);
list_for_each_entry_safe(chan, _chan, &dw->dma.channels,
vc.chan.device_node) {
- tasklet_kill(&chan->vc.task);
+ dmaengine_kill_bh(&chan->vc.chan);
list_del(&chan->vc.chan.device_node);
}
@@ -915,7 +915,7 @@ void fsl_edma_cleanup_vchan(struct dma_device *dmadev)
list_for_each_entry_safe(chan, _chan,
&dmadev->channels, vchan.chan.device_node) {
list_del(&chan->vchan.chan.device_node);
- tasklet_kill(&chan->vchan.task);
+ dmaengine_kill_bh(&chan->vchan.chan);
}
}
@@ -1255,7 +1255,7 @@ static void fsl_qdma_cleanup_vchan(struct dma_device *dmadev)
list_for_each_entry_safe(chan, _chan,
&dmadev->channels, vchan.chan.device_node) {
list_del(&chan->vchan.chan.device_node);
- tasklet_kill(&chan->vchan.task);
+ dmaengine_kill_bh(&chan->vchan.chan);
}
}
@@ -720,7 +720,7 @@ static void hisi_dma_disable_qps(struct hisi_dma_dev *hdma_dev)
for (i = 0; i < hdma_dev->chan_num; i++) {
hisi_dma_disable_qp(hdma_dev, i);
- tasklet_kill(&hdma_dev->chan[i].vc.task);
+ dmaengine_kill_bh(&hdma_dev->chan[i].vc.chan);
}
}
@@ -500,7 +500,7 @@ int hsu_dma_remove(struct hsu_dma_chip *chip)
for (i = 0; i < hsu->nr_channels; i++) {
struct hsu_dma_chan *hsuc = &hsu->chan[i];
- tasklet_kill(&hsuc->vchan.task);
+ dmaengine_kill_bh(&hsuc->vchan.chan);
}
return 0;
@@ -617,14 +617,14 @@ static void idma64_remove(struct idma64_chip *chip)
/*
* Explicitly call devm_request_irq() to avoid the side effects with
- * the scheduled tasklets.
+ * scheduled BH work.
*/
devm_free_irq(chip->dev, chip->irq, idma64);
for (i = 0; i < idma64->dma.chancnt; i++) {
struct idma64_chan *idma64c = &idma64->chan[i];
- tasklet_kill(&idma64c->vchan.task);
+ dmaengine_kill_bh(&idma64c->vchan.chan);
}
}
@@ -1031,7 +1031,7 @@ static void mdc_dma_remove(struct platform_device *pdev)
devm_free_irq(&pdev->dev, mchan->irq, mchan);
- tasklet_kill(&mchan->vc.task);
+ dmaengine_kill_bh(&mchan->vc.chan);
}
pm_runtime_disable(&pdev->dev);
@@ -2399,11 +2399,11 @@ static void sdma_remove(struct platform_device *pdev)
int i;
devm_free_irq(&pdev->dev, sdma->irq, sdma);
- /* Kill the tasklet */
+ /* Kill the channel BH */
for (i = 0; i < MAX_DMA_CHANNELS; i++) {
struct sdma_channel *sdmac = &sdma->channel[i];
- tasklet_kill(&sdmac->vc.task);
+ dmaengine_kill_bh(&sdmac->vc.chan);
sdma_free_chan_resources(&sdmac->vc.chan);
}
@@ -976,7 +976,7 @@ static void k3_dma_remove(struct platform_device *op)
list_for_each_entry_safe(c, cn, &d->slave.channels, vc.chan.device_node) {
list_del(&c->vc.chan.device_node);
- tasklet_kill(&c->vc.task);
+ dmaengine_kill_bh(&c->vc.chan);
}
tasklet_kill(&d->task);
clk_disable_unprepare(d->clk);
@@ -552,7 +552,7 @@ static void ls1x_dma_chan_remove(struct ls1x_dma *dma)
if (chan->vc.chan.device == &dma->ddev) {
list_del(&chan->vc.chan.device_node);
- tasklet_kill(&chan->vc.task);
+ dmaengine_kill_bh(&chan->vc.chan);
}
}
}
@@ -895,7 +895,7 @@ static void mtk_cqdma_remove(struct platform_device *pdev)
vc = &cqdma->vc[i];
list_del(&vc->vc.chan.device_node);
- tasklet_kill(&vc->vc.task);
+ dmaengine_kill_bh(&vc->vc.chan);
}
/* disable interrupt */
@@ -1020,7 +1020,7 @@ static void mtk_hsdma_remove(struct platform_device *pdev)
vc = &hsdma->vc[i];
list_del(&vc->vc.chan.device_node);
- tasklet_kill(&vc->vc.task);
+ dmaengine_kill_bh(&vc->vc.chan);
}
/* Disable DMA interrupt */
@@ -312,7 +312,7 @@ static void mtk_uart_apdma_free_chan_resources(struct dma_chan *chan)
free_irq(c->irq, chan);
- tasklet_kill(&c->vc.task);
+ dmaengine_kill_bh(&c->vc.chan);
vchan_free_chan_resources(&c->vc);
@@ -463,7 +463,7 @@ static void mtk_uart_apdma_free(struct mtk_uart_apdmadev *mtkd)
struct mtk_chan, vc.chan.device_node);
list_del(&c->vc.chan.device_node);
- tasklet_kill(&c->vc.task);
+ dmaengine_kill_bh(&c->vc.chan);
}
}
@@ -1055,7 +1055,7 @@ static inline void owl_dma_free(struct owl_dma *od)
list_for_each_entry_safe(vchan,
next, &od->dma.channels, vc.chan.device_node) {
list_del(&vchan->vc.chan.device_node);
- tasklet_kill(&vchan->vc.task);
+ dmaengine_kill_bh(&vchan->vc.chan);
}
}
@@ -1215,7 +1215,7 @@ static void pxad_free_channels(struct dma_device *dmadev)
list_for_each_entry_safe(c, cn, &dmadev->channels,
vc.chan.device_node) {
list_del(&c->vc.chan.device_node);
- tasklet_kill(&c->vc.task);
+ dmaengine_kill_bh(&c->vc.chan);
}
}
@@ -1387,7 +1387,7 @@ static int bam_dma_probe(struct platform_device *pdev)
dma_async_device_unregister(&bdev->common);
err_bam_channel_exit:
for (i = 0; i < bdev->num_channels; i++)
- tasklet_kill(&bdev->channels[i].vc.task);
+ dmaengine_kill_bh(&bdev->channels[i].vc.chan);
err_tasklet_kill:
tasklet_kill(&bdev->task);
err_disable_clk:
@@ -1413,7 +1413,7 @@ static void bam_dma_remove(struct platform_device *pdev)
for (i = 0; i < bdev->num_channels; i++) {
bam_dma_terminate_all(&bdev->channels[i].vc.chan);
- tasklet_kill(&bdev->channels[i].vc.task);
+ dmaengine_kill_bh(&bdev->channels[i].vc.chan);
if (!bdev->channels[i].fifo_virt)
continue;
@@ -918,8 +918,8 @@ static void adm_dma_remove(struct platform_device *pdev)
/* mask IRQs for this channel/EE pair */
writel(0, adev->regs + ADM_CH_RSLT_CONF(achan->id, adev->ee));
- tasklet_kill(&adev->channels[i].vc.task);
- adm_terminate_all(&adev->channels[i].vc.chan);
+ dmaengine_kill_bh(&achan->vc.chan);
+ adm_terminate_all(&achan->vc.chan);
}
devm_free_irq(adev->dev, adev->irq, adev);
@@ -891,7 +891,7 @@ static void sa11x0_dma_free_channels(struct dma_device *dmadev)
list_for_each_entry_safe(c, cn, &dmadev->channels, vc.chan.device_node) {
list_del(&c->vc.chan.device_node);
- tasklet_kill(&c->vc.task);
+ dmaengine_kill_bh(&c->vc.chan);
kfree(c);
}
}
@@ -602,7 +602,7 @@ static void sf_pdma_remove(struct platform_device *pdev)
devm_free_irq(&pdev->dev, ch->txirq, ch);
devm_free_irq(&pdev->dev, ch->errirq, ch);
list_del(&ch->vchan.chan.device_node);
- tasklet_kill(&ch->vchan.task);
+ dmaengine_kill_bh(&ch->vchan.chan);
tasklet_kill(&ch->done_tasklet);
tasklet_kill(&ch->err_tasklet);
}
@@ -1253,7 +1253,7 @@ static void sprd_dma_remove(struct platform_device *pdev)
list_for_each_entry_safe(c, cn, &sdev->dma_dev.channels,
vc.chan.device_node) {
list_del(&c->vc.chan.device_node);
- tasklet_kill(&c->vc.task);
+ dmaengine_kill_bh(&c->vc.chan);
}
of_dma_controller_free(pdev->dev.of_node);
@@ -733,7 +733,7 @@ static void st_fdma_free(struct st_fdma_dev *fdev)
for (i = 0; i < fdev->nr_channels; i++) {
fchan = &fdev->chans[i];
list_del(&fchan->vchan.chan.device_node);
- tasklet_kill(&fchan->vchan.task);
+ dmaengine_kill_bh(&fchan->vchan.chan);
}
}
@@ -1094,7 +1094,7 @@ static inline void sun6i_dma_free(struct sun6i_dma_dev *sdev)
struct sun6i_vchan *vchan = &sdev->vchans[i];
list_del(&vchan->vc.chan.device_node);
- tasklet_kill(&vchan->vc.task);
+ dmaengine_kill_bh(&vchan->vc.chan);
}
}
@@ -1284,7 +1284,7 @@ static void tegra_dma_free_chan_resources(struct dma_chan *dc)
tegra_dma_terminate_all(dc);
synchronize_irq(tdc->irq);
- tasklet_kill(&tdc->vc.task);
+ dmaengine_kill_bh(&tdc->vc.chan);
tdc->config_init = false;
tdc->slave_id = -1;
tdc->sid_dir = DMA_TRANS_NONE;
@@ -811,7 +811,7 @@ static void tegra_adma_free_chan_resources(struct dma_chan *dc)
tegra_adma_terminate_all(dc);
vchan_free_chan_resources(&tdc->vc);
- tasklet_kill(&tdc->vc.task);
+ dmaengine_kill_bh(&tdc->vc.chan);
free_irq(tdc->irq, tdc);
pm_runtime_put(tdc2dev(tdc));
@@ -2560,7 +2560,7 @@ static void edma_cleanupp_vchan(struct dma_device *dmadev)
list_for_each_entry_safe(echan, _echan,
&dmadev->channels, vchan.chan.device_node) {
list_del(&echan->vchan.chan.device_node);
- tasklet_kill(&echan->vchan.task);
+ dmaengine_kill_bh(&echan->vchan.chan);
}
}
@@ -4042,12 +4042,12 @@ static void udma_desc_pre_callback(struct virt_dma_chan *vc,
}
/*
- * This tasklet handles the completion of a DMA descriptor by
+ * This BH handles the completion of a DMA descriptor by
* calling its callback and freeing it.
*/
-static void udma_vchan_complete(struct tasklet_struct *t)
+static void udma_vchan_complete(struct dma_chan *chan)
{
- struct virt_dma_chan *vc = from_tasklet(vc, t, task);
+ struct virt_dma_chan *vc = to_virt_chan(chan);
struct virt_dma_desc *vd, *_vd;
struct dmaengine_desc_callback cb;
LIST_HEAD(head);
@@ -4112,7 +4112,7 @@ static void udma_free_chan_resources(struct dma_chan *chan)
}
vchan_free_chan_resources(&uc->vc);
- tasklet_kill(&uc->vc.task);
+ dmaengine_kill_bh(&uc->vc.chan);
bcdma_free_bchan_resources(uc);
udma_free_tx_resources(uc);
@@ -5627,8 +5627,8 @@ static int udma_probe(struct platform_device *pdev)
if (!uc->name)
return -ENOMEM;
vchan_init(&uc->vc, &ud->ddev);
- /* Use custom vchan completion handling */
- tasklet_setup(&uc->vc.task, udma_vchan_complete);
+ /* Override the default vchan completion handler */
+ uc->vc.chan.bh_work_fn = udma_vchan_complete;
init_completion(&uc->teardown_completed);
INIT_DELAYED_WORK(&uc->tx_drain.work, udma_check_tx_completion);
}
@@ -1521,7 +1521,7 @@ static void omap_dma_free(struct omap_dmadev *od)
struct omap_chan, vc.chan.device_node);
list_del(&c->vc.chan.device_node);
- tasklet_kill(&c->vc.task);
+ dmaengine_kill_bh(&c->vc.chan);
kfree(c);
}
}
@@ -77,12 +77,12 @@ struct virt_dma_desc *vchan_find_desc(struct virt_dma_chan *vc,
EXPORT_SYMBOL_GPL(vchan_find_desc);
/*
- * This tasklet handles the completion of a DMA descriptor by
- * calling its callback and freeing it.
+ * This bottom-half handler completes a DMA descriptor by invoking its
+ * callback and freeing it.
*/
-static void vchan_complete(struct tasklet_struct *t)
+static void vchan_complete(struct dma_chan *chan)
{
- struct virt_dma_chan *vc = from_tasklet(vc, t, task);
+ struct virt_dma_chan *vc = to_virt_chan(chan);
struct virt_dma_desc *vd, *_vd;
struct dmaengine_desc_callback cb;
LIST_HEAD(head);
@@ -98,7 +98,7 @@ static void vchan_complete(struct tasklet_struct *t)
}
spin_unlock_irq(&vc->lock);
- dmaengine_desc_callback_invoke(&cb, &vd->tx_result);
+ dmaengine_desc_callback_invoke(&cb, vd ? &vd->tx_result : NULL);
list_for_each_entry_safe(vd, _vd, &head, node) {
dmaengine_desc_get_callback(&vd->tx, &cb);
@@ -131,7 +131,7 @@ void vchan_init(struct virt_dma_chan *vc, struct dma_device *dmadev)
INIT_LIST_HEAD(&vc->desc_completed);
INIT_LIST_HEAD(&vc->desc_terminated);
- tasklet_setup(&vc->task, vchan_complete);
+ dmaengine_init_bh(&vc->chan, vchan_complete);
vc->chan.device = dmadev;
list_add_tail(&vc->chan.device_node, &dmadev->channels);
@@ -21,7 +21,6 @@ struct virt_dma_desc {
struct virt_dma_chan {
struct dma_chan chan;
- struct tasklet_struct task;
void (*desc_free)(struct virt_dma_desc *);
spinlock_t lock;
@@ -106,7 +105,7 @@ static inline void vchan_cookie_complete(struct virt_dma_desc *vd)
vd, cookie);
list_add_tail(&vd->node, &vc->desc_completed);
- tasklet_schedule(&vc->task);
+ dmaengine_schedule_bh(&vc->chan);
}
/**
@@ -137,7 +136,7 @@ static inline void vchan_cyclic_callback(struct virt_dma_desc *vd)
struct virt_dma_chan *vc = to_virt_chan(vd->tx.chan);
vc->cyclic = vd;
- tasklet_schedule(&vc->task);
+ dmaengine_schedule_bh(&vc->chan);
}
/**
@@ -223,7 +222,7 @@ static inline void vchan_synchronize(struct virt_dma_chan *vc)
LIST_HEAD(head);
unsigned long flags;
- tasklet_kill(&vc->task);
+ dmaengine_kill_bh(&vc->chan);
spin_lock_irqsave(&vc->lock, flags);
@@ -12,6 +12,7 @@
#include <linux/scatterlist.h>
#include <linux/bitmap.h>
#include <linux/types.h>
+#include <linux/interrupt.h>
#include <asm/page.h>
/**
@@ -295,6 +296,10 @@ enum dma_desc_metadata_mode {
DESC_METADATA_ENGINE = BIT(1),
};
+struct dma_chan;
+
+typedef void (*dmaengine_bh_work_fn)(struct dma_chan *chan);
+
/**
* struct dma_chan_percpu - the per-CPU part of struct dma_chan
* @memcpy_count: transaction counter
@@ -334,6 +339,9 @@ struct dma_router {
* @router: pointer to the DMA router structure
* @route_data: channel specific data for the router
* @private: private data for certain client-channel associations
+ * @bh_tasklet: bottom-half tasklet stored per-channel
+ * @bh_work_fn: callback executed when @bh_tasklet runs
+ * @bh_work_initialized: indicates whether @bh_tasklet has been initialized
*/
struct dma_chan {
struct dma_device *device;
@@ -359,6 +367,9 @@ struct dma_chan {
void *route_data;
void *private;
+ struct tasklet_struct bh_tasklet;
+ dmaengine_bh_work_fn bh_work_fn;
+ bool bh_work_initialized;
};
/**
@@ -1529,6 +1540,9 @@ struct dma_chan *devm_dma_request_chan(struct device *dev, const char *name);
void dma_release_channel(struct dma_chan *chan);
int dma_get_slave_caps(struct dma_chan *chan, struct dma_slave_caps *caps);
+void dmaengine_init_bh(struct dma_chan *chan, dmaengine_bh_work_fn fn);
+bool dmaengine_schedule_bh(struct dma_chan *chan);
+void dmaengine_kill_bh(struct dma_chan *chan);
#else
static inline struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type)
{
@@ -1576,6 +1590,20 @@ static inline int dma_get_slave_caps(struct dma_chan *chan,
{
return -ENXIO;
}
+
+static inline void dmaengine_init_bh(struct dma_chan *chan,
+ dmaengine_bh_work_fn fn)
+{
+}
+
+static inline bool dmaengine_schedule_bh(struct dma_chan *chan)
+{
+ return false;
+}
+
+static inline void dmaengine_kill_bh(struct dma_chan *chan)
+{
+}
#endif
static inline int dmaengine_desc_set_reuse(struct dma_async_tx_descriptor *tx)