diff options
author | 2016-04-11 10:41:07 +0300 | |
---|---|---|
committer | 2016-04-13 08:17:18 +0300 | |
commit | e09b41010ba33a20a87472ee821fa407a5b8da36 (patch) | |
tree | d10dc367189862e7ca5c592f033dc3726e1df4e3 /kernel/drivers/iommu/tegra-smmu.c | |
parent | f93b97fd65072de626c074dbe099a1fff05ce060 (diff) |
These changes are the raw update to linux-4.4.6-rt14. Kernel sources
are taken from kernel.org, and rt patch from the rt wiki download page.
During the rebasing, the following patch collided:
Force tick interrupt and get rid of softirq magic(I70131fb85).
Collisions have been removed because its logic was found on the
source already.
Change-Id: I7f57a4081d9deaa0d9ccfc41a6c8daccdee3b769
Signed-off-by: José Pekkarinen <jose.pekkarinen@nokia.com>
Diffstat (limited to 'kernel/drivers/iommu/tegra-smmu.c')
-rw-r--r-- | kernel/drivers/iommu/tegra-smmu.c | 406 |
1 files changed, 300 insertions, 106 deletions
diff --git a/kernel/drivers/iommu/tegra-smmu.c b/kernel/drivers/iommu/tegra-smmu.c index e0ff5f4d7..930596425 100644 --- a/kernel/drivers/iommu/tegra-smmu.c +++ b/kernel/drivers/iommu/tegra-smmu.c @@ -7,6 +7,7 @@ */ #include <linux/bitops.h> +#include <linux/debugfs.h> #include <linux/err.h> #include <linux/iommu.h> #include <linux/kernel.h> @@ -32,14 +33,18 @@ struct tegra_smmu { struct mutex lock; struct list_head list; + + struct dentry *debugfs; }; struct tegra_smmu_as { struct iommu_domain domain; struct tegra_smmu *smmu; unsigned int use_count; - struct page *count; + u32 *count; + struct page **pts; struct page *pd; + dma_addr_t pd_dma; unsigned id; u32 attr; }; @@ -78,9 +83,9 @@ static inline u32 smmu_readl(struct tegra_smmu *smmu, unsigned long offset) #define SMMU_PTB_ASID_VALUE(x) ((x) & 0x7f) #define SMMU_PTB_DATA 0x020 -#define SMMU_PTB_DATA_VALUE(page, attr) (page_to_phys(page) >> 12 | (attr)) +#define SMMU_PTB_DATA_VALUE(dma, attr) ((dma) >> 12 | (attr)) -#define SMMU_MK_PDE(page, attr) (page_to_phys(page) >> SMMU_PTE_SHIFT | (attr)) +#define SMMU_MK_PDE(dma, attr) ((dma) >> SMMU_PTE_SHIFT | (attr)) #define SMMU_TLB_FLUSH 0x030 #define SMMU_TLB_FLUSH_VA_MATCH_ALL (0 << 0) @@ -133,29 +138,49 @@ static inline u32 smmu_readl(struct tegra_smmu *smmu, unsigned long offset) #define SMMU_PTE_ATTR (SMMU_PTE_READABLE | SMMU_PTE_WRITABLE | \ SMMU_PTE_NONSECURE) -static inline void smmu_flush_ptc(struct tegra_smmu *smmu, struct page *page, +static unsigned int iova_pd_index(unsigned long iova) +{ + return (iova >> SMMU_PDE_SHIFT) & (SMMU_NUM_PDE - 1); +} + +static unsigned int iova_pt_index(unsigned long iova) +{ + return (iova >> SMMU_PTE_SHIFT) & (SMMU_NUM_PTE - 1); +} + +static bool smmu_dma_addr_valid(struct tegra_smmu *smmu, dma_addr_t addr) +{ + addr >>= 12; + return (addr & smmu->pfn_mask) == addr; +} + +static dma_addr_t smmu_pde_to_dma(u32 pde) +{ + return pde << 12; +} + +static void smmu_flush_ptc_all(struct tegra_smmu *smmu) +{ + smmu_writel(smmu, SMMU_PTC_FLUSH_TYPE_ALL, SMMU_PTC_FLUSH); +} + +static inline void smmu_flush_ptc(struct tegra_smmu *smmu, dma_addr_t dma, unsigned long offset) { - phys_addr_t phys = page ? page_to_phys(page) : 0; u32 value; - if (page) { - offset &= ~(smmu->mc->soc->atom_size - 1); + offset &= ~(smmu->mc->soc->atom_size - 1); - if (smmu->mc->soc->num_address_bits > 32) { -#ifdef CONFIG_PHYS_ADDR_T_64BIT - value = (phys >> 32) & SMMU_PTC_FLUSH_HI_MASK; + if (smmu->mc->soc->num_address_bits > 32) { +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + value = (dma >> 32) & SMMU_PTC_FLUSH_HI_MASK; #else - value = 0; + value = 0; #endif - smmu_writel(smmu, value, SMMU_PTC_FLUSH_HI); - } - - value = (phys + offset) | SMMU_PTC_FLUSH_TYPE_ADR; - } else { - value = SMMU_PTC_FLUSH_TYPE_ALL; + smmu_writel(smmu, value, SMMU_PTC_FLUSH_HI); } + value = (dma + offset) | SMMU_PTC_FLUSH_TYPE_ADR; smmu_writel(smmu, value, SMMU_PTC_FLUSH); } @@ -235,8 +260,6 @@ static bool tegra_smmu_capable(enum iommu_cap cap) static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type) { struct tegra_smmu_as *as; - unsigned int i; - uint32_t *pd; if (type != IOMMU_DOMAIN_UNMANAGED) return NULL; @@ -247,32 +270,26 @@ static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type) as->attr = SMMU_PD_READABLE | SMMU_PD_WRITABLE | SMMU_PD_NONSECURE; - as->pd = alloc_page(GFP_KERNEL | __GFP_DMA); + as->pd = alloc_page(GFP_KERNEL | __GFP_DMA | __GFP_ZERO); if (!as->pd) { kfree(as); return NULL; } - as->count = alloc_page(GFP_KERNEL); + as->count = kcalloc(SMMU_NUM_PDE, sizeof(u32), GFP_KERNEL); if (!as->count) { __free_page(as->pd); kfree(as); return NULL; } - /* clear PDEs */ - pd = page_address(as->pd); - SetPageReserved(as->pd); - - for (i = 0; i < SMMU_NUM_PDE; i++) - pd[i] = 0; - - /* clear PDE usage counters */ - pd = page_address(as->count); - SetPageReserved(as->count); - - for (i = 0; i < SMMU_NUM_PDE; i++) - pd[i] = 0; + as->pts = kcalloc(SMMU_NUM_PDE, sizeof(*as->pts), GFP_KERNEL); + if (!as->pts) { + kfree(as->count); + __free_page(as->pd); + kfree(as); + return NULL; + } /* setup aperture */ as->domain.geometry.aperture_start = 0; @@ -287,7 +304,6 @@ static void tegra_smmu_domain_free(struct iommu_domain *domain) struct tegra_smmu_as *as = to_smmu_as(domain); /* TODO: free page directory and page tables */ - ClearPageReserved(as->pd); kfree(as); } @@ -375,16 +391,26 @@ static int tegra_smmu_as_prepare(struct tegra_smmu *smmu, return 0; } + as->pd_dma = dma_map_page(smmu->dev, as->pd, 0, SMMU_SIZE_PD, + DMA_TO_DEVICE); + if (dma_mapping_error(smmu->dev, as->pd_dma)) + return -ENOMEM; + + /* We can't handle 64-bit DMA addresses */ + if (!smmu_dma_addr_valid(smmu, as->pd_dma)) { + err = -ENOMEM; + goto err_unmap; + } + err = tegra_smmu_alloc_asid(smmu, &as->id); if (err < 0) - return err; + goto err_unmap; - smmu->soc->ops->flush_dcache(as->pd, 0, SMMU_SIZE_PD); - smmu_flush_ptc(smmu, as->pd, 0); + smmu_flush_ptc(smmu, as->pd_dma, 0); smmu_flush_tlb_asid(smmu, as->id); smmu_writel(smmu, as->id & 0x7f, SMMU_PTB_ASID); - value = SMMU_PTB_DATA_VALUE(as->pd, as->attr); + value = SMMU_PTB_DATA_VALUE(as->pd_dma, as->attr); smmu_writel(smmu, value, SMMU_PTB_DATA); smmu_flush(smmu); @@ -392,6 +418,10 @@ static int tegra_smmu_as_prepare(struct tegra_smmu *smmu, as->use_count++; return 0; + +err_unmap: + dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE); + return err; } static void tegra_smmu_as_unprepare(struct tegra_smmu *smmu, @@ -401,6 +431,9 @@ static void tegra_smmu_as_unprepare(struct tegra_smmu *smmu, return; tegra_smmu_free_asid(smmu, as->id); + + dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE); + as->smmu = NULL; } @@ -464,96 +497,155 @@ static void tegra_smmu_detach_dev(struct iommu_domain *domain, struct device *de } } +static void tegra_smmu_set_pde(struct tegra_smmu_as *as, unsigned long iova, + u32 value) +{ + unsigned int pd_index = iova_pd_index(iova); + struct tegra_smmu *smmu = as->smmu; + u32 *pd = page_address(as->pd); + unsigned long offset = pd_index * sizeof(*pd); + + /* Set the page directory entry first */ + pd[pd_index] = value; + + /* The flush the page directory entry from caches */ + dma_sync_single_range_for_device(smmu->dev, as->pd_dma, offset, + sizeof(*pd), DMA_TO_DEVICE); + + /* And flush the iommu */ + smmu_flush_ptc(smmu, as->pd_dma, offset); + smmu_flush_tlb_section(smmu, as->id, iova); + smmu_flush(smmu); +} + +static u32 *tegra_smmu_pte_offset(struct page *pt_page, unsigned long iova) +{ + u32 *pt = page_address(pt_page); + + return pt + iova_pt_index(iova); +} + +static u32 *tegra_smmu_pte_lookup(struct tegra_smmu_as *as, unsigned long iova, + dma_addr_t *dmap) +{ + unsigned int pd_index = iova_pd_index(iova); + struct page *pt_page; + u32 *pd; + + pt_page = as->pts[pd_index]; + if (!pt_page) + return NULL; + + pd = page_address(as->pd); + *dmap = smmu_pde_to_dma(pd[pd_index]); + + return tegra_smmu_pte_offset(pt_page, iova); +} + static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, - struct page **pagep) + dma_addr_t *dmap) { - u32 *pd = page_address(as->pd), *pt, *count; - u32 pde = (iova >> SMMU_PDE_SHIFT) & 0x3ff; - u32 pte = (iova >> SMMU_PTE_SHIFT) & 0x3ff; + unsigned int pde = iova_pd_index(iova); struct tegra_smmu *smmu = as->smmu; - struct page *page; - unsigned int i; - if (pd[pde] == 0) { - page = alloc_page(GFP_KERNEL | __GFP_DMA); + if (!as->pts[pde]) { + struct page *page; + dma_addr_t dma; + + page = alloc_page(GFP_KERNEL | __GFP_DMA | __GFP_ZERO); if (!page) return NULL; - pt = page_address(page); - SetPageReserved(page); + dma = dma_map_page(smmu->dev, page, 0, SMMU_SIZE_PT, + DMA_TO_DEVICE); + if (dma_mapping_error(smmu->dev, dma)) { + __free_page(page); + return NULL; + } - for (i = 0; i < SMMU_NUM_PTE; i++) - pt[i] = 0; + if (!smmu_dma_addr_valid(smmu, dma)) { + dma_unmap_page(smmu->dev, dma, SMMU_SIZE_PT, + DMA_TO_DEVICE); + __free_page(page); + return NULL; + } - smmu->soc->ops->flush_dcache(page, 0, SMMU_SIZE_PT); + as->pts[pde] = page; - pd[pde] = SMMU_MK_PDE(page, SMMU_PDE_ATTR | SMMU_PDE_NEXT); + tegra_smmu_set_pde(as, iova, SMMU_MK_PDE(dma, SMMU_PDE_ATTR | + SMMU_PDE_NEXT)); - smmu->soc->ops->flush_dcache(as->pd, pde << 2, 4); - smmu_flush_ptc(smmu, as->pd, pde << 2); - smmu_flush_tlb_section(smmu, as->id, iova); - smmu_flush(smmu); + *dmap = dma; } else { - page = pfn_to_page(pd[pde] & smmu->pfn_mask); - pt = page_address(page); + u32 *pd = page_address(as->pd); + + *dmap = smmu_pde_to_dma(pd[pde]); } - *pagep = page; + return tegra_smmu_pte_offset(as->pts[pde], iova); +} - /* Keep track of entries in this page table. */ - count = page_address(as->count); - if (pt[pte] == 0) - count[pde]++; +static void tegra_smmu_pte_get_use(struct tegra_smmu_as *as, unsigned long iova) +{ + unsigned int pd_index = iova_pd_index(iova); - return &pt[pte]; + as->count[pd_index]++; } -static void as_put_pte(struct tegra_smmu_as *as, dma_addr_t iova) +static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova) { - u32 pde = (iova >> SMMU_PDE_SHIFT) & 0x3ff; - u32 pte = (iova >> SMMU_PTE_SHIFT) & 0x3ff; - u32 *count = page_address(as->count); - u32 *pd = page_address(as->pd), *pt; - struct page *page; - - page = pfn_to_page(pd[pde] & as->smmu->pfn_mask); - pt = page_address(page); + unsigned int pde = iova_pd_index(iova); + struct page *page = as->pts[pde]; /* * When no entries in this page table are used anymore, return the * memory page to the system. */ - if (pt[pte] != 0) { - if (--count[pde] == 0) { - ClearPageReserved(page); - __free_page(page); - pd[pde] = 0; - } + if (--as->count[pde] == 0) { + struct tegra_smmu *smmu = as->smmu; + u32 *pd = page_address(as->pd); + dma_addr_t pte_dma = smmu_pde_to_dma(pd[pde]); - pt[pte] = 0; + tegra_smmu_set_pde(as, iova, 0); + + dma_unmap_page(smmu->dev, pte_dma, SMMU_SIZE_PT, DMA_TO_DEVICE); + __free_page(page); + as->pts[pde] = NULL; } } +static void tegra_smmu_set_pte(struct tegra_smmu_as *as, unsigned long iova, + u32 *pte, dma_addr_t pte_dma, u32 val) +{ + struct tegra_smmu *smmu = as->smmu; + unsigned long offset = offset_in_page(pte); + + *pte = val; + + dma_sync_single_range_for_device(smmu->dev, pte_dma, offset, + 4, DMA_TO_DEVICE); + smmu_flush_ptc(smmu, pte_dma, offset); + smmu_flush_tlb_group(smmu, as->id, iova); + smmu_flush(smmu); +} + static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot) { struct tegra_smmu_as *as = to_smmu_as(domain); - struct tegra_smmu *smmu = as->smmu; - unsigned long offset; - struct page *page; + dma_addr_t pte_dma; u32 *pte; - pte = as_get_pte(as, iova, &page); + pte = as_get_pte(as, iova, &pte_dma); if (!pte) return -ENOMEM; - *pte = __phys_to_pfn(paddr) | SMMU_PTE_ATTR; - offset = offset_in_page(pte); + /* If we aren't overwriting a pre-existing entry, increment use */ + if (*pte == 0) + tegra_smmu_pte_get_use(as, iova); - smmu->soc->ops->flush_dcache(page, offset, 4); - smmu_flush_ptc(smmu, page, offset); - smmu_flush_tlb_group(smmu, as->id, iova); - smmu_flush(smmu); + tegra_smmu_set_pte(as, iova, pte, pte_dma, + __phys_to_pfn(paddr) | SMMU_PTE_ATTR); return 0; } @@ -562,22 +654,15 @@ static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size) { struct tegra_smmu_as *as = to_smmu_as(domain); - struct tegra_smmu *smmu = as->smmu; - unsigned long offset; - struct page *page; + dma_addr_t pte_dma; u32 *pte; - pte = as_get_pte(as, iova, &page); - if (!pte) + pte = tegra_smmu_pte_lookup(as, iova, &pte_dma); + if (!pte || !*pte) return 0; - offset = offset_in_page(pte); - as_put_pte(as, iova); - - smmu->soc->ops->flush_dcache(page, offset, 4); - smmu_flush_ptc(smmu, page, offset); - smmu_flush_tlb_group(smmu, as->id, iova); - smmu_flush(smmu); + tegra_smmu_set_pte(as, iova, pte, pte_dma, 0); + tegra_smmu_pte_put_use(as, iova); return size; } @@ -586,11 +671,14 @@ static phys_addr_t tegra_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { struct tegra_smmu_as *as = to_smmu_as(domain); - struct page *page; unsigned long pfn; + dma_addr_t pte_dma; u32 *pte; - pte = as_get_pte(as, iova, &page); + pte = tegra_smmu_pte_lookup(as, iova, &pte_dma); + if (!pte || !*pte) + return 0; + pfn = *pte & as->smmu->pfn_mask; return PFN_PHYS(pfn); @@ -675,6 +763,103 @@ static void tegra_smmu_ahb_enable(void) } } +static int tegra_smmu_swgroups_show(struct seq_file *s, void *data) +{ + struct tegra_smmu *smmu = s->private; + unsigned int i; + u32 value; + + seq_printf(s, "swgroup enabled ASID\n"); + seq_printf(s, "------------------------\n"); + + for (i = 0; i < smmu->soc->num_swgroups; i++) { + const struct tegra_smmu_swgroup *group = &smmu->soc->swgroups[i]; + const char *status; + unsigned int asid; + + value = smmu_readl(smmu, group->reg); + + if (value & SMMU_ASID_ENABLE) + status = "yes"; + else + status = "no"; + + asid = value & SMMU_ASID_MASK; + + seq_printf(s, "%-9s %-7s %#04x\n", group->name, status, + asid); + } + + return 0; +} + +static int tegra_smmu_swgroups_open(struct inode *inode, struct file *file) +{ + return single_open(file, tegra_smmu_swgroups_show, inode->i_private); +} + +static const struct file_operations tegra_smmu_swgroups_fops = { + .open = tegra_smmu_swgroups_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int tegra_smmu_clients_show(struct seq_file *s, void *data) +{ + struct tegra_smmu *smmu = s->private; + unsigned int i; + u32 value; + + seq_printf(s, "client enabled\n"); + seq_printf(s, "--------------------\n"); + + for (i = 0; i < smmu->soc->num_clients; i++) { + const struct tegra_mc_client *client = &smmu->soc->clients[i]; + const char *status; + + value = smmu_readl(smmu, client->smmu.reg); + + if (value & BIT(client->smmu.bit)) + status = "yes"; + else + status = "no"; + + seq_printf(s, "%-12s %s\n", client->name, status); + } + + return 0; +} + +static int tegra_smmu_clients_open(struct inode *inode, struct file *file) +{ + return single_open(file, tegra_smmu_clients_show, inode->i_private); +} + +static const struct file_operations tegra_smmu_clients_fops = { + .open = tegra_smmu_clients_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static void tegra_smmu_debugfs_init(struct tegra_smmu *smmu) +{ + smmu->debugfs = debugfs_create_dir("smmu", NULL); + if (!smmu->debugfs) + return; + + debugfs_create_file("swgroups", S_IRUGO, smmu->debugfs, smmu, + &tegra_smmu_swgroups_fops); + debugfs_create_file("clients", S_IRUGO, smmu->debugfs, smmu, + &tegra_smmu_clients_fops); +} + +static void tegra_smmu_debugfs_exit(struct tegra_smmu *smmu) +{ + debugfs_remove_recursive(smmu->debugfs); +} + struct tegra_smmu *tegra_smmu_probe(struct device *dev, const struct tegra_smmu_soc *soc, struct tegra_mc *mc) @@ -737,7 +922,7 @@ struct tegra_smmu *tegra_smmu_probe(struct device *dev, smmu_writel(smmu, value, SMMU_TLB_CONFIG); - smmu_flush_ptc(smmu, NULL, 0); + smmu_flush_ptc_all(smmu); smmu_flush_tlb(smmu); smmu_writel(smmu, SMMU_CONFIG_ENABLE, SMMU_CONFIG); smmu_flush(smmu); @@ -748,5 +933,14 @@ struct tegra_smmu *tegra_smmu_probe(struct device *dev, if (err < 0) return ERR_PTR(err); + if (IS_ENABLED(CONFIG_DEBUG_FS)) + tegra_smmu_debugfs_init(smmu); + return smmu; } + +void tegra_smmu_remove(struct tegra_smmu *smmu) +{ + if (IS_ENABLED(CONFIG_DEBUG_FS)) + tegra_smmu_debugfs_exit(smmu); +} |