diff --git a/arch/arm/mach-msm/board-8064.c b/arch/arm/mach-msm/board-8064.c index f5a90708041..362f34d4665 100644 --- a/arch/arm/mach-msm/board-8064.c +++ b/arch/arm/mach-msm/board-8064.c @@ -71,7 +71,6 @@ #include #include #include -#include #include #include #include @@ -165,9 +164,6 @@ static struct platform_device battery_bcl_device = { }; #endif -struct fmem_platform_data apq8064_fmem_pdata = { -}; - static struct memtype_reserve apq8064_reserve_table[] __initdata = { [MEMTYPE_SMI] = { }, @@ -191,36 +187,28 @@ static int apq8064_paddr_to_memtype(phys_addr_t paddr) return MEMTYPE_EBI1; } -#define FMEM_ENABLED 0 - #ifdef CONFIG_ION_MSM #ifdef CONFIG_MSM_MULTIMEDIA_USE_ION static struct ion_cp_heap_pdata cp_mm_apq8064_ion_pdata = { .permission_type = IPT_TYPE_MM_CARVEOUT, .align = PAGE_SIZE, - .reusable = FMEM_ENABLED, - .mem_is_fmem = FMEM_ENABLED, .fixed_position = FIXED_MIDDLE, }; static struct ion_cp_heap_pdata cp_mfc_apq8064_ion_pdata = { .permission_type = IPT_TYPE_MFC_SHAREDMEM, .align = PAGE_SIZE, - .reusable = 0, - .mem_is_fmem = FMEM_ENABLED, .fixed_position = FIXED_HIGH, }; static struct ion_co_heap_pdata co_apq8064_ion_pdata = { .adjacent_mem_id = INVALID_HEAP_ID, .align = PAGE_SIZE, - .mem_is_fmem = 0, }; static struct ion_co_heap_pdata fw_co_apq8064_ion_pdata = { .adjacent_mem_id = ION_CP_MM_HEAP_ID, .align = SZ_128K, - .mem_is_fmem = FMEM_ENABLED, .fixed_position = FIXED_LOW, }; #endif @@ -342,12 +330,6 @@ static struct platform_device apq8064_ion_dev = { }; #endif -static struct platform_device apq8064_fmem_device = { - .name = "fmem", - .id = 1, - .dev = { .platform_data = &apq8064_fmem_pdata }, -}; - static void __init reserve_mem_for_ion(enum ion_memory_types mem_type, unsigned long size) { @@ -373,15 +355,10 @@ static void __init apq8064_reserve_fixed_area(unsigned long fixed_area_size) } /** - * Reserve memory for ION and calculate amount of reusable memory for fmem. - * We only reserve memory for heaps that are not reusable. However, we only - * support one reusable heap at the moment so we ignore the reusable flag for - * other than the first heap with reusable flag set. Also handle special case + * Reserve memory for ION. Also handle special case * for video heaps (MM,FW, and MFC). Video requires heaps MM and MFC to be * at a higher address than FW in addition to not more than 256MB away from the - * base address of the firmware. This means that if MM is reusable the other - * two heaps must be allocated in the same region as FW. This is handled by the - * mem_is_fmem flag in the platform data. In addition the MM heap must be + * base address of the firmware. In addition the MM heap must be * adjacent to the FW heap for content protection purposes. */ static void __init reserve_ion_memory(void) @@ -2502,7 +2479,6 @@ static struct platform_device *ep_devices[] __initdata = { &apq8064_device_hsusb_host, &android_usb_device, &msm_device_wcnss_wlan, - &apq8064_fmem_device, #ifdef CONFIG_ION_MSM &apq8064_ion_dev, #endif @@ -2619,7 +2595,6 @@ static struct platform_device *common_devices[] __initdata = { &android_usb_device, &msm_device_wcnss_wlan, &msm_device_iris_fm, - &apq8064_fmem_device, #ifdef CONFIG_ION_MSM &apq8064_ion_dev, #endif diff --git a/arch/arm/mach-msm/board-8930.c b/arch/arm/mach-msm/board-8930.c index e097fafce43..d79464aa901 100644 --- a/arch/arm/mach-msm/board-8930.c +++ b/arch/arm/mach-msm/board-8930.c @@ -79,7 +79,6 @@ #include #include #include -#include #include #include @@ -183,9 +182,6 @@ static int __init msm_contig_mem_size_setup(char *p) early_param("msm_contig_mem_size", msm_contig_mem_size_setup); #endif -struct fmem_platform_data msm8930_fmem_pdata = { -}; - #define DSP_RAM_BASE_8960 0x8da00000 #define DSP_RAM_SIZE_8960 0x1800000 static int dspcrashd_pdata_8960 = 0xDEADDEAD; @@ -230,35 +226,28 @@ static int msm8930_paddr_to_memtype(phys_addr_t paddr) return MEMTYPE_EBI1; } -#define FMEM_ENABLED 0 #ifdef CONFIG_ION_MSM #ifdef CONFIG_MSM_MULTIMEDIA_USE_ION static struct ion_cp_heap_pdata cp_mm_msm8930_ion_pdata = { .permission_type = IPT_TYPE_MM_CARVEOUT, .align = PAGE_SIZE, - .reusable = FMEM_ENABLED, - .mem_is_fmem = FMEM_ENABLED, .fixed_position = FIXED_MIDDLE, }; static struct ion_cp_heap_pdata cp_mfc_msm8930_ion_pdata = { .permission_type = IPT_TYPE_MFC_SHAREDMEM, .align = PAGE_SIZE, - .reusable = 0, - .mem_is_fmem = FMEM_ENABLED, .fixed_position = FIXED_HIGH, }; static struct ion_co_heap_pdata co_msm8930_ion_pdata = { .adjacent_mem_id = INVALID_HEAP_ID, .align = PAGE_SIZE, - .mem_is_fmem = 0, }; static struct ion_co_heap_pdata fw_co_msm8930_ion_pdata = { .adjacent_mem_id = ION_CP_MM_HEAP_ID, .align = SZ_128K, - .mem_is_fmem = FMEM_ENABLED, .fixed_position = FIXED_LOW, }; #endif @@ -382,12 +371,6 @@ static struct platform_device msm8930_ion_dev = { }; #endif -struct platform_device msm8930_fmem_device = { - .name = "fmem", - .id = 1, - .dev = { .platform_data = &msm8930_fmem_pdata }, -}; - static void __init reserve_mem_for_ion(enum ion_memory_types mem_type, unsigned long size) { @@ -413,15 +396,10 @@ static void __init msm8930_reserve_fixed_area(unsigned long fixed_area_size) } /** - * Reserve memory for ION and calculate amount of reusable memory for fmem. - * We only reserve memory for heaps that are not reusable. However, we only - * support one reusable heap at the moment so we ignore the reusable flag for - * other than the first heap with reusable flag set. Also handle special case + * Reserve memory for ION. Also handle special case * for video heaps (MM,FW, and MFC). Video requires heaps MM and MFC to be * at a higher address than FW in addition to not more than 256MB away from the - * base address of the firmware. This means that if MM is reusable the other - * two heaps must be allocated in the same region as FW. This is handled by the - * mem_is_fmem flag in the platform data. In addition the MM heap must be + * base address of the firmware. In addition the MM heap must be * adjacent to the FW heap for content protection purposes. */ static void __init reserve_ion_memory(void) @@ -2274,7 +2252,6 @@ static struct platform_device *common_devices[] __initdata = { #ifdef CONFIG_MSM_FAKE_BATTERY &fish_battery_device, #endif - &msm8930_fmem_device, &msm_device_bam_dmux, &msm_fm_platform_init, diff --git a/arch/arm/mach-msm/board-8960.c b/arch/arm/mach-msm/board-8960.c index b45e690285c..37567ed04a7 100644 --- a/arch/arm/mach-msm/board-8960.c +++ b/arch/arm/mach-msm/board-8960.c @@ -86,7 +86,6 @@ #include #include -#include #include "timer.h" #include "devices.h" @@ -194,9 +193,6 @@ static int __init msm_contig_mem_size_setup(char *p) early_param("msm_contig_mem_size", msm_contig_mem_size_setup); #endif -struct fmem_platform_data msm8960_fmem_pdata = { -}; - #define DSP_RAM_BASE_8960 0x8da00000 #define DSP_RAM_SIZE_8960 0x1800000 static int dspcrashd_pdata_8960 = 0xDEADDEAD; @@ -247,8 +243,6 @@ static int msm8960_paddr_to_memtype(phys_addr_t paddr) static struct ion_cp_heap_pdata cp_mm_msm8960_ion_pdata = { .permission_type = IPT_TYPE_MM_CARVEOUT, .align = SZ_64K, - .reusable = FMEM_ENABLED, - .mem_is_fmem = FMEM_ENABLED, .fixed_position = FIXED_MIDDLE, .iommu_map_all = 1, .iommu_2x_map_domain = VIDEO_DOMAIN, @@ -257,21 +251,17 @@ static struct ion_cp_heap_pdata cp_mm_msm8960_ion_pdata = { static struct ion_cp_heap_pdata cp_mfc_msm8960_ion_pdata = { .permission_type = IPT_TYPE_MFC_SHAREDMEM, .align = PAGE_SIZE, - .reusable = 0, - .mem_is_fmem = FMEM_ENABLED, .fixed_position = FIXED_HIGH, }; static struct ion_co_heap_pdata co_msm8960_ion_pdata = { .adjacent_mem_id = INVALID_HEAP_ID, .align = PAGE_SIZE, - .mem_is_fmem = 0, }; static struct ion_co_heap_pdata fw_co_msm8960_ion_pdata = { .adjacent_mem_id = ION_CP_MM_HEAP_ID, .align = SZ_128K, - .mem_is_fmem = FMEM_ENABLED, .fixed_position = FIXED_LOW, }; #endif @@ -394,12 +384,6 @@ static struct platform_device msm8960_ion_dev = { }; #endif -struct platform_device msm8960_fmem_device = { - .name = "fmem", - .id = 1, - .dev = { .platform_data = &msm8960_fmem_pdata }, -}; - static void __init adjust_mem_for_liquid(void) { unsigned int i; @@ -450,15 +434,10 @@ static void __init msm8960_reserve_fixed_area(unsigned long fixed_area_size) } /** - * Reserve memory for ION and calculate amount of reusable memory for fmem. - * We only reserve memory for heaps that are not reusable. However, we only - * support one reusable heap at the moment so we ignore the reusable flag for - * other than the first heap with reusable flag set. Also handle special case + * Reserve memory for ION. Also handle special case * for video heaps (MM,FW, and MFC). Video requires heaps MM and MFC to be * at a higher address than FW in addition to not more than 256MB away from the - * base address of the firmware. This means that if MM is reusable the other - * two heaps must be allocated in the same region as FW. This is handled by the - * mem_is_fmem flag in the platform data. In addition the MM heap must be + * base address of the firmware. In addition the MM heap must be * adjacent to the FW heap for content protection purposes. */ static void __init reserve_ion_memory(void) @@ -2729,7 +2708,6 @@ static struct platform_device *common_devices[] __initdata = { #ifdef CONFIG_BATTERY_BCL &battery_bcl_device, #endif - &msm8960_fmem_device, &msm_device_bam_dmux, &msm_fm_platform_init, #if defined(CONFIG_TSIF) || defined(CONFIG_TSIF_MODULE) diff --git a/arch/arm/mach-msm/include/mach/msm_memtypes.h b/arch/arm/mach-msm/include/mach/msm_memtypes.h index 5c8f525ffeb..95c4fe38009 100644 --- a/arch/arm/mach-msm/include/mach/msm_memtypes.h +++ b/arch/arm/mach-msm/include/mach/msm_memtypes.h @@ -79,5 +79,4 @@ int __init dt_scan_for_memory_reserve(unsigned long node, const char *uname, int __init dt_scan_for_memory_hole(unsigned long node, const char *uname, int depth, void *data); void adjust_meminfo(unsigned long start, unsigned long size); -unsigned long __init reserve_memory_for_fmem(unsigned long, unsigned long); #endif diff --git a/arch/arm/mach-msm/memory.c b/arch/arm/mach-msm/memory.c index 7a7fb993cc7..2ce4fa0135e 100644 --- a/arch/arm/mach-msm/memory.c +++ b/arch/arm/mach-msm/memory.c @@ -43,7 +43,6 @@ /* fixme */ #include #include <../../mm/mm.h> -#include #if defined(CONFIG_ARCH_MSM7X27) static void *strongly_ordered_page; @@ -251,16 +250,6 @@ void store_ttbr0(void) : "=r" (msm_ttbr0)); } -int request_fmem_c_region(void *unused) -{ - return fmem_set_state(FMEM_C_STATE); -} - -int release_fmem_c_region(void *unused) -{ - return fmem_set_state(FMEM_T_STATE); -} - static char * const memtype_names[] = { [MEMTYPE_SMI_KERNEL] = "SMI_KERNEL", [MEMTYPE_SMI] = "SMI", diff --git a/drivers/gpu/ion/ion_cp_heap.c b/drivers/gpu/ion/ion_cp_heap.c index f1868a8c153..f2f4fad08a3 100644 --- a/drivers/gpu/ion/ion_cp_heap.c +++ b/drivers/gpu/ion/ion_cp_heap.c @@ -203,8 +203,7 @@ static void ion_on_last_free(struct ion_heap *heap) } /** - * Protects memory if heap is unsecured heap. Also ensures that we are in - * the correct FMEM state if this heap is a reusable heap. + * Protects memory if heap is unsecured heap. * Must be called with heap->lock locked. */ static int ion_cp_protect(struct ion_heap *heap, int version, void *data) @@ -244,8 +243,7 @@ out: } /** - * Unprotects memory if heap is secure heap. Also ensures that we are in - * the correct FMEM state if this heap is a reusable heap. + * Unprotects memory if heap is secure heap. * Must be called with heap->lock locked. */ static void ion_cp_unprotect(struct ion_heap *heap, int version, void *data) diff --git a/drivers/gpu/ion/msm/msm_ion.c b/drivers/gpu/ion/msm/msm_ion.c index 4a45313e3aa..5920f7b7561 100644 --- a/drivers/gpu/ion/msm/msm_ion.c +++ b/drivers/gpu/ion/msm/msm_ion.c @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include @@ -419,22 +418,9 @@ static void ion_set_base_address(struct ion_platform_heap *heap, struct ion_co_heap_pdata *co_heap_data, struct ion_cp_heap_pdata *cp_data) { - if (cp_data->reusable) { - const struct fmem_data *fmem_info = fmem_get_info(); - - if (!fmem_info) { - pr_err("fmem info pointer NULL!\n"); - BUG(); - } - - heap->base = fmem_info->phys - fmem_info->reserved_size_low; - cp_data->virt_addr = fmem_info->virt; - pr_info("ION heap %s using FMEM\n", shared_heap->name); - } else { - heap->base = msm_ion_get_base(heap->size + shared_heap->size, - shared_heap->memory_type, - co_heap_data->align); - } + heap->base = msm_ion_get_base(heap->size + shared_heap->size, + shared_heap->memory_type, + co_heap_data->align); if (heap->base) { shared_heap->base = heap->base + heap->size; cp_data->secure_base = heap->base; @@ -460,15 +446,6 @@ static void allocate_co_memory(struct ion_platform_heap *heap, struct ion_cp_heap_pdata *cp_data = (struct ion_cp_heap_pdata *) shared_heap->extra_data; if (cp_data->fixed_position == FIXED_MIDDLE) { - const struct fmem_data *fmem_info = - fmem_get_info(); - - if (!fmem_info) { - pr_err("fmem info pointer NULL!\n"); - BUG(); - } - - cp_data->virt_addr = fmem_info->virt; if (!cp_data->secure_base) { cp_data->secure_base = heap->base; cp_data->secure_size = @@ -520,17 +497,6 @@ static void msm_ion_allocate(struct ion_platform_heap *heap) struct ion_cp_heap_pdata *data = (struct ion_cp_heap_pdata *) heap->extra_data; - if (data->reusable) { - const struct fmem_data *fmem_info = - fmem_get_info(); - heap->base = fmem_info->phys; - data->virt_addr = fmem_info->virt; - pr_info("ION heap %s using FMEM\n", heap->name); - } else if (data->mem_is_fmem) { - const struct fmem_data *fmem_info = - fmem_get_info(); - heap->base = fmem_info->phys + fmem_info->size; - } align = data->align; break; } diff --git a/drivers/staging/Kconfig b/drivers/staging/Kconfig index bcec934a43e..97d412d9145 100644 --- a/drivers/staging/Kconfig +++ b/drivers/staging/Kconfig @@ -84,8 +84,6 @@ source "drivers/staging/zram/Kconfig" source "drivers/staging/zcache/Kconfig" -source "drivers/staging/qcache/Kconfig" - source "drivers/staging/zsmalloc/Kconfig" source "drivers/staging/wlags49_h2/Kconfig" diff --git a/drivers/staging/Makefile b/drivers/staging/Makefile index c31f2ec1cc4..ffe7d44374e 100644 --- a/drivers/staging/Makefile +++ b/drivers/staging/Makefile @@ -34,7 +34,6 @@ obj-$(CONFIG_DX_SEP) += sep/ obj-$(CONFIG_IIO) += iio/ obj-$(CONFIG_ZRAM) += zram/ obj-$(CONFIG_ZCACHE) += zcache/ -obj-$(CONFIG_QCACHE) += qcache/ obj-$(CONFIG_ZSMALLOC) += zsmalloc/ obj-$(CONFIG_WLAGS49_H2) += wlags49_h2/ obj-$(CONFIG_WLAGS49_H25) += wlags49_h25/ diff --git a/drivers/staging/qcache/Kconfig b/drivers/staging/qcache/Kconfig deleted file mode 100644 index 389341cf5d9..00000000000 --- a/drivers/staging/qcache/Kconfig +++ /dev/null @@ -1,8 +0,0 @@ -config QCACHE - tristate "Dynamic compression of clean pagecache pages" - depends on CLEANCACHE - select LZO_COMPRESS - select LZO_DECOMPRESS - default n - help - Qcache is the backend for fmem diff --git a/drivers/staging/qcache/Makefile b/drivers/staging/qcache/Makefile deleted file mode 100644 index 4fdf05c147a..00000000000 --- a/drivers/staging/qcache/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -qcache-y := qcache-main.o tmem.o fmem.o - -obj-$(CONFIG_QCACHE) += qcache.o diff --git a/drivers/staging/qcache/fmem.c b/drivers/staging/qcache/fmem.c deleted file mode 100644 index 0609f4af64b..00000000000 --- a/drivers/staging/qcache/fmem.c +++ /dev/null @@ -1,344 +0,0 @@ -/* - * - * Copyright (c) 2011-2012, The Linux Foundation. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 and - * only version 2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - */ - -#include -#include -#include -#include -#ifdef CONFIG_MEMORY_HOTPLUG -#include -#include -#endif -#include "tmem.h" -#include - -struct fmem_data fmem_data; -enum fmem_state fmem_state; -static spinlock_t fmem_state_lock; - -#ifdef CONFIG_MEMORY_HOTPLUG -static unsigned int section_powered_off[NR_MEM_SECTIONS]; -static unsigned int fmem_section_start, fmem_section_end; -#endif - -void *fmem_map_virtual_area(int cacheability) -{ - unsigned long addr; - const struct mem_type *type; - int ret; - - addr = (unsigned long) fmem_data.area->addr; - type = get_mem_type(cacheability); - ret = ioremap_pages(addr, fmem_data.phys, fmem_data.size, type); - if (ret) - return ERR_PTR(ret); - - fmem_data.virt = fmem_data.area->addr; - - return fmem_data.virt; -} - -void fmem_unmap_virtual_area(void) -{ - unmap_kernel_range((unsigned long)fmem_data.virt, fmem_data.size); - fmem_data.virt = NULL; -} - -static int fmem_probe(struct platform_device *pdev) -{ - struct fmem_platform_data *pdata = pdev->dev.platform_data; - - if (!pdata->phys) - pdata->phys = allocate_contiguous_ebi_nomap(pdata->size, - pdata->align); - -#ifdef CONFIG_MEMORY_HOTPLUG - fmem_section_start = pdata->phys >> PA_SECTION_SHIFT; - fmem_section_end = (pdata->phys - 1 + pdata->size) >> PA_SECTION_SHIFT; -#endif - fmem_data.phys = pdata->phys + pdata->reserved_size_low; - fmem_data.size = pdata->size - pdata->reserved_size_low - - pdata->reserved_size_high; - fmem_data.reserved_size_low = pdata->reserved_size_low; - fmem_data.reserved_size_high = pdata->reserved_size_high; - - if (!fmem_data.size) - return -ENODEV; - - fmem_data.area = get_vm_area(fmem_data.size, VM_IOREMAP); - if (!fmem_data.area) - return -ENOMEM; - - if (!fmem_map_virtual_area(MT_DEVICE_CACHED)) { - remove_vm_area(fmem_data.area->addr); - return -ENOMEM; - } - pr_info("fmem phys %lx virt %p size %lx\n", - fmem_data.phys, fmem_data.virt, fmem_data.size); - - spin_lock_init(&fmem_state_lock); - - return 0; -} - -static int fmem_remove(struct platform_device *pdev) -{ - return 0; -} - -static struct platform_driver fmem_driver = { - .probe = fmem_probe, - .remove = fmem_remove, - .driver = { .name = "fmem" } -}; - -#ifdef CONFIG_SYSFS -static ssize_t fmem_state_show(struct kobject *kobj, - struct kobj_attribute *attr, - char *buf) -{ - if (fmem_state == FMEM_T_STATE) - return snprintf(buf, 3, "t\n"); - else if (fmem_state == FMEM_C_STATE) - return snprintf(buf, 3, "c\n"); -#ifdef CONFIG_MEMORY_HOTPLUG - else if (fmem_state == FMEM_O_STATE) - return snprintf(buf, 3, "o\n"); -#endif - else if (fmem_state == FMEM_UNINITIALIZED) - return snprintf(buf, 15, "uninitialized\n"); - return snprintf(buf, 3, "?\n"); -} - -static ssize_t fmem_state_store(struct kobject *kobj, - struct kobj_attribute *attr, - const char *buf, size_t count) -{ - int ret = -EINVAL; - - if (!strncmp(buf, "t", 1)) - ret = fmem_set_state(FMEM_T_STATE); - else if (!strncmp(buf, "c", 1)) - ret = fmem_set_state(FMEM_C_STATE); -#ifdef CONFIG_MEMORY_HOTPLUG - else if (!strncmp(buf, "o", 1)) - ret = fmem_set_state(FMEM_O_STATE); -#endif - if (ret) - return ret; - return 1; -} - -static struct kobj_attribute fmem_state_attr = { - .attr = { .name = "state", .mode = 0644 }, - .show = fmem_state_show, - .store = fmem_state_store, -}; - -static struct attribute *fmem_attrs[] = { - &fmem_state_attr.attr, - NULL, -}; - -static struct attribute_group fmem_attr_group = { - .attrs = fmem_attrs, - .name = "fmem", -}; - -static int fmem_create_sysfs(void) -{ - int ret = 0; - - ret = sysfs_create_group(mm_kobj, &fmem_attr_group); - if (ret) - pr_err("fmem: can't create sysfs\n"); - return ret; -} - -#endif - -#ifdef CONFIG_MEMORY_HOTPLUG -bool fmem_is_disjoint(unsigned long start_pfn, unsigned long nr_pages) -{ - unsigned long fmem_start_pfn, fmem_end_pfn; - unsigned long unstable_end_pfn; - unsigned long highest_start_pfn, lowest_end_pfn; - - fmem_start_pfn = (fmem_data.phys - fmem_data.reserved_size_low) - >> PAGE_SHIFT; - fmem_end_pfn = (fmem_data.phys + fmem_data.size + - fmem_data.reserved_size_high - 1) >> PAGE_SHIFT; - unstable_end_pfn = start_pfn + nr_pages - 1; - - highest_start_pfn = max(fmem_start_pfn, start_pfn); - lowest_end_pfn = min(fmem_end_pfn, unstable_end_pfn); - - return lowest_end_pfn < highest_start_pfn; -} - -static int fmem_mem_going_offline_callback(void *arg) -{ - struct memory_notify *marg = arg; - - if (fmem_is_disjoint(marg->start_pfn, marg->nr_pages)) - return 0; - return fmem_set_state(FMEM_O_STATE); -} - -static void fmem_mem_online_callback(void *arg) -{ - struct memory_notify *marg = arg; - int i; - - section_powered_off[marg->start_pfn >> PFN_SECTION_SHIFT] = 0; - - if (fmem_state != FMEM_O_STATE) - return; - - for (i = fmem_section_start; i <= fmem_section_end; i++) { - if (section_powered_off[i]) - return; - } - - fmem_set_state(FMEM_T_STATE); -} - -static void fmem_mem_offline_callback(void *arg) -{ - struct memory_notify *marg = arg; - - section_powered_off[marg->start_pfn >> PFN_SECTION_SHIFT] = 1; -} - -static int fmem_memory_callback(struct notifier_block *self, - unsigned long action, void *arg) -{ - int ret = 0; - - if (fmem_state == FMEM_UNINITIALIZED) - return NOTIFY_OK; - - switch (action) { - case MEM_ONLINE: - fmem_mem_online_callback(arg); - break; - case MEM_GOING_OFFLINE: - ret = fmem_mem_going_offline_callback(arg); - break; - case MEM_OFFLINE: - fmem_mem_offline_callback(arg); - break; - case MEM_GOING_ONLINE: - case MEM_CANCEL_ONLINE: - case MEM_CANCEL_OFFLINE: - break; - } - if (ret) - ret = notifier_from_errno(ret); - else - ret = NOTIFY_OK; - return ret; -} -#endif - -static int __init fmem_init(void) -{ -#ifdef CONFIG_MEMORY_HOTPLUG - hotplug_memory_notifier(fmem_memory_callback, 0); -#endif - return platform_driver_register(&fmem_driver); -} - -static void __exit fmem_exit(void) -{ - platform_driver_unregister(&fmem_driver); -} - -struct fmem_data *fmem_get_info(void) -{ - return &fmem_data; -} -EXPORT_SYMBOL(fmem_get_info); - -void lock_fmem_state(void) -{ - spin_lock(&fmem_state_lock); -} - -void unlock_fmem_state(void) -{ - spin_unlock(&fmem_state_lock); -} - -int fmem_set_state(enum fmem_state new_state) -{ - int ret = 0; - int create_sysfs = 0; - - lock_fmem_state(); - if (fmem_state == new_state) - goto out; - - if (fmem_state == FMEM_UNINITIALIZED) { - if (new_state == FMEM_T_STATE) { - tmem_enable(); - create_sysfs = 1; - goto out_set; - } else { - ret = -EINVAL; - goto out; - } - } - -#ifdef CONFIG_MEMORY_HOTPLUG - if (fmem_state == FMEM_C_STATE && new_state == FMEM_O_STATE) { - ret = -EAGAIN; - goto out; - } - - if (fmem_state == FMEM_O_STATE && new_state == FMEM_C_STATE) { - pr_warn("attempting to use powered off memory as fmem\n"); - ret = -EAGAIN; - goto out; - } -#endif - - if (new_state == FMEM_T_STATE) { - void *v; - v = fmem_map_virtual_area(MT_DEVICE_CACHED); - if (IS_ERR_OR_NULL(v)) { - ret = PTR_ERR(v); - goto out; - } - tmem_enable(); - } else { - tmem_disable(); - fmem_unmap_virtual_area(); - } - -out_set: - fmem_state = new_state; -out: - unlock_fmem_state(); -#ifdef CONFIG_SYSFS - if (create_sysfs) - fmem_create_sysfs(); -#endif - return ret; -} -EXPORT_SYMBOL(fmem_set_state); - -arch_initcall(fmem_init); -module_exit(fmem_exit); diff --git a/drivers/staging/qcache/qcache-main.c b/drivers/staging/qcache/qcache-main.c deleted file mode 100644 index f416cfcbbc1..00000000000 --- a/drivers/staging/qcache/qcache-main.c +++ /dev/null @@ -1,1358 +0,0 @@ -/* - * Copyright (c) 2010,2011, Dan Magenheimer, Oracle Corp. - * Copyright (c) 2010,2011, Nitin Gupta - * Copyright (c) 2011-2012, The Linux Foundation. All rights reserved. - * - * Qcache provides an in-kernel "host implementation" for transcendent memory - * and, thus indirectly, for cleancache and frontswap. Qcache includes a - * page-accessible memory [1] interface, utilizing lzo1x compression: - * 1) "compression buddies" ("zbud") is used for ephemeral pages - * Zbud allows pairs (and potentially, - * in the future, more than a pair of) compressed pages to be closely linked - * so that reclaiming can be done via the kernel's physical-page-oriented - * "shrinker" interface. - * - * [1] For a definition of page-accessible memory (aka PAM), see: - * http://marc.info/?l=linux-mm&m=127811271605009 - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "tmem.h" - -#if !defined(CONFIG_CLEANCACHE) -#error "qcache is useless without CONFIG_CLEANCACHE" -#endif -#include - -#define ZCACHE_GFP_MASK \ - (__GFP_FS | __GFP_NORETRY | __GFP_NOWARN | __GFP_NOMEMALLOC) - -#define MAX_POOLS_PER_CLIENT 16 - -#define MAX_CLIENTS 16 -#define LOCAL_CLIENT ((uint16_t)-1) - -MODULE_LICENSE("GPL"); - -struct zcache_client { - struct tmem_pool *tmem_pools[MAX_POOLS_PER_CLIENT]; - struct xv_pool *xvpool; - bool allocated; - atomic_t refcount; -}; - -struct qcache_info { - void *addr; - unsigned long *bitmap; - spinlock_t lock; - unsigned pages; -}; -static struct qcache_info qcache_info; -static unsigned long zcache_qc_allocated; -static unsigned long zcache_qc_freed; -static unsigned long zcache_qc_used; -static unsigned long zcache_qc_max_used; - -static struct zcache_client zcache_host; -static struct zcache_client zcache_clients[MAX_CLIENTS]; - -static inline uint16_t get_client_id_from_client(struct zcache_client *cli) -{ - BUG_ON(cli == NULL); - if (cli == &zcache_host) - return LOCAL_CLIENT; - return cli - &zcache_clients[0]; -} - -static inline bool is_local_client(struct zcache_client *cli) -{ - return cli == &zcache_host; -} - -/********** - * Compression buddies ("zbud") provides for packing two (or, possibly - * in the future, more) compressed ephemeral pages into a single "raw" - * (physical) page and tracking them with data structures so that - * the raw pages can be easily reclaimed. - * - * A zbud page ("zbpg") is an aligned page containing a list_head, - * a lock, and two "zbud headers". The remainder of the physical - * page is divided up into aligned 64-byte "chunks" which contain - * the compressed data for zero, one, or two zbuds. Each zbpg - * resides on: (1) an "unused list" if it has no zbuds; (2) a - * "buddied" list if it is fully populated with two zbuds; or - * (3) one of PAGE_SIZE/64 "unbuddied" lists indexed by how many chunks - * the one unbuddied zbud uses. The data inside a zbpg cannot be - * read or written unless the zbpg's lock is held. - */ - -#define ZBH_SENTINEL 0x43214321 -#define ZBPG_SENTINEL 0xdeadbeef - -#define ZBUD_MAX_BUDS 2 - -struct zbud_hdr { - uint16_t client_id; - uint16_t pool_id; - struct tmem_oid oid; - uint32_t index; - uint16_t size; /* compressed size in bytes, zero means unused */ - DECL_SENTINEL -}; - -struct zbud_page { - struct list_head bud_list; - spinlock_t lock; - struct zbud_hdr buddy[ZBUD_MAX_BUDS]; - DECL_SENTINEL - /* followed by NUM_CHUNK aligned CHUNK_SIZE-byte chunks */ -}; - -#define CHUNK_SHIFT 6 -#define CHUNK_SIZE (1 << CHUNK_SHIFT) -#define CHUNK_MASK (~(CHUNK_SIZE-1)) -#define NCHUNKS (((PAGE_SIZE - sizeof(struct zbud_page)) & \ - CHUNK_MASK) >> CHUNK_SHIFT) -#define MAX_CHUNK (NCHUNKS-1) - -static struct { - struct list_head list; - unsigned count; -} zbud_unbuddied[NCHUNKS]; -/* list N contains pages with N chunks USED and NCHUNKS-N unused */ -/* element 0 is never used but optimizing that isn't worth it */ -static unsigned long zbud_cumul_chunk_counts[NCHUNKS]; - -struct list_head zbud_buddied_list; -static unsigned long zcache_zbud_buddied_count; - -/* protects the buddied list and all unbuddied lists */ -static DEFINE_SPINLOCK(zbud_budlists_spinlock); - -static atomic_t zcache_zbud_curr_raw_pages; -static atomic_t zcache_zbud_curr_zpages; -static unsigned long zcache_zbud_curr_zbytes; -static unsigned long zcache_zbud_cumul_zpages; -static unsigned long zcache_zbud_cumul_zbytes; -static unsigned long zcache_compress_poor; -static unsigned long zcache_mean_compress_poor; - -/* forward references */ -static void *zcache_get_free_page(void); - -static void *qcache_alloc(void) -{ - void *addr; - unsigned long flags; - int offset; - struct qcache_info *qc = &qcache_info; - - spin_lock_irqsave(&qc->lock, flags); - offset = bitmap_find_free_region(qc->bitmap, qc->pages, 0); - - if (offset < 0) { - spin_unlock_irqrestore(&qc->lock, flags); - return NULL; - } - - zcache_qc_allocated++; - zcache_qc_used++; - zcache_qc_max_used = max(zcache_qc_max_used, zcache_qc_used); - spin_unlock_irqrestore(&qc->lock, flags); - - addr = qc->addr + offset * PAGE_SIZE; - - return addr; -} - -static void qcache_free(void *addr) -{ - unsigned long flags; - int offset; - struct qcache_info *qc = &qcache_info; - - offset = (addr - qc->addr) / PAGE_SIZE; - - spin_lock_irqsave(&qc->lock, flags); - bitmap_release_region(qc->bitmap, offset, 0); - - zcache_qc_freed++; - zcache_qc_used--; - spin_unlock_irqrestore(&qc->lock, flags); -} - -/* - * zbud helper functions - */ - -static inline unsigned zbud_max_buddy_size(void) -{ - return MAX_CHUNK << CHUNK_SHIFT; -} - -static inline unsigned zbud_size_to_chunks(unsigned size) -{ - BUG_ON(size == 0 || size > zbud_max_buddy_size()); - return (size + CHUNK_SIZE - 1) >> CHUNK_SHIFT; -} - -static inline int zbud_budnum(struct zbud_hdr *zh) -{ - unsigned offset = (unsigned long)zh & (PAGE_SIZE - 1); - struct zbud_page *zbpg = NULL; - unsigned budnum = -1U; - int i; - - for (i = 0; i < ZBUD_MAX_BUDS; i++) - if (offset == offsetof(typeof(*zbpg), buddy[i])) { - budnum = i; - break; - } - BUG_ON(budnum == -1U); - return budnum; -} - -static char *zbud_data(struct zbud_hdr *zh, unsigned size) -{ - struct zbud_page *zbpg; - char *p; - unsigned budnum; - - ASSERT_SENTINEL(zh, ZBH); - budnum = zbud_budnum(zh); - BUG_ON(size == 0 || size > zbud_max_buddy_size()); - zbpg = container_of(zh, struct zbud_page, buddy[budnum]); - p = (char *)zbpg; - if (budnum == 0) - p += ((sizeof(struct zbud_page) + CHUNK_SIZE - 1) & - CHUNK_MASK); - else if (budnum == 1) - p += PAGE_SIZE - ((size + CHUNK_SIZE - 1) & CHUNK_MASK); - return p; -} - -/* - * zbud raw page management - */ - -static struct zbud_page *zbud_alloc_raw_page(void) -{ - struct zbud_page *zbpg = NULL; - struct zbud_hdr *zh0, *zh1; - - zbpg = zcache_get_free_page(); - if (likely(zbpg != NULL)) { - INIT_LIST_HEAD(&zbpg->bud_list); - zh0 = &zbpg->buddy[0]; zh1 = &zbpg->buddy[1]; - spin_lock_init(&zbpg->lock); - atomic_inc(&zcache_zbud_curr_raw_pages); - INIT_LIST_HEAD(&zbpg->bud_list); - SET_SENTINEL(zbpg, ZBPG); - zh0->size = 0; zh1->size = 0; - tmem_oid_set_invalid(&zh0->oid); - tmem_oid_set_invalid(&zh1->oid); - } - return zbpg; -} - -static void zbud_free_raw_page(struct zbud_page *zbpg) -{ - struct zbud_hdr *zh0 = &zbpg->buddy[0], *zh1 = &zbpg->buddy[1]; - - ASSERT_SENTINEL(zbpg, ZBPG); - BUG_ON(!list_empty(&zbpg->bud_list)); - BUG_ON(zh0->size != 0 || tmem_oid_valid(&zh0->oid)); - BUG_ON(zh1->size != 0 || tmem_oid_valid(&zh1->oid)); - INVERT_SENTINEL(zbpg, ZBPG); - spin_unlock(&zbpg->lock); - qcache_free(zbpg); -} - -/* - * core zbud handling routines - */ - -static unsigned zbud_free(struct zbud_hdr *zh) -{ - unsigned size; - - ASSERT_SENTINEL(zh, ZBH); - BUG_ON(!tmem_oid_valid(&zh->oid)); - size = zh->size; - BUG_ON(zh->size == 0 || zh->size > zbud_max_buddy_size()); - zh->size = 0; - tmem_oid_set_invalid(&zh->oid); - INVERT_SENTINEL(zh, ZBH); - zcache_zbud_curr_zbytes -= size; - atomic_dec(&zcache_zbud_curr_zpages); - return size; -} - -static void zbud_free_and_delist(struct zbud_hdr *zh) -{ - unsigned chunks; - struct zbud_hdr *zh_other; - unsigned budnum = zbud_budnum(zh), size; - struct zbud_page *zbpg = - container_of(zh, struct zbud_page, buddy[budnum]); - - spin_lock(&zbpg->lock); - if (list_empty(&zbpg->bud_list)) { - spin_unlock(&zbpg->lock); - return; - } - size = zbud_free(zh); - zh_other = &zbpg->buddy[(budnum == 0) ? 1 : 0]; - if (zh_other->size == 0) { /* was unbuddied: unlist and free */ - chunks = zbud_size_to_chunks(size) ; - spin_lock(&zbud_budlists_spinlock); - BUG_ON(list_empty(&zbud_unbuddied[chunks].list)); - list_del_init(&zbpg->bud_list); - zbud_unbuddied[chunks].count--; - spin_unlock(&zbud_budlists_spinlock); - zbud_free_raw_page(zbpg); - } else { /* was buddied: move remaining buddy to unbuddied list */ - chunks = zbud_size_to_chunks(zh_other->size) ; - spin_lock(&zbud_budlists_spinlock); - list_del_init(&zbpg->bud_list); - zcache_zbud_buddied_count--; - list_add_tail(&zbpg->bud_list, &zbud_unbuddied[chunks].list); - zbud_unbuddied[chunks].count++; - spin_unlock(&zbud_budlists_spinlock); - spin_unlock(&zbpg->lock); - } -} - -static struct zbud_hdr *zbud_create(uint16_t client_id, uint16_t pool_id, - struct tmem_oid *oid, - uint32_t index, struct page *page, - void *cdata, unsigned size) -{ - struct zbud_hdr *zh0, *zh1, *zh = NULL; - struct zbud_page *zbpg = NULL, *ztmp; - unsigned nchunks; - char *to; - int i, found_good_buddy = 0; - - nchunks = zbud_size_to_chunks(size) ; - for (i = MAX_CHUNK - nchunks + 1; i > 0; i--) { - spin_lock(&zbud_budlists_spinlock); - if (!list_empty(&zbud_unbuddied[i].list)) { - list_for_each_entry_safe(zbpg, ztmp, - &zbud_unbuddied[i].list, bud_list) { - if (spin_trylock(&zbpg->lock)) { - found_good_buddy = i; - goto found_unbuddied; - } - } - } - spin_unlock(&zbud_budlists_spinlock); - } - /* didn't find a good buddy, try allocating a new page */ - zbpg = zbud_alloc_raw_page(); - if (unlikely(zbpg == NULL)) - goto out; - /* ok, have a page, now compress the data before taking locks */ - spin_lock(&zbpg->lock); - spin_lock(&zbud_budlists_spinlock); - list_add_tail(&zbpg->bud_list, &zbud_unbuddied[nchunks].list); - zbud_unbuddied[nchunks].count++; - zh = &zbpg->buddy[0]; - goto init_zh; - -found_unbuddied: - zh0 = &zbpg->buddy[0]; zh1 = &zbpg->buddy[1]; - BUG_ON(!((zh0->size == 0) ^ (zh1->size == 0))); - if (zh0->size != 0) { /* buddy0 in use, buddy1 is vacant */ - ASSERT_SENTINEL(zh0, ZBH); - zh = zh1; - } else if (zh1->size != 0) { /* buddy1 in use, buddy0 is vacant */ - ASSERT_SENTINEL(zh1, ZBH); - zh = zh0; - } else - BUG(); - list_del_init(&zbpg->bud_list); - zbud_unbuddied[found_good_buddy].count--; - list_add_tail(&zbpg->bud_list, &zbud_buddied_list); - zcache_zbud_buddied_count++; - -init_zh: - SET_SENTINEL(zh, ZBH); - zh->size = size; - zh->index = index; - zh->oid = *oid; - zh->pool_id = pool_id; - zh->client_id = client_id; - /* can wait to copy the data until the list locks are dropped */ - spin_unlock(&zbud_budlists_spinlock); - - to = zbud_data(zh, size); - memcpy(to, cdata, size); - spin_unlock(&zbpg->lock); - zbud_cumul_chunk_counts[nchunks]++; - atomic_inc(&zcache_zbud_curr_zpages); - zcache_zbud_cumul_zpages++; - zcache_zbud_curr_zbytes += size; - zcache_zbud_cumul_zbytes += size; -out: - return zh; -} - -static int zbud_decompress(struct page *page, struct zbud_hdr *zh) -{ - struct zbud_page *zbpg; - unsigned budnum = zbud_budnum(zh); - size_t out_len = PAGE_SIZE; - char *to_va, *from_va; - unsigned size; - int ret = 0; - - zbpg = container_of(zh, struct zbud_page, buddy[budnum]); - spin_lock(&zbpg->lock); - if (list_empty(&zbpg->bud_list)) { - ret = -EINVAL; - goto out; - } - ASSERT_SENTINEL(zh, ZBH); - BUG_ON(zh->size == 0 || zh->size > zbud_max_buddy_size()); - to_va = kmap_atomic(page); - size = zh->size; - from_va = zbud_data(zh, size); - ret = lzo1x_decompress_safe(from_va, size, to_va, &out_len); - BUG_ON(ret != LZO_E_OK); - BUG_ON(out_len != PAGE_SIZE); - kunmap_atomic(to_va); -out: - spin_unlock(&zbpg->lock); - return ret; -} - -static struct tmem_pool *zcache_get_pool_by_id(uint16_t cli_id, - uint16_t poolid); -static void zcache_put_pool(struct tmem_pool *pool); - -static void zbud_init(void) -{ - int i; - - INIT_LIST_HEAD(&zbud_buddied_list); - zcache_zbud_buddied_count = 0; - for (i = 0; i < NCHUNKS; i++) { - INIT_LIST_HEAD(&zbud_unbuddied[i].list); - zbud_unbuddied[i].count = 0; - } -} - -#ifdef CONFIG_SYSFS -/* - * These sysfs routines show a nice distribution of how many zbpg's are - * currently (and have ever been placed) in each unbuddied list. It's fun - * to watch but can probably go away before final merge. - */ -static int zbud_show_unbuddied_list_counts(char *buf) -{ - int i; - char *p = buf; - - for (i = 0; i < NCHUNKS; i++) - p += sprintf(p, "%u ", zbud_unbuddied[i].count); - return p - buf; -} - -static int zbud_show_cumul_chunk_counts(char *buf) -{ - unsigned long i, chunks = 0, total_chunks = 0, sum_total_chunks = 0; - unsigned long total_chunks_lte_21 = 0, total_chunks_lte_32 = 0; - unsigned long total_chunks_lte_42 = 0; - char *p = buf; - - for (i = 0; i < NCHUNKS; i++) { - p += sprintf(p, "%lu ", zbud_cumul_chunk_counts[i]); - chunks += zbud_cumul_chunk_counts[i]; - total_chunks += zbud_cumul_chunk_counts[i]; - sum_total_chunks += i * zbud_cumul_chunk_counts[i]; - if (i == 21) - total_chunks_lte_21 = total_chunks; - if (i == 32) - total_chunks_lte_32 = total_chunks; - if (i == 42) - total_chunks_lte_42 = total_chunks; - } - p += sprintf(p, "<=21:%lu <=32:%lu <=42:%lu, mean:%lu\n", - total_chunks_lte_21, total_chunks_lte_32, total_chunks_lte_42, - chunks == 0 ? 0 : sum_total_chunks / chunks); - return p - buf; -} -#endif - -/* - * zcache core code starts here - */ - -/* useful stats not collected by cleancache or frontswap */ -static unsigned long zcache_flush_total; -static unsigned long zcache_flush_found; -static unsigned long zcache_flobj_total; -static unsigned long zcache_flobj_found; -static unsigned long zcache_failed_eph_puts; - -/* - * Tmem operations assume the poolid implies the invoking client. - * Zcache only has one client (the kernel itself): LOCAL_CLIENT. - * RAMster has each client numbered by cluster node, and a KVM version - * of zcache would have one client per guest and each client might - * have a poolid==N. - */ -static struct tmem_pool *zcache_get_pool_by_id(uint16_t cli_id, uint16_t poolid) -{ - struct tmem_pool *pool = NULL; - struct zcache_client *cli = NULL; - - if (cli_id == LOCAL_CLIENT) - cli = &zcache_host; - else { - if (cli_id >= MAX_CLIENTS) - goto out; - cli = &zcache_clients[cli_id]; - if (cli == NULL) - goto out; - atomic_inc(&cli->refcount); - } - if (poolid < MAX_POOLS_PER_CLIENT) { - pool = cli->tmem_pools[poolid]; - if (pool != NULL) - atomic_inc(&pool->refcount); - } -out: - return pool; -} - -static void zcache_put_pool(struct tmem_pool *pool) -{ - struct zcache_client *cli = NULL; - - if (pool == NULL) - BUG(); - cli = pool->client; - atomic_dec(&pool->refcount); - atomic_dec(&cli->refcount); -} - -int zcache_new_client(uint16_t cli_id) -{ - struct zcache_client *cli = NULL; - int ret = -1; - - if (cli_id == LOCAL_CLIENT) - cli = &zcache_host; - else if ((unsigned int)cli_id < MAX_CLIENTS) - cli = &zcache_clients[cli_id]; - if (cli == NULL) - goto out; - if (cli->allocated) - goto out; - cli->allocated = 1; - ret = 0; -out: - return ret; -} - -/* counters for debugging */ -static unsigned long zcache_failed_get_free_pages; -static unsigned long zcache_failed_alloc; -static unsigned long zcache_put_to_flush; -static unsigned long zcache_aborted_preload; -static unsigned long zcache_aborted_shrink; - -/* - * Ensure that memory allocation requests in zcache don't result - * in direct reclaim requests via the shrinker, which would cause - * an infinite loop. Maybe a GFP flag would be better? - */ -static DEFINE_SPINLOCK(zcache_direct_reclaim_lock); - -/* - * for now, used named slabs so can easily track usage; later can - * either just use kmalloc, or perhaps add a slab-like allocator - * to more carefully manage total memory utilization - */ -static struct kmem_cache *zcache_objnode_cache; -static struct kmem_cache *zcache_obj_cache; -static atomic_t zcache_curr_obj_count = ATOMIC_INIT(0); -static unsigned long zcache_curr_obj_count_max; -static atomic_t zcache_curr_objnode_count = ATOMIC_INIT(0); -static unsigned long zcache_curr_objnode_count_max; - -/* - * to avoid memory allocation recursion (e.g. due to direct reclaim), we - * preload all necessary data structures so the hostops callbacks never - * actually do a malloc - */ -struct zcache_preload { - void *page; - struct tmem_obj *obj; - int nr; - struct tmem_objnode *objnodes[OBJNODE_TREE_MAX_PATH]; -}; -static DEFINE_PER_CPU(struct zcache_preload, zcache_preloads) = { 0, }; - -static int zcache_do_preload(struct tmem_pool *pool) -{ - struct zcache_preload *kp; - struct tmem_objnode *objnode; - struct tmem_obj *obj; - void *page; - int ret = -ENOMEM; - - if (unlikely(zcache_objnode_cache == NULL)) - goto out; - if (unlikely(zcache_obj_cache == NULL)) - goto out; - if (!spin_trylock(&zcache_direct_reclaim_lock)) { - zcache_aborted_preload++; - goto out; - } - preempt_disable(); - kp = &__get_cpu_var(zcache_preloads); - while (kp->nr < ARRAY_SIZE(kp->objnodes)) { - preempt_enable_no_resched(); - objnode = kmem_cache_alloc(zcache_objnode_cache, - ZCACHE_GFP_MASK); - if (unlikely(objnode == NULL)) { - zcache_failed_alloc++; - goto unlock_out; - } - preempt_disable(); - kp = &__get_cpu_var(zcache_preloads); - if (kp->nr < ARRAY_SIZE(kp->objnodes)) - kp->objnodes[kp->nr++] = objnode; - else - kmem_cache_free(zcache_objnode_cache, objnode); - } - preempt_enable_no_resched(); - obj = kmem_cache_alloc(zcache_obj_cache, ZCACHE_GFP_MASK); - if (unlikely(obj == NULL)) { - zcache_failed_alloc++; - goto unlock_out; - } - page = qcache_alloc(); - if (unlikely(page == NULL)) { - zcache_failed_get_free_pages++; - kmem_cache_free(zcache_obj_cache, obj); - goto unlock_out; - } - preempt_disable(); - kp = &__get_cpu_var(zcache_preloads); - if (kp->obj == NULL) - kp->obj = obj; - else - kmem_cache_free(zcache_obj_cache, obj); - if (kp->page == NULL) - kp->page = page; - else - qcache_free(page); - ret = 0; -unlock_out: - spin_unlock(&zcache_direct_reclaim_lock); -out: - return ret; -} - -static void *zcache_get_free_page(void) -{ - struct zcache_preload *kp; - void *page; - - kp = &__get_cpu_var(zcache_preloads); - page = kp->page; - BUG_ON(page == NULL); - kp->page = NULL; - return page; -} - -/* - * zcache implementation for tmem host ops - */ - -static struct tmem_objnode *zcache_objnode_alloc(struct tmem_pool *pool) -{ - struct tmem_objnode *objnode = NULL; - unsigned long count; - struct zcache_preload *kp; - - kp = &__get_cpu_var(zcache_preloads); - if (kp->nr <= 0) - goto out; - objnode = kp->objnodes[kp->nr - 1]; - BUG_ON(objnode == NULL); - kp->objnodes[kp->nr - 1] = NULL; - kp->nr--; - count = atomic_inc_return(&zcache_curr_objnode_count); - if (count > zcache_curr_objnode_count_max) - zcache_curr_objnode_count_max = count; -out: - return objnode; -} - -static void zcache_objnode_free(struct tmem_objnode *objnode, - struct tmem_pool *pool) -{ - atomic_dec(&zcache_curr_objnode_count); - BUG_ON(atomic_read(&zcache_curr_objnode_count) < 0); - kmem_cache_free(zcache_objnode_cache, objnode); -} - -static struct tmem_obj *zcache_obj_alloc(struct tmem_pool *pool) -{ - struct tmem_obj *obj = NULL; - unsigned long count; - struct zcache_preload *kp; - - kp = &__get_cpu_var(zcache_preloads); - obj = kp->obj; - BUG_ON(obj == NULL); - kp->obj = NULL; - count = atomic_inc_return(&zcache_curr_obj_count); - if (count > zcache_curr_obj_count_max) - zcache_curr_obj_count_max = count; - return obj; -} - -static void zcache_obj_free(struct tmem_obj *obj, struct tmem_pool *pool) -{ - atomic_dec(&zcache_curr_obj_count); - BUG_ON(atomic_read(&zcache_curr_obj_count) < 0); - kmem_cache_free(zcache_obj_cache, obj); -} - -static void zcache_flush_all_obj(void) -{ - struct tmem_pool *pool; - int pool_id; - struct zcache_preload *kp; - - kp = &__get_cpu_var(zcache_preloads); - - for (pool_id = 0; pool_id < MAX_POOLS_PER_CLIENT; pool_id++) { - pool = zcache_get_pool_by_id(LOCAL_CLIENT, pool_id); - tmem_flush_pool(pool); - if (pool) - zcache_put_pool(pool); - } - if (kp->page) { - qcache_free(kp->page); - kp->page = NULL; - } - if (zcache_qc_used) - pr_warn("pages used not 0 after qcache flush all, is %ld\n", - zcache_qc_used); -} - -/* - * When zcache is disabled ("frozen"), pools can be created and destroyed, - * but all puts (and thus all other operations that require memory allocation) - * must fail. If zcache is unfrozen, accepts puts, then frozen again, - * data consistency requires all puts while frozen to be converted into - * flushes. - */ -static bool zcache_freeze; - -static void zcache_control(bool freeze) -{ - zcache_freeze = freeze; -} - -static struct tmem_hostops zcache_hostops = { - .obj_alloc = zcache_obj_alloc, - .obj_free = zcache_obj_free, - .objnode_alloc = zcache_objnode_alloc, - .objnode_free = zcache_objnode_free, - .flush_all_obj = zcache_flush_all_obj, - .control = zcache_control, -}; - -/* - * zcache implementations for PAM page descriptor ops - */ - -static atomic_t zcache_curr_eph_pampd_count = ATOMIC_INIT(0); -static unsigned long zcache_curr_eph_pampd_count_max; - -/* forward reference */ -static int zcache_compress(struct page *from, void **out_va, size_t *out_len); - -static void *zcache_pampd_create(char *data, size_t size, bool raw, int eph, - struct tmem_pool *pool, struct tmem_oid *oid, - uint32_t index) -{ - void *pampd = NULL, *cdata; - size_t clen; - int ret; - unsigned long count; - struct page *page = (struct page *)(data); - struct zcache_client *cli = pool->client; - uint16_t client_id = get_client_id_from_client(cli); - - ret = zcache_compress(page, &cdata, &clen); - if (ret == 0) - goto out; - if (clen == 0 || clen > zbud_max_buddy_size()) { - zcache_compress_poor++; - goto out; - } - pampd = (void *)zbud_create(client_id, pool->pool_id, oid, - index, page, cdata, clen); - if (pampd != NULL) { - count = atomic_inc_return(&zcache_curr_eph_pampd_count); - if (count > zcache_curr_eph_pampd_count_max) - zcache_curr_eph_pampd_count_max = count; - } -out: - return pampd; -} - -/* - * fill the pageframe corresponding to the struct page with the data - * from the passed pampd - */ -static int zcache_pampd_get_data(char *data, size_t *bufsize, bool raw, - void *pampd, struct tmem_pool *pool, - struct tmem_oid *oid, uint32_t index) -{ - BUG(); - return 0; -} - -/* - * fill the pageframe corresponding to the struct page with the data - * from the passed pampd - */ -static int zcache_pampd_get_data_and_free(char *data, size_t *bufsize, bool raw, - void *pampd, struct tmem_pool *pool, - struct tmem_oid *oid, uint32_t index) -{ - int ret = 0; - - zbud_decompress((struct page *)(data), pampd); - zbud_free_and_delist((struct zbud_hdr *)pampd); - atomic_dec(&zcache_curr_eph_pampd_count); - return ret; -} - -/* - * free the pampd and remove it from any zcache lists - * pampd must no longer be pointed to from any tmem data structures! - */ -static void zcache_pampd_free(void *pampd, struct tmem_pool *pool, - struct tmem_oid *oid, uint32_t index) -{ - zbud_free_and_delist((struct zbud_hdr *)pampd); - atomic_dec(&zcache_curr_eph_pampd_count); - BUG_ON(atomic_read(&zcache_curr_eph_pampd_count) < 0); -} - -static void zcache_pampd_free_obj(struct tmem_pool *pool, struct tmem_obj *obj) -{ -} - -static void zcache_pampd_new_obj(struct tmem_obj *obj) -{ -} - -static int zcache_pampd_replace_in_obj(void *pampd, struct tmem_obj *obj) -{ - return -1; -} - -static bool zcache_pampd_is_remote(void *pampd) -{ - return 0; -} - -static struct tmem_pamops zcache_pamops = { - .create = zcache_pampd_create, - .get_data = zcache_pampd_get_data, - .get_data_and_free = zcache_pampd_get_data_and_free, - .free = zcache_pampd_free, - .free_obj = zcache_pampd_free_obj, - .new_obj = zcache_pampd_new_obj, - .replace_in_obj = zcache_pampd_replace_in_obj, - .is_remote = zcache_pampd_is_remote, -}; - -/* - * zcache compression/decompression and related per-cpu stuff - */ - -#define LZO_WORKMEM_BYTES LZO1X_1_MEM_COMPRESS -#define LZO_DSTMEM_PAGE_ORDER 1 -static DEFINE_PER_CPU(unsigned char *, zcache_workmem); -static DEFINE_PER_CPU(unsigned char *, zcache_dstmem); - -static int zcache_compress(struct page *from, void **out_va, size_t *out_len) -{ - int ret = 0; - unsigned char *dmem = __get_cpu_var(zcache_dstmem); - unsigned char *wmem = __get_cpu_var(zcache_workmem); - char *from_va; - - BUG_ON(!irqs_disabled()); - if (unlikely(dmem == NULL || wmem == NULL)) - goto out; /* no buffer, so can't compress */ - from_va = kmap_atomic(from); - mb(); - ret = lzo1x_1_compress(from_va, PAGE_SIZE, dmem, out_len, wmem); - BUG_ON(ret != LZO_E_OK); - *out_va = dmem; - kunmap_atomic(from_va); - ret = 1; -out: - return ret; -} - -#ifdef CONFIG_SYSFS -#define ZCACHE_SYSFS_RO(_name) \ - static ssize_t zcache_##_name##_show(struct kobject *kobj, \ - struct kobj_attribute *attr, char *buf) \ - { \ - return sprintf(buf, "%lu\n", zcache_##_name); \ - } \ - static struct kobj_attribute zcache_##_name##_attr = { \ - .attr = { .name = __stringify(_name), .mode = 0444 }, \ - .show = zcache_##_name##_show, \ - } - -#define ZCACHE_SYSFS_RO_ATOMIC(_name) \ - static ssize_t zcache_##_name##_show(struct kobject *kobj, \ - struct kobj_attribute *attr, char *buf) \ - { \ - return sprintf(buf, "%d\n", atomic_read(&zcache_##_name)); \ - } \ - static struct kobj_attribute zcache_##_name##_attr = { \ - .attr = { .name = __stringify(_name), .mode = 0444 }, \ - .show = zcache_##_name##_show, \ - } - -#define ZCACHE_SYSFS_RO_CUSTOM(_name, _func) \ - static ssize_t zcache_##_name##_show(struct kobject *kobj, \ - struct kobj_attribute *attr, char *buf) \ - { \ - return _func(buf); \ - } \ - static struct kobj_attribute zcache_##_name##_attr = { \ - .attr = { .name = __stringify(_name), .mode = 0444 }, \ - .show = zcache_##_name##_show, \ - } - -ZCACHE_SYSFS_RO(curr_obj_count_max); -ZCACHE_SYSFS_RO(curr_objnode_count_max); -ZCACHE_SYSFS_RO(flush_total); -ZCACHE_SYSFS_RO(flush_found); -ZCACHE_SYSFS_RO(flobj_total); -ZCACHE_SYSFS_RO(flobj_found); -ZCACHE_SYSFS_RO(failed_eph_puts); -ZCACHE_SYSFS_RO(zbud_curr_zbytes); -ZCACHE_SYSFS_RO(zbud_cumul_zpages); -ZCACHE_SYSFS_RO(zbud_cumul_zbytes); -ZCACHE_SYSFS_RO(zbud_buddied_count); -ZCACHE_SYSFS_RO(failed_get_free_pages); -ZCACHE_SYSFS_RO(failed_alloc); -ZCACHE_SYSFS_RO(put_to_flush); -ZCACHE_SYSFS_RO(aborted_preload); -ZCACHE_SYSFS_RO(aborted_shrink); -ZCACHE_SYSFS_RO(compress_poor); -ZCACHE_SYSFS_RO(mean_compress_poor); -ZCACHE_SYSFS_RO(qc_allocated); -ZCACHE_SYSFS_RO(qc_freed); -ZCACHE_SYSFS_RO(qc_used); -ZCACHE_SYSFS_RO(qc_max_used); -ZCACHE_SYSFS_RO_ATOMIC(zbud_curr_raw_pages); -ZCACHE_SYSFS_RO_ATOMIC(zbud_curr_zpages); -ZCACHE_SYSFS_RO_ATOMIC(curr_obj_count); -ZCACHE_SYSFS_RO_ATOMIC(curr_objnode_count); -ZCACHE_SYSFS_RO_CUSTOM(zbud_unbuddied_list_counts, - zbud_show_unbuddied_list_counts); -ZCACHE_SYSFS_RO_CUSTOM(zbud_cumul_chunk_counts, - zbud_show_cumul_chunk_counts); - -static struct attribute *qcache_attrs[] = { - &zcache_curr_obj_count_attr.attr, - &zcache_curr_obj_count_max_attr.attr, - &zcache_curr_objnode_count_attr.attr, - &zcache_curr_objnode_count_max_attr.attr, - &zcache_flush_total_attr.attr, - &zcache_flobj_total_attr.attr, - &zcache_flush_found_attr.attr, - &zcache_flobj_found_attr.attr, - &zcache_failed_eph_puts_attr.attr, - &zcache_compress_poor_attr.attr, - &zcache_mean_compress_poor_attr.attr, - &zcache_zbud_curr_raw_pages_attr.attr, - &zcache_zbud_curr_zpages_attr.attr, - &zcache_zbud_curr_zbytes_attr.attr, - &zcache_zbud_cumul_zpages_attr.attr, - &zcache_zbud_cumul_zbytes_attr.attr, - &zcache_zbud_buddied_count_attr.attr, - &zcache_failed_get_free_pages_attr.attr, - &zcache_failed_alloc_attr.attr, - &zcache_put_to_flush_attr.attr, - &zcache_aborted_preload_attr.attr, - &zcache_aborted_shrink_attr.attr, - &zcache_zbud_unbuddied_list_counts_attr.attr, - &zcache_zbud_cumul_chunk_counts_attr.attr, - &zcache_qc_allocated_attr.attr, - &zcache_qc_freed_attr.attr, - &zcache_qc_used_attr.attr, - &zcache_qc_max_used_attr.attr, - NULL, -}; - -static struct attribute_group qcache_attr_group = { - .attrs = qcache_attrs, - .name = "qcache", -}; - -#endif /* CONFIG_SYSFS */ - -/* - * zcache shims between cleancache ops and tmem - */ - -static int zcache_put_page(int cli_id, int pool_id, struct tmem_oid *oidp, - uint32_t index, struct page *page) -{ - struct tmem_pool *pool; - int ret = -1; - - BUG_ON(!irqs_disabled()); - pool = zcache_get_pool_by_id(cli_id, pool_id); - if (unlikely(pool == NULL)) - goto out; - if (!zcache_freeze && zcache_do_preload(pool) == 0) { - /* preload does preempt_disable on success */ - ret = tmem_put(pool, oidp, index, (char *)(page), - PAGE_SIZE, 0, is_ephemeral(pool)); - if (ret < 0) { - zcache_failed_eph_puts++; - } - zcache_put_pool(pool); - preempt_enable_no_resched(); - } else { - zcache_put_to_flush++; - if (atomic_read(&pool->obj_count) > 0) - /* the put fails whether the flush succeeds or not */ - (void)tmem_flush_page(pool, oidp, index); - zcache_put_pool(pool); - } -out: - return ret; -} - -static int zcache_get_page(int cli_id, int pool_id, struct tmem_oid *oidp, - uint32_t index, struct page *page) -{ - struct tmem_pool *pool; - int ret = -1; - unsigned long flags; - size_t size = PAGE_SIZE; - - local_irq_save(flags); - pool = zcache_get_pool_by_id(cli_id, pool_id); - if (likely(pool != NULL)) { - if (atomic_read(&pool->obj_count) > 0) - ret = tmem_get(pool, oidp, index, (char *)(page), - &size, 0, is_ephemeral(pool)); - zcache_put_pool(pool); - } - local_irq_restore(flags); - return ret; -} - -static int zcache_flush_page(int cli_id, int pool_id, - struct tmem_oid *oidp, uint32_t index) -{ - struct tmem_pool *pool; - int ret = -1; - unsigned long flags; - - local_irq_save(flags); - zcache_flush_total++; - pool = zcache_get_pool_by_id(cli_id, pool_id); - if (likely(pool != NULL)) { - if (atomic_read(&pool->obj_count) > 0) - ret = tmem_flush_page(pool, oidp, index); - zcache_put_pool(pool); - } - if (ret >= 0) - zcache_flush_found++; - local_irq_restore(flags); - return ret; -} - -static int zcache_flush_object(int cli_id, int pool_id, - struct tmem_oid *oidp) -{ - struct tmem_pool *pool; - int ret = -1; - unsigned long flags; - - local_irq_save(flags); - zcache_flobj_total++; - pool = zcache_get_pool_by_id(cli_id, pool_id); - if (likely(pool != NULL)) { - if (atomic_read(&pool->obj_count) > 0) - ret = tmem_flush_object(pool, oidp); - zcache_put_pool(pool); - } - if (ret >= 0) - zcache_flobj_found++; - local_irq_restore(flags); - return ret; -} - -static int zcache_destroy_pool(int cli_id, int pool_id) -{ - struct tmem_pool *pool = NULL; - struct zcache_client *cli = NULL; - int ret = -1; - - if (pool_id < 0) - goto out; - if (cli_id == LOCAL_CLIENT) - cli = &zcache_host; - else if ((unsigned int)cli_id < MAX_CLIENTS) - cli = &zcache_clients[cli_id]; - if (cli == NULL) - goto out; - atomic_inc(&cli->refcount); - pool = cli->tmem_pools[pool_id]; - if (pool == NULL) - goto out; - cli->tmem_pools[pool_id] = NULL; - /* wait for pool activity on other cpus to quiesce */ - while (atomic_read(&pool->refcount) != 0) - ; - atomic_dec(&cli->refcount); - local_bh_disable(); - ret = tmem_destroy_pool(pool); - local_bh_enable(); - kfree(pool); - pr_info("qcache: destroyed pool id=%d, cli_id=%d\n", - pool_id, cli_id); -out: - return ret; -} - -static int zcache_new_pool(uint16_t cli_id, uint32_t flags) -{ - int poolid = -1; - struct tmem_pool *pool; - struct zcache_client *cli = NULL; - - if (cli_id == LOCAL_CLIENT) - cli = &zcache_host; - else if ((unsigned int)cli_id < MAX_CLIENTS) - cli = &zcache_clients[cli_id]; - if (cli == NULL) - goto out; - atomic_inc(&cli->refcount); - pool = kmalloc(sizeof(struct tmem_pool), GFP_KERNEL); - if (pool == NULL) { - pr_info("qcache: pool creation failed: out of memory\n"); - goto out; - } - - for (poolid = 0; poolid < MAX_POOLS_PER_CLIENT; poolid++) - if (cli->tmem_pools[poolid] == NULL) - break; - if (poolid >= MAX_POOLS_PER_CLIENT) { - pr_info("qcache: pool creation failed: max exceeded\n"); - kfree(pool); - poolid = -1; - goto out; - } - atomic_set(&pool->refcount, 0); - pool->client = cli; - pool->pool_id = poolid; - tmem_new_pool(pool, flags); - cli->tmem_pools[poolid] = pool; - pr_info("qcache: created %s tmem pool, id=%d, client=%d\n", - flags & TMEM_POOL_PERSIST ? "persistent" : "ephemeral", - poolid, cli_id); -out: - if (cli != NULL) - atomic_dec(&cli->refcount); - return poolid; -} - -/********** - * Two kernel functionalities currently can be layered on top of tmem. - * These are "cleancache" which is used as a second-chance cache for clean - * page cache pages; and "frontswap" which is used for swap pages - * to avoid writes to disk. A generic "shim" is provided here for each - * to translate in-kernel semantics to zcache semantics. - */ - -static void zcache_cleancache_put_page(int pool_id, - struct cleancache_filekey key, - pgoff_t index, struct page *page) -{ - u32 ind = (u32) index; - struct tmem_oid oid = *(struct tmem_oid *)&key; - - if (likely(ind == index)) - (void)zcache_put_page(LOCAL_CLIENT, pool_id, &oid, index, page); -} - -static int zcache_cleancache_get_page(int pool_id, - struct cleancache_filekey key, - pgoff_t index, struct page *page) -{ - u32 ind = (u32) index; - struct tmem_oid oid = *(struct tmem_oid *)&key; - int ret = -1; - - if (likely(ind == index)) - ret = zcache_get_page(LOCAL_CLIENT, pool_id, &oid, index, page); - return ret; -} - -static void zcache_cleancache_flush_page(int pool_id, - struct cleancache_filekey key, - pgoff_t index) -{ - u32 ind = (u32) index; - struct tmem_oid oid = *(struct tmem_oid *)&key; - - if (likely(ind == index)) - (void)zcache_flush_page(LOCAL_CLIENT, pool_id, &oid, ind); -} - -static void zcache_cleancache_flush_inode(int pool_id, - struct cleancache_filekey key) -{ - struct tmem_oid oid = *(struct tmem_oid *)&key; - - (void)zcache_flush_object(LOCAL_CLIENT, pool_id, &oid); -} - -static void zcache_cleancache_flush_fs(int pool_id) -{ - if (pool_id >= 0) - (void)zcache_destroy_pool(LOCAL_CLIENT, pool_id); -} - -static int zcache_cleancache_init_fs(size_t pagesize) -{ - BUG_ON(sizeof(struct cleancache_filekey) != - sizeof(struct tmem_oid)); - BUG_ON(pagesize != PAGE_SIZE); - return zcache_new_pool(LOCAL_CLIENT, 0); -} - -static int zcache_cleancache_init_shared_fs(char *uuid, size_t pagesize) -{ - /* shared pools are unsupported and map to private */ - BUG_ON(sizeof(struct cleancache_filekey) != - sizeof(struct tmem_oid)); - BUG_ON(pagesize != PAGE_SIZE); - return zcache_new_pool(LOCAL_CLIENT, 0); -} - -static struct cleancache_ops zcache_cleancache_ops = { - .put_page = zcache_cleancache_put_page, - .get_page = zcache_cleancache_get_page, - .invalidate_page = zcache_cleancache_flush_page, - .invalidate_inode = zcache_cleancache_flush_inode, - .invalidate_fs = zcache_cleancache_flush_fs, - .init_shared_fs = zcache_cleancache_init_shared_fs, - .init_fs = zcache_cleancache_init_fs -}; - -struct cleancache_ops zcache_cleancache_register_ops(void) -{ - struct cleancache_ops old_ops = - cleancache_register_ops(&zcache_cleancache_ops); - - return old_ops; -} - -static int __init qcache_init(void) -{ - int ret = 0; - struct qcache_info *qc = &qcache_info; - struct fmem_data *fdp; - int bitmap_size; - unsigned int cpu; - struct cleancache_ops old_ops; - -#ifdef CONFIG_SYSFS - ret = sysfs_create_group(mm_kobj, &qcache_attr_group); - if (ret) { - pr_err("qcache: can't create sysfs\n"); - goto out; - } -#endif /* CONFIG_SYSFS */ - - fdp = fmem_get_info(); - qc->addr = fdp->virt; - qc->pages = fdp->size >> PAGE_SHIFT; - if (!qc->pages) - goto out; - - tmem_register_hostops(&zcache_hostops); - tmem_register_pamops(&zcache_pamops); - for_each_online_cpu(cpu) { - per_cpu(zcache_dstmem, cpu) = (void *)__get_free_pages( - GFP_KERNEL | __GFP_REPEAT, - LZO_DSTMEM_PAGE_ORDER), - per_cpu(zcache_workmem, cpu) = - kzalloc(LZO1X_MEM_COMPRESS, - GFP_KERNEL | __GFP_REPEAT); - } - zcache_objnode_cache = kmem_cache_create("zcache_objnode", - sizeof(struct tmem_objnode), 0, 0, NULL); - zcache_obj_cache = kmem_cache_create("zcache_obj", - sizeof(struct tmem_obj), 0, 0, NULL); - ret = zcache_new_client(LOCAL_CLIENT); - if (ret) { - pr_err("qcache: can't create client\n"); - goto out; - } - - zbud_init(); - old_ops = zcache_cleancache_register_ops(); - pr_info("qcache: cleancache enabled using kernel " - "transcendent memory and compression buddies\n"); - if (old_ops.init_fs != NULL) - pr_warning("qcache: cleancache_ops overridden"); - - - bitmap_size = BITS_TO_LONGS(qc->pages) * sizeof(long); - - qc->bitmap = kzalloc(bitmap_size, GFP_KERNEL); - if (!qc->bitmap) { - pr_info("can't allocate qcache bitmap!\n"); - ret = -ENOMEM; - goto out; - } - spin_lock_init(&qc->lock); - - fmem_set_state(FMEM_T_STATE); - -out: - return ret; -} - -module_init(qcache_init) diff --git a/drivers/staging/qcache/tmem.c b/drivers/staging/qcache/tmem.c deleted file mode 100644 index 40f22460ba8..00000000000 --- a/drivers/staging/qcache/tmem.c +++ /dev/null @@ -1,833 +0,0 @@ -/* - * In-kernel transcendent memory (generic implementation) - * - * Copyright (c) 2009-2011, Dan Magenheimer, Oracle Corp. - * Copyright (c) 2011, The Linux Foundation. All rights reserved. - * - * The primary purpose of Transcedent Memory ("tmem") is to map object-oriented - * "handles" (triples containing a pool id, and object id, and an index), to - * pages in a page-accessible memory (PAM). Tmem references the PAM pages via - * an abstract "pampd" (PAM page-descriptor), which can be operated on by a - * set of functions (pamops). Each pampd contains some representation of - * PAGE_SIZE bytes worth of data. Tmem must support potentially millions of - * pages and must be able to insert, find, and delete these pages at a - * potential frequency of thousands per second concurrently across many CPUs, - * (and, if used with KVM, across many vcpus across many guests). - * Tmem is tracked with a hierarchy of data structures, organized by - * the elements in a handle-tuple: pool_id, object_id, and page index. - * One or more "clients" (e.g. guests) each provide one or more tmem_pools. - * Each pool, contains a hash table of rb_trees of tmem_objs. Each - * tmem_obj contains a radix-tree-like tree of pointers, with intermediate - * nodes called tmem_objnodes. Each leaf pointer in this tree points to - * a pampd, which is accessible only through a small set of callbacks - * registered by the PAM implementation (see tmem_register_pamops). Tmem - * does all memory allocation via a set of callbacks registered by the tmem - * host implementation (e.g. see tmem_register_hostops). - */ - -#include -#include -#include - -#include "tmem.h" - -/* data structure sentinels used for debugging... see tmem.h */ -#define POOL_SENTINEL 0x87658765 -#define OBJ_SENTINEL 0x12345678 -#define OBJNODE_SENTINEL 0xfedcba09 - -static bool tmem_enabled; - -static void lock_tmem_state(void) -{ - lock_fmem_state(); -} - -static void unlock_tmem_state(void) -{ - unlock_fmem_state(); -} - -/* - * A tmem host implementation must use this function to register callbacks - * for memory allocation. - */ -static struct tmem_hostops tmem_hostops; - -static void tmem_objnode_tree_init(void); - -void tmem_register_hostops(struct tmem_hostops *m) -{ - tmem_objnode_tree_init(); - tmem_hostops = *m; -} - -/* - * A tmem host implementation must use this function to register - * callbacks for a page-accessible memory (PAM) implementation - */ -static struct tmem_pamops tmem_pamops; - -void tmem_register_pamops(struct tmem_pamops *m) -{ - tmem_pamops = *m; -} - -/* - * Oid's are potentially very sparse and tmem_objs may have an indeterminately - * short life, being added and deleted at a relatively high frequency. - * So an rb_tree is an ideal data structure to manage tmem_objs. But because - * of the potentially huge number of tmem_objs, each pool manages a hashtable - * of rb_trees to reduce search, insert, delete, and rebalancing time. - * Each hashbucket also has a lock to manage concurrent access. - * - * The following routines manage tmem_objs. When any tmem_obj is accessed, - * the hashbucket lock must be held. - */ - -/* searches for object==oid in pool, returns locked object if found */ -static struct tmem_obj *tmem_obj_find(struct tmem_hashbucket *hb, - struct tmem_oid *oidp) -{ - struct rb_node *rbnode; - struct tmem_obj *obj; - - rbnode = hb->obj_rb_root.rb_node; - while (rbnode) { - BUG_ON(RB_EMPTY_NODE(rbnode)); - obj = rb_entry(rbnode, struct tmem_obj, rb_tree_node); - switch (tmem_oid_compare(oidp, &obj->oid)) { - case 0: /* equal */ - goto out; - case -1: - rbnode = rbnode->rb_left; - break; - case 1: - rbnode = rbnode->rb_right; - break; - } - } - obj = NULL; -out: - return obj; -} - -static void tmem_pampd_destroy_all_in_obj(struct tmem_obj *); - -/* free an object that has no more pampds in it */ -static void tmem_obj_free(struct tmem_obj *obj, struct tmem_hashbucket *hb) -{ - struct tmem_pool *pool; - - BUG_ON(obj == NULL); - ASSERT_SENTINEL(obj, OBJ); - BUG_ON(obj->pampd_count > 0); - pool = obj->pool; - BUG_ON(pool == NULL); - if (obj->objnode_tree_root != NULL) /* may be "stump" with no leaves */ - tmem_pampd_destroy_all_in_obj(obj); - BUG_ON(obj->objnode_tree_root != NULL); - BUG_ON((long)obj->objnode_count != 0); - atomic_dec(&pool->obj_count); - BUG_ON(atomic_read(&pool->obj_count) < 0); - INVERT_SENTINEL(obj, OBJ); - obj->pool = NULL; - tmem_oid_set_invalid(&obj->oid); - rb_erase(&obj->rb_tree_node, &hb->obj_rb_root); -} - -/* - * initialize, and insert an tmem_object_root (called only if find failed) - */ -static void tmem_obj_init(struct tmem_obj *obj, struct tmem_hashbucket *hb, - struct tmem_pool *pool, - struct tmem_oid *oidp) -{ - struct rb_root *root = &hb->obj_rb_root; - struct rb_node **new = &(root->rb_node), *parent = NULL; - struct tmem_obj *this; - - BUG_ON(pool == NULL); - atomic_inc(&pool->obj_count); - obj->objnode_tree_height = 0; - obj->objnode_tree_root = NULL; - obj->pool = pool; - obj->oid = *oidp; - obj->objnode_count = 0; - obj->pampd_count = 0; - (*tmem_pamops.new_obj)(obj); - SET_SENTINEL(obj, OBJ); - while (*new) { - BUG_ON(RB_EMPTY_NODE(*new)); - this = rb_entry(*new, struct tmem_obj, rb_tree_node); - parent = *new; - switch (tmem_oid_compare(oidp, &this->oid)) { - case 0: - BUG(); /* already present; should never happen! */ - break; - case -1: - new = &(*new)->rb_left; - break; - case 1: - new = &(*new)->rb_right; - break; - } - } - rb_link_node(&obj->rb_tree_node, parent, new); - rb_insert_color(&obj->rb_tree_node, root); -} - -/* - * Tmem is managed as a set of tmem_pools with certain attributes, such as - * "ephemeral" vs "persistent". These attributes apply to all tmem_objs - * and all pampds that belong to a tmem_pool. A tmem_pool is created - * or deleted relatively rarely (for example, when a filesystem is - * mounted or unmounted. - */ - -/* flush all data from a pool and, optionally, free it */ -static void tmem_pool_flush(struct tmem_pool *pool, bool destroy) -{ - struct rb_node *rbnode; - struct tmem_obj *obj; - struct tmem_hashbucket *hb = &pool->hashbucket[0]; - int i; - - BUG_ON(pool == NULL); - for (i = 0; i < TMEM_HASH_BUCKETS; i++, hb++) { - spin_lock(&hb->lock); - rbnode = rb_first(&hb->obj_rb_root); - while (rbnode != NULL) { - obj = rb_entry(rbnode, struct tmem_obj, rb_tree_node); - rbnode = rb_next(rbnode); - tmem_pampd_destroy_all_in_obj(obj); - tmem_obj_free(obj, hb); - (*tmem_hostops.obj_free)(obj, pool); - } - spin_unlock(&hb->lock); - } - if (destroy) - list_del(&pool->pool_list); -} - -/* - * A tmem_obj contains a radix-tree-like tree in which the intermediate - * nodes are called tmem_objnodes. (The kernel lib/radix-tree.c implementation - * is very specialized and tuned for specific uses and is not particularly - * suited for use from this code, though some code from the core algorithms has - * been reused, thus the copyright notices below). Each tmem_objnode contains - * a set of pointers which point to either a set of intermediate tmem_objnodes - * or a set of of pampds. - * - * Portions Copyright (C) 2001 Momchil Velikov - * Portions Copyright (C) 2001 Christoph Hellwig - * Portions Copyright (C) 2005 SGI, Christoph Lameter - */ - -struct tmem_objnode_tree_path { - struct tmem_objnode *objnode; - int offset; -}; - -/* objnode height_to_maxindex translation */ -static unsigned long tmem_objnode_tree_h2max[OBJNODE_TREE_MAX_PATH + 1]; - -static void tmem_objnode_tree_init(void) -{ - unsigned int ht, tmp; - - for (ht = 0; ht < ARRAY_SIZE(tmem_objnode_tree_h2max); ht++) { - tmp = ht * OBJNODE_TREE_MAP_SHIFT; - if (tmp >= OBJNODE_TREE_INDEX_BITS) - tmem_objnode_tree_h2max[ht] = ~0UL; - else - tmem_objnode_tree_h2max[ht] = - (~0UL >> (OBJNODE_TREE_INDEX_BITS - tmp - 1)) >> 1; - } -} - -static struct tmem_objnode *tmem_objnode_alloc(struct tmem_obj *obj) -{ - struct tmem_objnode *objnode; - - ASSERT_SENTINEL(obj, OBJ); - BUG_ON(obj->pool == NULL); - ASSERT_SENTINEL(obj->pool, POOL); - objnode = (*tmem_hostops.objnode_alloc)(obj->pool); - if (unlikely(objnode == NULL)) - goto out; - objnode->obj = obj; - SET_SENTINEL(objnode, OBJNODE); - memset(&objnode->slots, 0, sizeof(objnode->slots)); - objnode->slots_in_use = 0; - obj->objnode_count++; -out: - return objnode; -} - -static void tmem_objnode_free(struct tmem_objnode *objnode) -{ - struct tmem_pool *pool; - int i; - - BUG_ON(objnode == NULL); - for (i = 0; i < OBJNODE_TREE_MAP_SIZE; i++) - BUG_ON(objnode->slots[i] != NULL); - ASSERT_SENTINEL(objnode, OBJNODE); - INVERT_SENTINEL(objnode, OBJNODE); - BUG_ON(objnode->obj == NULL); - ASSERT_SENTINEL(objnode->obj, OBJ); - pool = objnode->obj->pool; - BUG_ON(pool == NULL); - ASSERT_SENTINEL(pool, POOL); - objnode->obj->objnode_count--; - objnode->obj = NULL; - (*tmem_hostops.objnode_free)(objnode, pool); -} - -/* - * lookup index in object and return associated pampd (or NULL if not found) - */ -static void **__tmem_pampd_lookup_in_obj(struct tmem_obj *obj, uint32_t index) -{ - unsigned int height, shift; - struct tmem_objnode **slot = NULL; - - BUG_ON(obj == NULL); - ASSERT_SENTINEL(obj, OBJ); - BUG_ON(obj->pool == NULL); - ASSERT_SENTINEL(obj->pool, POOL); - - height = obj->objnode_tree_height; - if (index > tmem_objnode_tree_h2max[obj->objnode_tree_height]) - goto out; - if (height == 0 && obj->objnode_tree_root) { - slot = &obj->objnode_tree_root; - goto out; - } - shift = (height-1) * OBJNODE_TREE_MAP_SHIFT; - slot = &obj->objnode_tree_root; - while (height > 0) { - if (*slot == NULL) - goto out; - slot = (struct tmem_objnode **) - ((*slot)->slots + - ((index >> shift) & OBJNODE_TREE_MAP_MASK)); - shift -= OBJNODE_TREE_MAP_SHIFT; - height--; - } -out: - return slot != NULL ? (void **)slot : NULL; -} - -static void *tmem_pampd_lookup_in_obj(struct tmem_obj *obj, uint32_t index) -{ - struct tmem_objnode **slot; - - slot = (struct tmem_objnode **)__tmem_pampd_lookup_in_obj(obj, index); - return slot != NULL ? *slot : NULL; -} - -static void *tmem_pampd_replace_in_obj(struct tmem_obj *obj, uint32_t index, - void *new_pampd) -{ - struct tmem_objnode **slot; - void *ret = NULL; - - slot = (struct tmem_objnode **)__tmem_pampd_lookup_in_obj(obj, index); - if ((slot != NULL) && (*slot != NULL)) { - void *old_pampd = *(void **)slot; - *(void **)slot = new_pampd; - (*tmem_pamops.free)(old_pampd, obj->pool, NULL, 0); - ret = new_pampd; - } - return ret; -} - -static int tmem_pampd_add_to_obj(struct tmem_obj *obj, uint32_t index, - void *pampd) -{ - int ret = 0; - struct tmem_objnode *objnode = NULL, *newnode, *slot; - unsigned int height, shift; - int offset = 0; - - /* if necessary, extend the tree to be higher */ - if (index > tmem_objnode_tree_h2max[obj->objnode_tree_height]) { - height = obj->objnode_tree_height + 1; - if (index > tmem_objnode_tree_h2max[height]) - while (index > tmem_objnode_tree_h2max[height]) - height++; - if (obj->objnode_tree_root == NULL) { - obj->objnode_tree_height = height; - goto insert; - } - do { - newnode = tmem_objnode_alloc(obj); - if (!newnode) { - ret = -ENOMEM; - goto out; - } - newnode->slots[0] = obj->objnode_tree_root; - newnode->slots_in_use = 1; - obj->objnode_tree_root = newnode; - obj->objnode_tree_height++; - } while (height > obj->objnode_tree_height); - } -insert: - slot = obj->objnode_tree_root; - height = obj->objnode_tree_height; - shift = (height-1) * OBJNODE_TREE_MAP_SHIFT; - while (height > 0) { - if (slot == NULL) { - /* add a child objnode. */ - slot = tmem_objnode_alloc(obj); - if (!slot) { - ret = -ENOMEM; - goto out; - } - if (objnode) { - - objnode->slots[offset] = slot; - objnode->slots_in_use++; - } else - obj->objnode_tree_root = slot; - } - /* go down a level */ - offset = (index >> shift) & OBJNODE_TREE_MAP_MASK; - objnode = slot; - slot = objnode->slots[offset]; - shift -= OBJNODE_TREE_MAP_SHIFT; - height--; - } - BUG_ON(slot != NULL); - if (objnode) { - objnode->slots_in_use++; - objnode->slots[offset] = pampd; - } else - obj->objnode_tree_root = pampd; - obj->pampd_count++; -out: - return ret; -} - -static void *tmem_pampd_delete_from_obj(struct tmem_obj *obj, uint32_t index) -{ - struct tmem_objnode_tree_path path[OBJNODE_TREE_MAX_PATH + 1]; - struct tmem_objnode_tree_path *pathp = path; - struct tmem_objnode *slot = NULL; - unsigned int height, shift; - int offset; - - BUG_ON(obj == NULL); - ASSERT_SENTINEL(obj, OBJ); - BUG_ON(obj->pool == NULL); - ASSERT_SENTINEL(obj->pool, POOL); - height = obj->objnode_tree_height; - if (index > tmem_objnode_tree_h2max[height]) - goto out; - slot = obj->objnode_tree_root; - if (height == 0 && obj->objnode_tree_root) { - obj->objnode_tree_root = NULL; - goto out; - } - shift = (height - 1) * OBJNODE_TREE_MAP_SHIFT; - pathp->objnode = NULL; - do { - if (slot == NULL) - goto out; - pathp++; - offset = (index >> shift) & OBJNODE_TREE_MAP_MASK; - pathp->offset = offset; - pathp->objnode = slot; - slot = slot->slots[offset]; - shift -= OBJNODE_TREE_MAP_SHIFT; - height--; - } while (height > 0); - if (slot == NULL) - goto out; - while (pathp->objnode) { - pathp->objnode->slots[pathp->offset] = NULL; - pathp->objnode->slots_in_use--; - if (pathp->objnode->slots_in_use) { - if (pathp->objnode == obj->objnode_tree_root) { - while (obj->objnode_tree_height > 0 && - obj->objnode_tree_root->slots_in_use == 1 && - obj->objnode_tree_root->slots[0]) { - struct tmem_objnode *to_free = - obj->objnode_tree_root; - - obj->objnode_tree_root = - to_free->slots[0]; - obj->objnode_tree_height--; - to_free->slots[0] = NULL; - to_free->slots_in_use = 0; - tmem_objnode_free(to_free); - } - } - goto out; - } - tmem_objnode_free(pathp->objnode); /* 0 slots used, free it */ - pathp--; - } - obj->objnode_tree_height = 0; - obj->objnode_tree_root = NULL; - -out: - if (slot != NULL) - obj->pampd_count--; - BUG_ON(obj->pampd_count < 0); - return slot; -} - -/* recursively walk the objnode_tree destroying pampds and objnodes */ -static void tmem_objnode_node_destroy(struct tmem_obj *obj, - struct tmem_objnode *objnode, - unsigned int ht) -{ - int i; - - if (ht == 0) - return; - for (i = 0; i < OBJNODE_TREE_MAP_SIZE; i++) { - if (objnode->slots[i]) { - if (ht == 1) { - obj->pampd_count--; - (*tmem_pamops.free)(objnode->slots[i], - obj->pool, NULL, 0); - objnode->slots[i] = NULL; - continue; - } - tmem_objnode_node_destroy(obj, objnode->slots[i], ht-1); - tmem_objnode_free(objnode->slots[i]); - objnode->slots[i] = NULL; - } - } -} - -static void tmem_pampd_destroy_all_in_obj(struct tmem_obj *obj) -{ - if (obj->objnode_tree_root == NULL) - return; - if (obj->objnode_tree_height == 0) { - obj->pampd_count--; - (*tmem_pamops.free)(obj->objnode_tree_root, obj->pool, NULL, 0); - } else { - tmem_objnode_node_destroy(obj, obj->objnode_tree_root, - obj->objnode_tree_height); - tmem_objnode_free(obj->objnode_tree_root); - obj->objnode_tree_height = 0; - } - obj->objnode_tree_root = NULL; - (*tmem_pamops.free_obj)(obj->pool, obj); -} - -/* - * Tmem is operated on by a set of well-defined actions: - * "put", "get", "flush", "flush_object", "new pool" and "destroy pool". - * (The tmem ABI allows for subpages and exchanges but these operations - * are not included in this implementation.) - * - * These "tmem core" operations are implemented in the following functions. - */ - -/* - * "Put" a page, e.g. copy a page from the kernel into newly allocated - * PAM space (if such space is available). Tmem_put is complicated by - * a corner case: What if a page with matching handle already exists in - * tmem? To guarantee coherency, one of two actions is necessary: Either - * the data for the page must be overwritten, or the page must be - * "flushed" so that the data is not accessible to a subsequent "get". - * Since these "duplicate puts" are relatively rare, this implementation - * always flushes for simplicity. - */ -int tmem_put(struct tmem_pool *pool, struct tmem_oid *oidp, uint32_t index, - char *data, size_t size, bool raw, bool ephemeral) -{ - struct tmem_obj *obj = NULL, *objfound = NULL, *objnew = NULL; - void *pampd = NULL, *pampd_del = NULL; - int ret = -ENOMEM; - struct tmem_hashbucket *hb; - - lock_tmem_state(); - if (!tmem_enabled) - goto disabled; - hb = &pool->hashbucket[tmem_oid_hash(oidp)]; - spin_lock(&hb->lock); - obj = objfound = tmem_obj_find(hb, oidp); - if (obj != NULL) { - pampd = tmem_pampd_lookup_in_obj(objfound, index); - if (pampd != NULL) { - /* if found, is a dup put, flush the old one */ - pampd_del = tmem_pampd_delete_from_obj(obj, index); - BUG_ON(pampd_del != pampd); - (*tmem_pamops.free)(pampd, pool, oidp, index); - if (obj->pampd_count == 0) { - objnew = obj; - objfound = NULL; - } - pampd = NULL; - } - } else { - obj = objnew = (*tmem_hostops.obj_alloc)(pool); - if (unlikely(obj == NULL)) { - ret = -ENOMEM; - goto out; - } - tmem_obj_init(obj, hb, pool, oidp); - } - BUG_ON(obj == NULL); - BUG_ON(((objnew != obj) && (objfound != obj)) || (objnew == objfound)); - pampd = (*tmem_pamops.create)(data, size, raw, ephemeral, - obj->pool, &obj->oid, index); - if (unlikely(pampd == NULL)) - goto free; - ret = tmem_pampd_add_to_obj(obj, index, pampd); - if (unlikely(ret == -ENOMEM)) - /* may have partially built objnode tree ("stump") */ - goto delete_and_free; - goto out; - -delete_and_free: - (void)tmem_pampd_delete_from_obj(obj, index); -free: - if (pampd) - (*tmem_pamops.free)(pampd, pool, NULL, 0); - if (objnew) { - tmem_obj_free(objnew, hb); - (*tmem_hostops.obj_free)(objnew, pool); - } -out: - spin_unlock(&hb->lock); -disabled: - unlock_tmem_state(); - return ret; -} - -/* - * "Get" a page, e.g. if one can be found, copy the tmem page with the - * matching handle from PAM space to the kernel. By tmem definition, - * when a "get" is successful on an ephemeral page, the page is "flushed", - * and when a "get" is successful on a persistent page, the page is retained - * in tmem. Note that to preserve - * coherency, "get" can never be skipped if tmem contains the data. - * That is, if a get is done with a certain handle and fails, any - * subsequent "get" must also fail (unless of course there is a - * "put" done with the same handle). - - */ -int tmem_get(struct tmem_pool *pool, struct tmem_oid *oidp, uint32_t index, - char *data, size_t *size, bool raw, int get_and_free) -{ - struct tmem_obj *obj; - void *pampd; - bool ephemeral = is_ephemeral(pool); - int ret = -1; - struct tmem_hashbucket *hb; - bool free = (get_and_free == 1) || ((get_and_free == 0) && ephemeral); - bool lock_held = false; - - lock_tmem_state(); - if (!tmem_enabled) - goto disabled; - hb = &pool->hashbucket[tmem_oid_hash(oidp)]; - spin_lock(&hb->lock); - lock_held = true; - obj = tmem_obj_find(hb, oidp); - if (obj == NULL) - goto out; - if (free) - pampd = tmem_pampd_delete_from_obj(obj, index); - else - pampd = tmem_pampd_lookup_in_obj(obj, index); - if (pampd == NULL) - goto out; - if (free) { - if (obj->pampd_count == 0) { - tmem_obj_free(obj, hb); - (*tmem_hostops.obj_free)(obj, pool); - obj = NULL; - } - } - if (tmem_pamops.is_remote(pampd)) { - lock_held = false; - spin_unlock(&hb->lock); - } - if (free) - ret = (*tmem_pamops.get_data_and_free)( - data, size, raw, pampd, pool, oidp, index); - else - ret = (*tmem_pamops.get_data)( - data, size, raw, pampd, pool, oidp, index); - if (ret < 0) - goto out; - ret = 0; -out: - if (lock_held) - spin_unlock(&hb->lock); -disabled: - unlock_tmem_state(); - return ret; -} - -/* - * If a page in tmem matches the handle, "flush" this page from tmem such - * that any subsequent "get" does not succeed (unless, of course, there - * was another "put" with the same handle). - */ -int tmem_flush_page(struct tmem_pool *pool, - struct tmem_oid *oidp, uint32_t index) -{ - struct tmem_obj *obj; - void *pampd; - int ret = -1; - struct tmem_hashbucket *hb; - - hb = &pool->hashbucket[tmem_oid_hash(oidp)]; - spin_lock(&hb->lock); - obj = tmem_obj_find(hb, oidp); - if (obj == NULL) - goto out; - pampd = tmem_pampd_delete_from_obj(obj, index); - if (pampd == NULL) - goto out; - (*tmem_pamops.free)(pampd, pool, oidp, index); - if (obj->pampd_count == 0) { - tmem_obj_free(obj, hb); - (*tmem_hostops.obj_free)(obj, pool); - } - ret = 0; - -out: - spin_unlock(&hb->lock); - return ret; -} - -/* - * If a page in tmem matches the handle, replace the page so that any - * subsequent "get" gets the new page. Returns 0 if - * there was a page to replace, else returns -1. - */ -int tmem_replace(struct tmem_pool *pool, struct tmem_oid *oidp, - uint32_t index, void *new_pampd) -{ - struct tmem_obj *obj; - int ret = -1; - struct tmem_hashbucket *hb; - - lock_tmem_state(); - if (!tmem_enabled) - goto disabled; - hb = &pool->hashbucket[tmem_oid_hash(oidp)]; - spin_lock(&hb->lock); - obj = tmem_obj_find(hb, oidp); - if (obj == NULL) - goto out; - new_pampd = tmem_pampd_replace_in_obj(obj, index, new_pampd); - ret = (*tmem_pamops.replace_in_obj)(new_pampd, obj); -out: - spin_unlock(&hb->lock); -disabled: - unlock_tmem_state(); - return ret; -} - -/* - * "Flush" all pages in tmem matching this oid. - */ -int tmem_flush_object(struct tmem_pool *pool, struct tmem_oid *oidp) -{ - struct tmem_obj *obj; - struct tmem_hashbucket *hb; - int ret = -1; - - hb = &pool->hashbucket[tmem_oid_hash(oidp)]; - spin_lock(&hb->lock); - obj = tmem_obj_find(hb, oidp); - if (obj == NULL) - goto out; - tmem_pampd_destroy_all_in_obj(obj); - tmem_obj_free(obj, hb); - (*tmem_hostops.obj_free)(obj, pool); - ret = 0; - -out: - spin_unlock(&hb->lock); - return ret; -} - -/* - * "Flush" all pages (and tmem_objs) from this tmem_pool and disable - * all subsequent access to this tmem_pool. - */ -int tmem_destroy_pool(struct tmem_pool *pool) -{ - int ret = -1; - - if (pool == NULL) - goto out; - tmem_pool_flush(pool, 1); - ret = 0; -out: - return ret; -} - -int tmem_flush_pool(struct tmem_pool *pool) -{ - int ret = -1; - - if (pool == NULL) - goto out; - tmem_pool_flush(pool, 0); - ret = 0; -out: - return ret; -} - -static LIST_HEAD(tmem_global_pool_list); - -/* - * Create a new tmem_pool with the provided flag and return - * a pool id provided by the tmem host implementation. - */ -void tmem_new_pool(struct tmem_pool *pool, uint32_t flags) -{ - int persistent = flags & TMEM_POOL_PERSIST; - int shared = flags & TMEM_POOL_SHARED; - struct tmem_hashbucket *hb = &pool->hashbucket[0]; - int i; - - for (i = 0; i < TMEM_HASH_BUCKETS; i++, hb++) { - hb->obj_rb_root = RB_ROOT; - spin_lock_init(&hb->lock); - } - INIT_LIST_HEAD(&pool->pool_list); - atomic_set(&pool->obj_count, 0); - SET_SENTINEL(pool, POOL); - list_add_tail(&pool->pool_list, &tmem_global_pool_list); - pool->persistent = persistent; - pool->shared = shared; -} - -/* The following must be called with tmem state locked */ -static void tmem_cleanup(void) -{ - (*tmem_hostops.flush_all_obj)(); -} - -void tmem_enable(void) -{ - pr_info("turning tmem on\n"); - tmem_enabled = true; - - (*tmem_hostops.control)(false); -} - -void tmem_disable(void) -{ - pr_info("turning tmem off\n"); - tmem_enabled = false; - - tmem_cleanup(); - (*tmem_hostops.control)(true); -} diff --git a/drivers/staging/qcache/tmem.h b/drivers/staging/qcache/tmem.h deleted file mode 100644 index 359c2010523..00000000000 --- a/drivers/staging/qcache/tmem.h +++ /dev/null @@ -1,212 +0,0 @@ -/* - * tmem.h - * - * Transcendent memory - * - * Copyright (c) 2009-2011, Dan Magenheimer, Oracle Corp. - * Copyright (c) 2011-2012, The Linux Foundation. All rights reserved. - */ - -#ifndef _TMEM_H_ -#define _TMEM_H_ - -#include -#include -#include -#include -#include - -/* - * These are pre-defined by the Xen<->Linux ABI - */ -#define TMEM_PUT_PAGE 4 -#define TMEM_GET_PAGE 5 -#define TMEM_FLUSH_PAGE 6 -#define TMEM_FLUSH_OBJECT 7 -#define TMEM_POOL_PERSIST 1 -#define TMEM_POOL_SHARED 2 -#define TMEM_POOL_PRECOMPRESSED 4 -#define TMEM_POOL_PAGESIZE_SHIFT 4 -#define TMEM_POOL_PAGESIZE_MASK 0xf -#define TMEM_POOL_RESERVED_BITS 0x00ffff00 - -/* - * sentinels have proven very useful for debugging but can be removed - * or disabled before final merge. - */ -#define SENTINELS -#ifdef SENTINELS -#define DECL_SENTINEL uint32_t sentinel; -#define SET_SENTINEL(_x, _y) (_x->sentinel = _y##_SENTINEL) -#define INVERT_SENTINEL(_x, _y) (_x->sentinel = ~_y##_SENTINEL) -#define ASSERT_SENTINEL(_x, _y) WARN_ON(_x->sentinel != _y##_SENTINEL) -#define ASSERT_INVERTED_SENTINEL(_x, _y) WARN_ON(_x->sentinel != ~_y##_SENTINEL) -#else -#define DECL_SENTINEL -#define SET_SENTINEL(_x, _y) do { } while (0) -#define INVERT_SENTINEL(_x, _y) do { } while (0) -#define ASSERT_SENTINEL(_x, _y) do { } while (0) -#define ASSERT_INVERTED_SENTINEL(_x, _y) do { } while (0) -#endif - -/* - * A pool is the highest-level data structure managed by tmem and - * usually corresponds to a large independent set of pages such as - * a filesystem. Each pool has an id, and certain attributes and counters. - * It also contains a set of hash buckets, each of which contains an rbtree - * of objects and a lock to manage concurrency within the pool. - */ - -#define TMEM_HASH_BUCKET_BITS 8 -#define TMEM_HASH_BUCKETS (1<persistent) -#define is_ephemeral(_p) (!(_p->persistent)) - -/* - * An object id ("oid") is large: 192-bits (to ensure, for example, files - * in a modern filesystem can be uniquely identified). - */ - -struct tmem_oid { - uint64_t oid[3]; -}; - -static inline void tmem_oid_set_invalid(struct tmem_oid *oidp) -{ - oidp->oid[0] = oidp->oid[1] = oidp->oid[2] = -1UL; -} - -static inline bool tmem_oid_valid(struct tmem_oid *oidp) -{ - return oidp->oid[0] != -1UL || oidp->oid[1] != -1UL || - oidp->oid[2] != -1UL; -} - -static inline int tmem_oid_compare(struct tmem_oid *left, - struct tmem_oid *right) -{ - int ret; - - if (left->oid[2] == right->oid[2]) { - if (left->oid[1] == right->oid[1]) { - if (left->oid[0] == right->oid[0]) - ret = 0; - else if (left->oid[0] < right->oid[0]) - ret = -1; - else - return 1; - } else if (left->oid[1] < right->oid[1]) - ret = -1; - else - ret = 1; - } else if (left->oid[2] < right->oid[2]) - ret = -1; - else - ret = 1; - return ret; -} - -static inline unsigned tmem_oid_hash(struct tmem_oid *oidp) -{ - return hash_long(oidp->oid[0] ^ oidp->oid[1] ^ oidp->oid[2], - TMEM_HASH_BUCKET_BITS); -} - -/* - * A tmem_obj contains an identifier (oid), pointers to the parent - * pool and the rb_tree to which it belongs, counters, and an ordered - * set of pampds, structured in a radix-tree-like tree. The intermediate - * nodes of the tree are called tmem_objnodes. - */ - -struct tmem_objnode; - -struct tmem_obj { - struct tmem_oid oid; - struct tmem_pool *pool; - struct rb_node rb_tree_node; - struct tmem_objnode *objnode_tree_root; - unsigned int objnode_tree_height; - unsigned long objnode_count; - long pampd_count; - void *extra; /* for private use by pampd implementation */ - DECL_SENTINEL -}; - -#define OBJNODE_TREE_MAP_SHIFT 6 -#define OBJNODE_TREE_MAP_SIZE (1UL << OBJNODE_TREE_MAP_SHIFT) -#define OBJNODE_TREE_MAP_MASK (OBJNODE_TREE_MAP_SIZE-1) -#define OBJNODE_TREE_INDEX_BITS (8 /* CHAR_BIT */ * sizeof(unsigned long)) -#define OBJNODE_TREE_MAX_PATH \ - (OBJNODE_TREE_INDEX_BITS/OBJNODE_TREE_MAP_SHIFT + 2) - -struct tmem_objnode { - struct tmem_obj *obj; - DECL_SENTINEL - void *slots[OBJNODE_TREE_MAP_SIZE]; - unsigned int slots_in_use; -}; - -/* pampd abstract datatype methods provided by the PAM implementation */ -struct tmem_pamops { - void *(*create)(char *, size_t, bool, int, - struct tmem_pool *, struct tmem_oid *, uint32_t); - int (*get_data)(char *, size_t *, bool, void *, struct tmem_pool *, - struct tmem_oid *, uint32_t); - int (*get_data_and_free)(char *, size_t *, bool, void *, - struct tmem_pool *, struct tmem_oid *, - uint32_t); - void (*free)(void *, struct tmem_pool *, struct tmem_oid *, uint32_t); - void (*free_obj)(struct tmem_pool *, struct tmem_obj *); - bool (*is_remote)(void *); - void (*new_obj)(struct tmem_obj *); - int (*replace_in_obj)(void *, struct tmem_obj *); -}; -extern void tmem_register_pamops(struct tmem_pamops *m); - -/* memory allocation methods provided by the host implementation */ -struct tmem_hostops { - struct tmem_obj *(*obj_alloc)(struct tmem_pool *); - void (*obj_free)(struct tmem_obj *, struct tmem_pool *); - struct tmem_objnode *(*objnode_alloc)(struct tmem_pool *); - void (*objnode_free)(struct tmem_objnode *, struct tmem_pool *); - void (*flush_all_obj)(void); - void (*control)(bool); -}; -extern void tmem_register_hostops(struct tmem_hostops *m); - -/* core tmem accessor functions */ -extern int tmem_put(struct tmem_pool *, struct tmem_oid *, uint32_t index, - char *, size_t, bool, bool); -extern int tmem_get(struct tmem_pool *, struct tmem_oid *, uint32_t index, - char *, size_t *, bool, int); -extern int tmem_replace(struct tmem_pool *, struct tmem_oid *, uint32_t index, - void *); -extern int tmem_flush_page(struct tmem_pool *, struct tmem_oid *, - uint32_t index); -extern int tmem_flush_object(struct tmem_pool *, struct tmem_oid *); -extern int tmem_destroy_pool(struct tmem_pool *); -extern int tmem_flush_pool(struct tmem_pool *); -extern void tmem_new_pool(struct tmem_pool *, uint32_t); - -extern void tmem_enable(void); -extern void tmem_disable(void); -#endif /* _TMEM_H */ diff --git a/include/linux/android_pmem.h b/include/linux/android_pmem.h index cfca4917e02..f338d152ccf 100644 --- a/include/linux/android_pmem.h +++ b/include/linux/android_pmem.h @@ -151,10 +151,6 @@ struct android_pmem_platform_data * indicates that this region should be mapped/unmaped as needed */ int map_on_demand; - /* - * indicates this pmem may be reused via fmem - */ - int reusable; }; int pmem_setup(struct android_pmem_platform_data *pdata, diff --git a/include/linux/fmem.h b/include/linux/fmem.h deleted file mode 100644 index cda4a0fe13d..00000000000 --- a/include/linux/fmem.h +++ /dev/null @@ -1,62 +0,0 @@ -/* - * - * Copyright (c) 2011-2012, The Linux Foundation. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 and - * only version 2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - */ -#ifndef _FMEM_H_ -#define _FMEM_H_ - -#include - -struct fmem_platform_data { - unsigned long phys; - unsigned long size; - unsigned long reserved_size_low; - unsigned long reserved_size_high; - unsigned long align; -}; - -struct fmem_data { - unsigned long phys; - void *virt; - struct vm_struct *area; - unsigned long size; - unsigned long reserved_size_low; - unsigned long reserved_size_high; -}; - -enum fmem_state { - FMEM_UNINITIALIZED = 0, - FMEM_C_STATE, - FMEM_T_STATE, - FMEM_O_STATE, -}; - -#ifdef CONFIG_QCACHE -struct fmem_data *fmem_get_info(void); -int fmem_set_state(enum fmem_state); -void lock_fmem_state(void); -void unlock_fmem_state(void); -void *fmem_map_virtual_area(int cacheability); -void fmem_unmap_virtual_area(void); -#else -static inline struct fmem_data *fmem_get_info(void) { return NULL; } -static inline int fmem_set_state(enum fmem_state f) { return -ENODEV; } -static inline void lock_fmem_state(void) { return; } -static inline void unlock_fmem_state(void) { return; } -static inline void *fmem_map_virtual_area(int cacheability) { return NULL; } -static inline void fmem_unmap_virtual_area(void) { return; } -#endif - -int request_fmem_c_region(void *unused); -int release_fmem_c_region(void *unused); -#endif diff --git a/include/linux/msm_ion.h b/include/linux/msm_ion.h index 20b7317cfdc..6a8633b357e 100644 --- a/include/linux/msm_ion.h +++ b/include/linux/msm_ion.h @@ -97,7 +97,6 @@ enum cp_mem_usage { #define ION_PIL1_HEAP_NAME "pil_1" #define ION_PIL2_HEAP_NAME "pil_2" #define ION_QSECOM_HEAP_NAME "qsecom" -#define ION_FMEM_HEAP_NAME "fmem" #define ION_SET_CACHED(__cache) (__cache | ION_FLAG_CACHED) #define ION_SET_UNCACHED(__cache) (__cache & ~ION_FLAG_CACHED) @@ -129,12 +128,7 @@ enum cp_mem_usage { * @secure_size: Memory size for securing the heap. * Note: This might be different from actual size * of this heap in the case of a shared heap. - * @reusable Flag indicating whether this heap is reusable of not. - * (see FMEM) - * @mem_is_fmem Flag indicating whether this memory is coming from fmem - * or not. * @fixed_position If nonzero, position in the fixed area. - * @virt_addr: Virtual address used when using fmem. * @iommu_map_all: Indicates whether we should map whole heap into IOMMU. * @iommu_2x_map_domain: Indicates the domain to use for overmapping. * @request_region: function to be called when the number of allocations @@ -153,13 +147,10 @@ struct ion_cp_heap_pdata { unsigned int align; ion_phys_addr_t secure_base; /* Base addr used when heap is shared */ size_t secure_size; /* Size used for securing heap when heap is shared*/ - int reusable; - int mem_is_fmem; int is_cma; enum ion_fixed_position fixed_position; int iommu_map_all; int iommu_2x_map_domain; - void *virt_addr; int (*request_region)(void *); int (*release_region)(void *); void *(*setup_region)(void); @@ -171,8 +162,6 @@ struct ion_cp_heap_pdata { * struct ion_co_heap_pdata - defines a carveout heap in the given platform * @adjacent_mem_id: Id of heap that this heap must be adjacent to. * @align: Alignment requirement for the memory - * @mem_is_fmem Flag indicating whether this memory is coming from fmem - * or not. * @fixed_position If nonzero, position in the fixed area. * @request_region: function to be called when the number of allocations * goes from 0 -> 1 @@ -185,7 +174,6 @@ struct ion_cp_heap_pdata { struct ion_co_heap_pdata { int adjacent_mem_id; unsigned int align; - int mem_is_fmem; enum ion_fixed_position fixed_position; int (*request_region)(void *); int (*release_region)(void *);