From 377c92f5216163eeb271231335cac7e6d0c56a51 Mon Sep 17 00:00:00 2001 From: Ravinder Konka Date: Tue, 6 Oct 2015 12:33:24 +0530 Subject: [PATCH 001/168] defconfig: msm8974: Enable CONFIG_NETFILTER_XT_TARGET_IDLETIMER Enabling the config option to support IDLETIMER Change-Id: Ic59260f34e372952dd455ca1942b5f5ccaaabf24 CRs-Fixed: 724368 Signed-off-by: Ravinder Konka --- arch/arm/configs/msm8974-perf_defconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm/configs/msm8974-perf_defconfig b/arch/arm/configs/msm8974-perf_defconfig index bbd59bafbc4..8bc7e431897 100755 --- a/arch/arm/configs/msm8974-perf_defconfig +++ b/arch/arm/configs/msm8974-perf_defconfig @@ -161,6 +161,7 @@ CONFIG_NF_CT_NETLINK=y CONFIG_NETFILTER_TPROXY=y CONFIG_NETFILTER_XT_TARGET_CLASSIFY=y CONFIG_NETFILTER_XT_TARGET_CONNMARK=y +CONFIG_NETFILTER_XT_TARGET_IDLETIMER=y CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=y CONFIG_NETFILTER_XT_TARGET_LOG=y CONFIG_NETFILTER_XT_TARGET_MARK=y From 039078391201f7089d20d770071de15f5cdae771 Mon Sep 17 00:00:00 2001 From: Raghavendra Ambadas Date: Thu, 10 Sep 2015 10:52:26 +0530 Subject: [PATCH 002/168] msm: mdss: Initialize pipe list while cleanup Crash is seen in cleanup function while accessing corrupted pipe list. To avoid this, initialize the pipe list before destroying the pipe. Change-Id: I64ae0113d99491a18ad4d9f6e30e935f8ec0edbb Signed-off-by: Raghavendra Ambadas --- drivers/video/msm/mdss/mdss_mdp_overlay.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/video/msm/mdss/mdss_mdp_overlay.c b/drivers/video/msm/mdss/mdss_mdp_overlay.c index a1d8c322fc4..5be40d83d6b 100644 --- a/drivers/video/msm/mdss/mdss_mdp_overlay.c +++ b/drivers/video/msm/mdss/mdss_mdp_overlay.c @@ -977,6 +977,7 @@ static void mdss_mdp_overlay_cleanup(struct msm_fb_data_type *mfd, else __mdss_mdp_overlay_free_list_add(mfd, &pipe->front_buf); mdss_mdp_overlay_free_buf(&pipe->back_buf); + list_del_init(&pipe->list); mdss_mdp_pipe_destroy(pipe); } mutex_unlock(&mdp5_data->list_lock); From d8aac434f05e3dd4f5fa72c8f4f69720d6e29ec8 Mon Sep 17 00:00:00 2001 From: roel kluin Date: Tue, 15 Sep 2015 17:08:24 +0530 Subject: [PATCH 003/168] printk: remove unused code from kernel/printk.c both log_buf_copy() and log_buf_len are unused. Change-Id: Ibf4621dcf6c70c0dfce6441a5e9c2e033642f620 Signed-off-by: Ingo Molnar Git-commit: acff181d3574244e651913df77332e897b88bff4 Git-repo: https://android.googlesource.com/kernel/common/ [schikk@codeaurora.org: removed log_buf_clear also ] Signed-off-by: Swetha Chikkaboraiah --- kernel/printk.c | 47 ----------------------------------------------- 1 file changed, 47 deletions(-) diff --git a/kernel/printk.c b/kernel/printk.c index 7f51b039e0f..46531b9304a 100644 --- a/kernel/printk.c +++ b/kernel/printk.c @@ -294,53 +294,6 @@ static inline void boot_delay_msec(void) } #endif -/* - * Return the number of unread characters in the log buffer. - */ -static int log_buf_get_len(void) -{ - return logged_chars; -} - -/* - * Clears the ring-buffer - */ -void log_buf_clear(void) -{ - logged_chars = 0; -} - -/* - * Copy a range of characters from the log buffer. - */ -int log_buf_copy(char *dest, int idx, int len) -{ - int ret, max; - bool took_lock = false; - - if (!oops_in_progress) { - raw_spin_lock_irq(&logbuf_lock); - took_lock = true; - } - - max = log_buf_get_len(); - if (idx < 0 || idx >= max) { - ret = -1; - } else { - if (len > max - idx) - len = max - idx; - ret = len; - idx += (log_end - max); - while (len-- > 0) - dest[len] = LOG_BUF(idx + len); - } - - if (took_lock) - raw_spin_unlock_irq(&logbuf_lock); - - return ret; -} - #ifdef CONFIG_SECURITY_DMESG_RESTRICT int dmesg_restrict = 1; #else From eae38ecef4e08c5b0da904718cc3bace912fadc2 Mon Sep 17 00:00:00 2001 From: Kay Sievers Date: Tue, 15 Sep 2015 12:35:48 +0530 Subject: [PATCH 004/168] printk: convert byte-buffer to variable-length record buffer - Record-based stream instead of the traditional byte stream buffer. All records carry a 64 bit timestamp, the syslog facility and priority in the record header. - Records consume almost the same amount, sometimes less memory than the traditional byte stream buffer (if printk_time is enabled). The record header is 16 bytes long, plus some padding bytes at the end if needed. The byte-stream buffer needed 3 chars for the syslog prefix, 15 char for the timestamp and a newline. - Buffer management is based on message sequence numbers. When records need to be discarded, the reading heads move on to the next full record. Unlike the byte-stream buffer, no old logged lines get truncated or partly overwritten by new ones. Sequence numbers also allow consumers of the log stream to get notified if any message in the stream they are about to read gets discarded during the time of reading. - Better buffered IO support for KERN_CONT continuation lines, when printk() is called multiple times for a single line. The use of KERN_CONT is now mandatory to use continuation; a few places in the kernel need trivial fixes here. The buffering could possibly be extended to per-cpu variables to allow better thread-safety for multiple printk() invocations for a single line. - Full-featured syslog facility value support. Different facilities can tag their messages. All userspace-injected messages enforce a facility value > 0 now, to be able to reliably distinguish them from the kernel-generated messages. Independent subsystems like a baseband processor running its own firmware, or a kernel-related userspace process can use their own unique facility values. Multiple independent log streams can co-exist that way in the same buffer. All share the same global sequence number counter to ensure proper ordering (and interleaving) and to allow the consumers of the log to reliably correlate the events from different facilities. Change-Id: I17dc787fe0557a489313b3ca9ed8127a5cafd050 Tested-by: William Douglas Signed-off-by: Kay Sievers Signed-off-by: Greg Kroah-Hartman Git-commit: 7ff9554bb578ba02166071d2d487b7fc7d860d62 Git-repo: https://android.googlesource.com/kernel/common/ [schikk@codeaurora.org: Resolved merge conflicts ] Signed-off-by: Swetha Chikkaboraiah --- drivers/char/mem.c | 55 ++- include/linux/printk.h | 11 + kernel/printk.c | 1015 +++++++++++++++++++++++----------------- 3 files changed, 635 insertions(+), 446 deletions(-) diff --git a/drivers/char/mem.c b/drivers/char/mem.c index 67e19b63199..48d794084f9 100644 --- a/drivers/char/mem.c +++ b/drivers/char/mem.c @@ -825,33 +825,54 @@ static const struct file_operations oldmem_fops = { static ssize_t kmsg_writev(struct kiocb *iocb, const struct iovec *iv, unsigned long count, loff_t pos) { - char *line, *p; + char *buf, *line; int i; - ssize_t ret = -EFAULT; + int level = default_message_loglevel; + int facility = 1; /* LOG_USER */ size_t len = iov_length(iv, count); + ssize_t ret = len; - line = kmalloc(len + 1, GFP_KERNEL); - if (line == NULL) + if (len > 1024) + return -EINVAL; + buf = kmalloc(len+1, GFP_KERNEL); + if (buf == NULL) return -ENOMEM; - /* - * copy all vectors into a single string, to ensure we do - * not interleave our log line with other printk calls - */ - p = line; + line = buf; for (i = 0; i < count; i++) { - if (copy_from_user(p, iv[i].iov_base, iv[i].iov_len)) + if (copy_from_user(line, iv[i].iov_base, iv[i].iov_len)) goto out; - p += iv[i].iov_len; + line += iv[i].iov_len; } - p[0] = '\0'; - ret = printk("%s", line); - /* printk can add a prefix */ - if (ret > len) - ret = len; + /* + * Extract and skip the syslog prefix <[0-9]*>. Coming from userspace + * the decimal value represents 32bit, the lower 3 bit are the log + * level, the rest are the log facility. + * + * If no prefix or no userspace facility is specified, we + * enforce LOG_USER, to be able to reliably distinguish + * kernel-generated messages from userspace-injected ones. + */ + line = buf; + if (line[0] == '<') { + char *endp = NULL; + + i = simple_strtoul(line+1, &endp, 10); + if (endp && endp[0] == '>') { + level = i & 7; + if (i >> 3) + facility = i >> 3; + endp++; + len -= endp - line; + line = endp; + } + } + line[len] = '\0'; + + printk_emit(facility, level, NULL, 0, "%s", line); out: - kfree(line); + kfree(buf); return ret; } diff --git a/include/linux/printk.h b/include/linux/printk.h index 0525927f203..2442de11cd1 100644 --- a/include/linux/printk.h +++ b/include/linux/printk.h @@ -95,8 +95,19 @@ extern int printk_needs_cpu(int cpu); extern void printk_tick(void); #ifdef CONFIG_PRINTK +asmlinkage __printf(5, 0) +int vprintk_emit(int facility, int level, + const char *dict, size_t dictlen, + const char *fmt, va_list args); + asmlinkage __printf(1, 0) int vprintk(const char *fmt, va_list args); + +asmlinkage __printf(5, 6) __cold +asmlinkage int printk_emit(int facility, int level, + const char *dict, size_t dictlen, + const char *fmt, ...); + asmlinkage __printf(1, 2) __cold int printk(const char *fmt, ...); diff --git a/kernel/printk.c b/kernel/printk.c index 46531b9304a..11cee7f386e 100644 --- a/kernel/printk.c +++ b/kernel/printk.c @@ -55,8 +55,6 @@ void asmlinkage __attribute__((weak)) early_printk(const char *fmt, ...) { } -#define __LOG_BUF_LEN (1 << CONFIG_LOG_BUF_SHIFT) - /* printk's without a loglevel use this.. */ #define DEFAULT_MESSAGE_LOGLEVEL CONFIG_DEFAULT_MESSAGE_LOGLEVEL @@ -99,23 +97,6 @@ EXPORT_SYMBOL_GPL(console_drivers); */ static int console_locked, console_suspended; -/* - * logbuf_lock protects log_buf, log_start, log_end, con_start and logged_chars - * It is also used in interesting ways to provide interlocking in - * console_unlock();. - */ -static DEFINE_RAW_SPINLOCK(logbuf_lock); - -#define LOG_BUF_MASK (log_buf_len-1) -#define LOG_BUF(idx) (log_buf[(idx) & LOG_BUF_MASK]) - -/* - * The indices into log_buf are not constrained to log_buf_len - they - * must be masked before subscripting - */ -static unsigned log_start; /* Index into log_buf: next char to be read by syslog() */ -static unsigned con_start; /* Index into log_buf: next char to be sent to consoles */ -static unsigned log_end; /* Index into log_buf: most-recently-written-char + 1 */ /* * If exclusive_console is non-NULL then only this console is to be printed to. @@ -147,12 +128,176 @@ EXPORT_SYMBOL(console_set_on_cmdline); static int console_may_schedule; #ifdef CONFIG_PRINTK +/* + * The printk log buffer consists of a chain of concatenated variable + * length records. Every record starts with a record header, containing + * the overall length of the record. + * + * The heads to the first and last entry in the buffer, as well as the + * sequence numbers of these both entries are maintained when messages + * are stored.. + * + * If the heads indicate available messages, the length in the header + * tells the start next message. A length == 0 for the next message + * indicates a wrap-around to the beginning of the buffer. + * + * Every record carries the monotonic timestamp in microseconds, as well as + * the standard userspace syslog level and syslog facility. The usual + * kernel messages use LOG_KERN; userspace-injected messages always carry + * a matching syslog facility, by default LOG_USER. The origin of every + * message can be reliably determined that way. + * + * The human readable log message directly follows the message header. The + * length of the message text is stored in the header, the stored message + * is not terminated. + * + */ +struct log { + u64 ts_nsec; /* timestamp in nanoseconds */ + u16 len; /* length of entire record */ + u16 text_len; /* length of text buffer */ + u16 dict_len; /* length of dictionary buffer */ + u16 level; /* syslog level + facility */ +}; + +/* + * The logbuf_lock protects kmsg buffer, indices, counters. It is also + * used in interesting ways to provide interlocking in console_unlock(); + */ +static DEFINE_RAW_SPINLOCK(logbuf_lock); + +/* cpu currently holding logbuf_lock */ +static volatile unsigned int logbuf_cpu = UINT_MAX; + +#define LOG_LINE_MAX 1024 + +/* record buffer */ +#define __LOG_BUF_LEN (1 << CONFIG_LOG_BUF_SHIFT) static char __log_buf[__LOG_BUF_LEN]; static char *log_buf = __log_buf; -static int log_buf_len = __LOG_BUF_LEN; -static unsigned logged_chars; /* Number of chars produced since last read+clear operation */ -static int saved_console_loglevel = -1; +static u32 log_buf_len = __LOG_BUF_LEN; + +/* index and sequence number of the first record stored in the buffer */ +static u64 log_first_seq; +static u32 log_first_idx; + +/* index and sequence number of the next record to store in the buffer */ +static u64 log_next_seq; +static u32 log_next_idx; + +/* the next printk record to read after the last 'clear' command */ +static u64 clear_seq; +static u32 clear_idx; + +/* the next printk record to read by syslog(READ) or /proc/kmsg */ +static u64 syslog_seq; +static u32 syslog_idx; + +/* human readable text of the record */ +static char *log_text(const struct log *msg) +{ + return (char *)msg + sizeof(struct log); +} + +/* optional key/value pair dictionary attached to the record */ +static char *log_dict(const struct log *msg) +{ + return (char *)msg + sizeof(struct log) + msg->text_len; +} + +/* get record by index; idx must point to valid msg */ +static struct log *log_from_idx(u32 idx) +{ + struct log *msg = (struct log *)(log_buf + idx); + + /* + * A length == 0 record is the end of buffer marker. Wrap around and + * read the message at the start of the buffer. + */ + if (!msg->len) + return (struct log *)log_buf; + return msg; +} + +/* get next record; idx must point to valid msg */ +static u32 log_next(u32 idx) +{ + struct log *msg = (struct log *)(log_buf + idx); + + /* length == 0 indicates the end of the buffer; wrap */ + /* + * A length == 0 record is the end of buffer marker. Wrap around and + * read the message at the start of the buffer as *this* one, and + * return the one after that. + */ + if (!msg->len) { + msg = (struct log *)log_buf; + return msg->len; + } + return idx + msg->len; +} + +#if !defined(CONFIG_64BIT) || defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) +#define LOG_ALIGN 4 +#else +#define LOG_ALIGN 8 +#endif + +/* insert record into the buffer, discard old ones, update heads */ +static void log_store(int facility, int level, + const char *dict, u16 dict_len, + const char *text, u16 text_len) +{ + struct log *msg; + u32 size, pad_len; + + /* number of '\0' padding bytes to next message */ + size = sizeof(struct log) + text_len + dict_len; + pad_len = (-size) & (LOG_ALIGN - 1); + size += pad_len; + + while (log_first_seq < log_next_seq) { + u32 free; + + if (log_next_idx > log_first_idx) + free = max(log_buf_len - log_next_idx, log_first_idx); + else + free = log_first_idx - log_next_idx; + + if (free > size + sizeof(struct log)) + break; + + /* drop old messages until we have enough contiuous space */ + log_first_idx = log_next(log_first_idx); + log_first_seq++; + } + + if (log_next_idx + size + sizeof(struct log) >= log_buf_len) { + /* + * This message + an additional empty header does not fit + * at the end of the buffer. Add an empty header with len == 0 + * to signify a wrap around. + */ + memset(log_buf + log_next_idx, 0, sizeof(struct log)); + log_next_idx = 0; + } + + /* fill message */ + msg = (struct log *)(log_buf + log_next_idx); + memcpy(log_text(msg), text, text_len); + msg->text_len = text_len; + memcpy(log_dict(msg), dict, dict_len); + msg->dict_len = dict_len; + msg->level = (facility << 3) | (level & 7); + msg->ts_nsec = local_clock(); + memset(log_dict(msg) + dict_len, 0, pad_len); + msg->len = sizeof(struct log) + text_len + dict_len + pad_len; + + /* insert message */ + log_next_idx += msg->len; + log_next_seq++; +} #ifdef CONFIG_KEXEC /* @@ -166,9 +311,9 @@ static int saved_console_loglevel = -1; void log_buf_kexec_setup(void) { VMCOREINFO_SYMBOL(log_buf); - VMCOREINFO_SYMBOL(log_end); VMCOREINFO_SYMBOL(log_buf_len); - VMCOREINFO_SYMBOL(logged_chars); + VMCOREINFO_SYMBOL(log_first_idx); + VMCOREINFO_SYMBOL(log_next_idx); } #endif @@ -192,7 +337,6 @@ early_param("log_buf_len", log_buf_len_setup); void __init setup_log_buf(int early) { unsigned long flags; - unsigned start, dest_idx, offset; char *new_log_buf; int free; @@ -220,20 +364,8 @@ void __init setup_log_buf(int early) log_buf_len = new_log_buf_len; log_buf = new_log_buf; new_log_buf_len = 0; - free = __LOG_BUF_LEN - log_end; - - offset = start = min(con_start, log_start); - dest_idx = 0; - while (start != log_end) { - unsigned log_idx_mask = start & (__LOG_BUF_LEN - 1); - - log_buf[dest_idx] = __log_buf[log_idx_mask]; - start++; - dest_idx++; - } - log_start -= offset; - con_start -= offset; - log_end -= offset; + free = __LOG_BUF_LEN - log_next_idx; + memcpy(log_buf, __log_buf, __LOG_BUF_LEN); raw_spin_unlock_irqrestore(&logbuf_lock, flags); pr_info("log_buf_len: %d\n", log_buf_len); @@ -333,11 +465,165 @@ static int check_syslog_permissions(int type, bool from_file) return 0; } +#if defined(CONFIG_PRINTK_TIME) +static bool printk_time = 1; +#else +static bool printk_time; +#endif +module_param_named(time, printk_time, bool, S_IRUGO | S_IWUSR); + +static int syslog_print_line(u32 idx, char *text, size_t size) +{ + struct log *msg; + size_t len; + + msg = log_from_idx(idx); + if (!text) { + /* calculate length only */ + len = 3; + + if (msg->level > 9) + len++; + if (msg->level > 99) + len++; + + if (printk_time) + len += 15; + + len += msg->text_len; + len++; + return len; + } + + len = sprintf(text, "<%u>", msg->level); + + if (printk_time) { + unsigned long long t = msg->ts_nsec; + unsigned long rem_ns = do_div(t, 1000000000); + + len += sprintf(text + len, "[%5lu.%06lu] ", + (unsigned long) t, rem_ns / 1000); + } + + if (len + msg->text_len > size) + return -EINVAL; + memcpy(text + len, log_text(msg), msg->text_len); + len += msg->text_len; + text[len++] = '\n'; + return len; +} + +static int syslog_print(char __user *buf, int size) +{ + char *text; + int len; + + text = kmalloc(LOG_LINE_MAX, GFP_KERNEL); + if (!text) + return -ENOMEM; + + raw_spin_lock_irq(&logbuf_lock); + if (syslog_seq < log_first_seq) { + /* messages are gone, move to first one */ + syslog_seq = log_first_seq; + syslog_idx = log_first_idx; + } + len = syslog_print_line(syslog_idx, text, LOG_LINE_MAX); + syslog_idx = log_next(syslog_idx); + syslog_seq++; + raw_spin_unlock_irq(&logbuf_lock); + + if (len > 0 && copy_to_user(buf, text, len)) + len = -EFAULT; + + kfree(text); + return len; +} + +static int syslog_print_all(char __user *buf, int size, bool clear) +{ + char *text; + int len = 0; + + text = kmalloc(LOG_LINE_MAX, GFP_KERNEL); + if (!text) + return -ENOMEM; + + raw_spin_lock_irq(&logbuf_lock); + if (buf) { + u64 next_seq; + u64 seq; + u32 idx; + + if (clear_seq < log_first_seq) { + /* messages are gone, move to first available one */ + clear_seq = log_first_seq; + clear_idx = log_first_idx; + } + + /* + * Find first record that fits, including all following records, + * into the user-provided buffer for this dump. + */ + seq = clear_seq; + idx = clear_idx; + while (seq < log_next_seq) { + len += syslog_print_line(idx, NULL, 0); + idx = log_next(idx); + seq++; + } + seq = clear_seq; + idx = clear_idx; + while (len > size && seq < log_next_seq) { + len -= syslog_print_line(idx, NULL, 0); + idx = log_next(idx); + seq++; + } + + /* last message in this dump */ + next_seq = log_next_seq; + + len = 0; + while (len >= 0 && seq < next_seq) { + int textlen; + + textlen = syslog_print_line(idx, text, LOG_LINE_MAX); + if (textlen < 0) { + len = textlen; + break; + } + idx = log_next(idx); + seq++; + + raw_spin_unlock_irq(&logbuf_lock); + if (copy_to_user(buf + len, text, textlen)) + len = -EFAULT; + else + len += textlen; + raw_spin_lock_irq(&logbuf_lock); + + if (seq < log_first_seq) { + /* messages are gone, move to next one */ + seq = log_first_seq; + idx = log_first_idx; + } + } + } + + if (clear) { + clear_seq = log_next_seq; + clear_idx = log_next_idx; + } + raw_spin_unlock_irq(&logbuf_lock); + + kfree(text); + return len; +} + int do_syslog(int type, char __user *buf, int len, bool from_file) { - unsigned i, j, limit, count; - int do_clear = 0; - char c; + bool clear = false; + static int saved_console_loglevel = -1; int error; error = check_syslog_permissions(type, from_file); @@ -365,28 +651,14 @@ int do_syslog(int type, char __user *buf, int len, bool from_file) goto out; } error = wait_event_interruptible(log_wait, - (log_start - log_end)); + syslog_seq != log_next_seq); if (error) goto out; - i = 0; - raw_spin_lock_irq(&logbuf_lock); - while (!error && (log_start != log_end) && i < len) { - c = LOG_BUF(log_start); - log_start++; - raw_spin_unlock_irq(&logbuf_lock); - error = __put_user(c,buf); - buf++; - i++; - cond_resched(); - raw_spin_lock_irq(&logbuf_lock); - } - raw_spin_unlock_irq(&logbuf_lock); - if (!error) - error = i; + error = syslog_print(buf, len); break; /* Read/clear last kernel messages */ case SYSLOG_ACTION_READ_CLEAR: - do_clear = 1; + clear = true; /* FALL THRU */ /* Read last kernel messages */ case SYSLOG_ACTION_READ_ALL: @@ -400,52 +672,11 @@ int do_syslog(int type, char __user *buf, int len, bool from_file) error = -EFAULT; goto out; } - count = len; - if (count > log_buf_len) - count = log_buf_len; - raw_spin_lock_irq(&logbuf_lock); - if (count > logged_chars) - count = logged_chars; - if (do_clear) - logged_chars = 0; - limit = log_end; - /* - * __put_user() could sleep, and while we sleep - * printk() could overwrite the messages - * we try to copy to user space. Therefore - * the messages are copied in reverse. - */ - for (i = 0; i < count && !error; i++) { - j = limit-1-i; - if (j + log_buf_len < log_end) - break; - c = LOG_BUF(j); - raw_spin_unlock_irq(&logbuf_lock); - error = __put_user(c,&buf[count-1-i]); - cond_resched(); - raw_spin_lock_irq(&logbuf_lock); - } - raw_spin_unlock_irq(&logbuf_lock); - if (error) - break; - error = i; - if (i != count) { - int offset = count-error; - /* buffer overflow during copy, correct user buffer. */ - for (i = 0; i < error; i++) { - if (__get_user(c,&buf[i+offset]) || - __put_user(c,&buf[i])) { - error = -EFAULT; - break; - } - cond_resched(); - } - } + error = syslog_print_all(buf, len, clear); break; /* Clear ring buffer */ case SYSLOG_ACTION_CLEAR: - logged_chars = 0; - break; + syslog_print_all(NULL, 0, true); /* Disable logging to console */ case SYSLOG_ACTION_CONSOLE_OFF: if (saved_console_loglevel == -1) @@ -473,7 +704,33 @@ int do_syslog(int type, char __user *buf, int len, bool from_file) break; /* Number of chars in the log buffer */ case SYSLOG_ACTION_SIZE_UNREAD: - error = log_end - log_start; + raw_spin_lock_irq(&logbuf_lock); + if (syslog_seq < log_first_seq) { + /* messages are gone, move to first one */ + syslog_seq = log_first_seq; + syslog_idx = log_first_idx; + } + if (from_file) { + /* + * Short-cut for poll(/"proc/kmsg") which simply checks + * for pending data, not the size; return the count of + * records, not the length. + */ + error = log_next_idx - syslog_idx; + } else { + u64 seq; + u32 idx; + + error = 0; + seq = syslog_seq; + idx = syslog_idx; + while (seq < log_next_seq) { + error += syslog_print_line(idx, NULL, 0); + idx = log_next(idx); + seq++; + } + } + raw_spin_unlock_irq(&logbuf_lock); break; /* Size of the log buffer */ case SYSLOG_ACTION_SIZE_BUFFER: @@ -502,29 +759,11 @@ void kdb_syslog_data(char *syslog_data[4]) { syslog_data[0] = log_buf; syslog_data[1] = log_buf + log_buf_len; - syslog_data[2] = log_buf + log_end - - (logged_chars < log_buf_len ? logged_chars : log_buf_len); - syslog_data[3] = log_buf + log_end; + syslog_data[2] = log_buf + log_first_idx; + syslog_data[3] = log_buf + log_next_idx; } #endif /* CONFIG_KGDB_KDB */ -/* - * Call the console drivers on a range of log_buf - */ -static void __call_console_drivers(unsigned start, unsigned end) -{ - struct console *con; - - for_each_console(con) { - if (exclusive_console && con != exclusive_console) - continue; - if ((con->flags & CON_ENABLED) && con->write && - (cpu_online(smp_processor_id()) || - (con->flags & CON_ANYTIME))) - con->write(con, &LOG_BUF(start), end - start); - } -} - static bool __read_mostly ignore_loglevel; static int __init ignore_loglevel_setup(char *str) @@ -540,144 +779,35 @@ module_param(ignore_loglevel, bool, S_IRUGO | S_IWUSR); MODULE_PARM_DESC(ignore_loglevel, "ignore loglevel setting, to" "print all kernel messages to the console."); -/* - * Write out chars from start to end - 1 inclusive - */ -static void _call_console_drivers(unsigned start, - unsigned end, int msg_log_level) -{ - trace_console(&LOG_BUF(0), start, end, log_buf_len); - - if ((msg_log_level < console_loglevel || ignore_loglevel) && - console_drivers && start != end) { - if ((start & LOG_BUF_MASK) > (end & LOG_BUF_MASK)) { - /* wrapped write */ - __call_console_drivers(start & LOG_BUF_MASK, - log_buf_len); - __call_console_drivers(0, end & LOG_BUF_MASK); - } else { - __call_console_drivers(start, end); - } - } -} - -/* - * Parse the syslog header <[0-9]*>. The decimal value represents 32bit, the - * lower 3 bit are the log level, the rest are the log facility. In case - * userspace passes usual userspace syslog messages to /dev/kmsg or - * /dev/ttyprintk, the log prefix might contain the facility. Printk needs - * to extract the correct log level for in-kernel processing, and not mangle - * the original value. - * - * If a prefix is found, the length of the prefix is returned. If 'level' is - * passed, it will be filled in with the log level without a possible facility - * value. If 'special' is passed, the special printk prefix chars are accepted - * and returned. If no valid header is found, 0 is returned and the passed - * variables are not touched. - */ -static size_t log_prefix(const char *p, unsigned int *level, char *special) -{ - unsigned int lev = 0; - char sp = '\0'; - size_t len; - - if (p[0] != '<' || !p[1]) - return 0; - if (p[2] == '>') { - /* usual single digit level number or special char */ - switch (p[1]) { - case '0' ... '7': - lev = p[1] - '0'; - break; - case 'c': /* KERN_CONT */ - case 'd': /* KERN_DEFAULT */ - sp = p[1]; - break; - default: - return 0; - } - len = 3; - } else { - /* multi digit including the level and facility number */ - char *endp = NULL; - - lev = (simple_strtoul(&p[1], &endp, 10) & 7); - if (endp == NULL || endp[0] != '>') - return 0; - len = (endp + 1) - p; - } - - /* do not accept special char if not asked for */ - if (sp && !special) - return 0; - - if (special) { - *special = sp; - /* return special char, do not touch level */ - if (sp) - return len; - } - - if (level) - *level = lev; - return len; -} - /* * Call the console drivers, asking them to write out * log_buf[start] to log_buf[end - 1]. * The console_lock must be held. */ -static void call_console_drivers(unsigned start, unsigned end) +static void call_console_drivers(int level, const char *text, size_t len) { - unsigned cur_index, start_print; - static int msg_level = -1; + struct console *con; + trace_console(text, 0, len, len); - BUG_ON(((int)(start - end)) > 0); + if (level >= console_loglevel && !ignore_loglevel) + return; + if (!console_drivers) + return; - cur_index = start; - start_print = start; - while (cur_index != end) { - if (msg_level < 0 && ((end - cur_index) > 2)) { - /* strip log prefix */ - cur_index += log_prefix(&LOG_BUF(cur_index), &msg_level, NULL); - start_print = cur_index; - } - while (cur_index != end) { - char c = LOG_BUF(cur_index); - - cur_index++; - if (c == '\n') { - if (msg_level < 0) { - /* - * printk() has already given us loglevel tags in - * the buffer. This code is here in case the - * log buffer has wrapped right round and scribbled - * on those tags - */ - msg_level = default_message_loglevel; - } - _call_console_drivers(start_print, cur_index, msg_level); - msg_level = -1; - start_print = cur_index; - break; - } - } + for_each_console(con) { + if (exclusive_console && con != exclusive_console) + continue; + if (!(con->flags & CON_ENABLED)) + continue; + if (!con->write) + continue; + if (!cpu_online(smp_processor_id()) && + !(con->flags & CON_ANYTIME)) + continue; + con->write(con, text, len); } - _call_console_drivers(start_print, end, msg_level); } -static void emit_log_char(char c) -{ - LOG_BUF(log_end) = c; - log_end++; - if (log_end - log_start > log_buf_len) - log_start = log_end - log_buf_len; - if (log_end - con_start > log_buf_len) - con_start = log_end - log_buf_len; - if (logged_chars < log_buf_len) - logged_chars++; -} /* * Zap console related locks when oopsing. Only zap at most once @@ -701,16 +831,6 @@ static void zap_locks(void) sema_init(&console_sem, 1); } -#if defined(CONFIG_PRINTK_TIME) -static bool printk_time = 1; -#else -static bool printk_time = 0; -#endif -module_param_named(time, printk_time, bool, S_IRUGO | S_IWUSR); - -static bool always_kmsg_dump; -module_param_named(always_kmsg_dump, always_kmsg_dump, bool, S_IRUGO | S_IWUSR); - /* Check if we have any console registered that can be called early in boot. */ static int have_callable_console(void) { @@ -723,56 +843,6 @@ static int have_callable_console(void) return 0; } -/** - * printk - print a kernel message - * @fmt: format string - * - * This is printk(). It can be called from any context. We want it to work. - * - * We try to grab the console_lock. If we succeed, it's easy - we log the output and - * call the console drivers. If we fail to get the semaphore we place the output - * into the log buffer and return. The current holder of the console_sem will - * notice the new output in console_unlock(); and will send it to the - * consoles before releasing the lock. - * - * One effect of this deferred printing is that code which calls printk() and - * then changes console_loglevel may break. This is because console_loglevel - * is inspected when the actual printing occurs. - * - * See also: - * printf(3) - * - * See the vsnprintf() documentation for format string extensions over C99. - */ - -asmlinkage int printk(const char *fmt, ...) -{ - va_list args; - int r; -#ifdef CONFIG_MSM_RTB - void *caller = __builtin_return_address(0); - - uncached_logk_pc(LOGK_LOGBUF, caller, (void *)log_end); -#endif - -#ifdef CONFIG_KGDB_KDB - if (unlikely(kdb_trap_printk)) { - va_start(args, fmt); - r = vkdb_printf(fmt, args); - va_end(args); - return r; - } -#endif - va_start(args, fmt); - r = vprintk(fmt, args); - va_end(args); - - return r; -} - -/* cpu currently holding logbuf_lock */ -static volatile unsigned int printk_cpu = UINT_MAX; - /* * Can we actually use the console at this time on this cpu? * @@ -816,17 +886,12 @@ static int console_trylock_for_printk(unsigned int cpu) retval = 0; } } - printk_cpu = UINT_MAX; + logbuf_cpu = UINT_MAX; if (wake) up(&console_sem); raw_spin_unlock(&logbuf_lock); return retval; } -static const char recursion_bug_msg [] = - KERN_CRIT "BUG: recent printk recursion!\n"; -static int recursion_bug; -static int new_text_line = 1; -static char printk_buf[1024]; int printk_delay_msec __read_mostly; @@ -842,15 +907,22 @@ static inline void printk_delay(void) } } -asmlinkage int vprintk(const char *fmt, va_list args) +asmlinkage int vprintk_emit(int facility, int level, + const char *dict, size_t dictlen, + const char *fmt, va_list args) { - int printed_len = 0; - int current_log_level = default_message_loglevel; + static int recursion_bug; + static char buf[LOG_LINE_MAX]; + static size_t buflen; + static int buflevel; + static char textbuf[LOG_LINE_MAX]; + char *text = textbuf; + size_t textlen; unsigned long flags; int this_cpu; - char *p; - size_t plen; - char special; + bool newline = false; + bool cont = false; + int printed_len = 0; boot_delay_msec(); printk_delay(); @@ -862,7 +934,7 @@ asmlinkage int vprintk(const char *fmt, va_list args) /* * Ouch, printk recursed into itself! */ - if (unlikely(printk_cpu == this_cpu)) { + if (unlikely(logbuf_cpu == this_cpu)) { /* * If a crash is occurring during printk() on this CPU, * then try to get the crash message out but make sure @@ -879,99 +951,93 @@ asmlinkage int vprintk(const char *fmt, va_list args) lockdep_off(); raw_spin_lock(&logbuf_lock); - printk_cpu = this_cpu; + logbuf_cpu = this_cpu; if (recursion_bug) { + static const char recursion_msg[] = + "BUG: recent printk recursion!"; + recursion_bug = 0; - strcpy(printk_buf, recursion_bug_msg); - printed_len = strlen(recursion_bug_msg); + printed_len += strlen(recursion_msg); + /* emit KERN_CRIT message */ + log_store(0, 2, NULL, 0, recursion_msg, printed_len); + } - /* Emit the output into the temporary buffer */ - printed_len += vscnprintf(printk_buf + printed_len, - sizeof(printk_buf) - printed_len, fmt, args); + /* + * The printf needs to come first; we need the syslog + * prefix which might be passed-in as a parameter. + */ + textlen = vscnprintf(text, sizeof(textbuf), fmt, args); + /* mark and strip a trailing newline */ + if (textlen && text[textlen-1] == '\n') { + textlen--; + newline = true; + } - p = printk_buf; - - /* Read log level and handle special printk prefix */ - plen = log_prefix(p, ¤t_log_level, &special); - if (plen) { - p += plen; - - switch (special) { - case 'c': /* Strip KERN_CONT, continue line */ - plen = 0; + /* strip syslog prefix and extract log level or flags */ + if (text[0] == '<' && text[1] && text[2] == '>') { + switch (text[1]) { + case '0' ... '7': + if (level == -1) + level = text[1] - '0'; + text += 3; + textlen -= 3; + break; + case 'c': /* KERN_CONT */ + cont = true; + case 'd': /* KERN_DEFAULT */ + text += 3; + textlen -= 3; break; - case 'd': /* Strip KERN_DEFAULT, start new line */ - plen = 0; - default: - if (!new_text_line) { - emit_log_char('\n'); - new_text_line = 1; - } } } - /* - * Copy the output into log_buf. If the caller didn't provide - * the appropriate log prefix, we insert them here - */ - for (; *p; p++) { - if (new_text_line) { - new_text_line = 0; + if (buflen && (!cont || dict)) { + /* no continuation; flush existing buffer */ + log_store(facility, buflevel, NULL, 0, buf, buflen); + printed_len += buflen; + buflen = 0; + } - if (plen) { - /* Copy original log prefix */ - int i; + if (buflen == 0) { + /* remember level for first message in the buffer */ + if (level == -1) + buflevel = default_message_loglevel; + else + buflevel = level; + } - for (i = 0; i < plen; i++) - emit_log_char(printk_buf[i]); - printed_len += plen; - } else { - /* Add log prefix */ - emit_log_char('<'); - emit_log_char(current_log_level + '0'); - emit_log_char('>'); - printed_len += 3; - } + if (buflen || !newline) { + /* append to existing buffer, or buffer until next message */ + if (buflen + textlen > sizeof(buf)) + textlen = sizeof(buf) - buflen; + memcpy(buf + buflen, text, textlen); + buflen += textlen; + } - if (printk_time) { - /* Add the current time stamp */ - char tbuf[50], *tp; - unsigned tlen; - unsigned long long t; - unsigned long nanosec_rem; - - t = cpu_clock(printk_cpu); - nanosec_rem = do_div(t, 1000000000); - tlen = sprintf(tbuf, "[%5lu.%06lu] ", - (unsigned long) t, - nanosec_rem / 1000); - - for (tp = tbuf; tp < tbuf + tlen; tp++) - emit_log_char(*tp); - printed_len += tlen; - } - - if (!*p) - break; + if (newline) { + /* end of line; flush buffer */ + if (buflen) { + log_store(facility, buflevel, + dict, dictlen, buf, buflen); + printed_len += buflen; + buflen = 0; + } else { + log_store(facility, buflevel, + dict, dictlen, text, textlen); + printed_len += textlen; } - emit_log_char(*p); - if (*p == '\n') - new_text_line = 1; } /* - * Try to acquire and then immediately release the - * console semaphore. The release will do all the - * actual magic (print out buffers, wake up klogd, - * etc). - * - * The console_trylock_for_printk() function - * will release 'logbuf_lock' regardless of whether it - * actually gets the semaphore or not. - */ + * Try to acquire and then immediately release the console semaphore. + * The release will print out buffers and wake up /dev/kmsg and syslog() + * users. + * The console_trylock_for_printk() function will release 'logbuf_lock' + * regardless of whether it actually gets the console semaphore or not. + */ if (console_trylock_for_printk(this_cpu)) console_unlock(); @@ -981,12 +1047,73 @@ out_restore_irqs: return printed_len; } -EXPORT_SYMBOL(printk); +EXPORT_SYMBOL(vprintk_emit); + +asmlinkage int vprintk(const char *fmt, va_list args) +{ + return vprintk_emit(0, -1, NULL, 0, fmt, args); +} + EXPORT_SYMBOL(vprintk); +asmlinkage int printk_emit(int facility, int level, + const char *dict, size_t dictlen, + const char *fmt, ...) +{ + va_list args; + int r; + + va_start(args, fmt); + r = vprintk_emit(facility, level, dict, dictlen, fmt, args); + va_end(args); + + return r; +} +EXPORT_SYMBOL(printk_emit); + +/** + * printk - print a kernel message + * @fmt: format string + * + * This is printk(). It can be called from any context. We want it to work. + * + * We try to grab the console_lock. If we succeed, it's easy - we log the + * output and call the console drivers. If we fail to get the semaphore, we + * place the output into the log buffer and return. The current holder of + * the console_sem will notice the new output in console_unlock(); and will + * send it to the consoles before releasing the lock. + * + * One effect of this deferred printing is that code which calls printk() and + * then changes console_loglevel may break. This is because console_loglevel + * is inspected when the actual printing occurs. + * + * See also: + * printf(3) + * + * See the vsnprintf() documentation for format string extensions over C99. + */ +asmlinkage int printk(const char *fmt, ...) +{ + va_list args; + int r; + +#ifdef CONFIG_KGDB_KDB + if (unlikely(kdb_trap_printk)) { + va_start(args, fmt); + r = vkdb_printf(fmt, args); + va_end(args); + return r; + } +#endif + va_start(args, fmt); + r = vprintk_emit(0, -1, NULL, 0, fmt, args); + va_end(args); + + return r; +} +EXPORT_SYMBOL(printk); #else - -static void call_console_drivers(unsigned start, unsigned end) +static void call_console_drivers(int level, const char *text, size_t len) { } @@ -1241,7 +1368,7 @@ int is_console_locked(void) } /* - * Delayed printk facility, for scheduler-internal messages: + * Delayed printk version, for scheduler-internal messages: */ #define PRINTK_BUF_SIZE 512 @@ -1276,6 +1403,9 @@ void wake_up_klogd(void) if (waitqueue_active(&log_wait)) this_cpu_or(printk_pending, PRINTK_PENDING_WAKEUP); } +/* the next printk record to write to the console */ +static u64 console_seq; +static u32 console_idx; /** * console_unlock - unlock the console system @@ -1287,15 +1417,16 @@ void wake_up_klogd(void) * by printk(). If this is the case, console_unlock(); emits * the output prior to releasing the lock. * - * If there is output waiting for klogd, we wake it up. + * If there is output waiting, we wake it /dev/kmsg and syslog() users. * * console_unlock(); may be called from any context. */ void console_unlock(void) { + static u64 seen_seq; unsigned long flags; - unsigned _con_start, _log_end; - unsigned wake_klogd = 0, retry = 0; + bool wake_klogd = false; + bool retry; if (console_suspended) { up(&console_sem); @@ -1305,17 +1436,40 @@ void console_unlock(void) console_may_schedule = 0; again: - for ( ; ; ) { + for (;;) { + struct log *msg; + static char text[LOG_LINE_MAX]; + size_t len; + int level; raw_spin_lock_irqsave(&logbuf_lock, flags); - wake_klogd |= log_start - log_end; - if (con_start == log_end) - break; /* Nothing to print */ - _con_start = con_start; - _log_end = log_end; - con_start = log_end; /* Flush */ + if (seen_seq != log_next_seq) { + wake_klogd = true; + seen_seq = log_next_seq; + } + + if (console_seq < log_first_seq) { + /* messages are gone, move to first one */ + console_seq = log_first_seq; + console_idx = log_first_idx; + } + + if (console_seq == log_next_seq) + break; + + msg = log_from_idx(console_idx); + level = msg->level & 7; + len = msg->text_len; + if (len+1 >= sizeof(text)) + len = sizeof(text)-1; + memcpy(text, log_text(msg), len); + text[len++] = '\n'; + + console_idx = log_next(console_idx); + console_seq++; + raw_spin_unlock(&logbuf_lock); stop_critical_timings(); /* don't trace print latency */ - call_console_drivers(_con_start, _log_end); + call_console_drivers(level, text, len); start_critical_timings(); local_irq_restore(flags); } @@ -1336,8 +1490,7 @@ again: * flush, no worries. */ raw_spin_lock(&logbuf_lock); - if (con_start != log_end) - retry = 1; + retry = console_seq != log_next_seq; raw_spin_unlock_irqrestore(&logbuf_lock, flags); if (retry && console_trylock()) @@ -1573,7 +1726,8 @@ void register_console(struct console *newcon) * for us. */ raw_spin_lock_irqsave(&logbuf_lock, flags); - con_start = log_start; + console_seq = syslog_seq; + console_idx = syslog_idx; raw_spin_unlock_irqrestore(&logbuf_lock, flags); /* * We're about to replay the log buffer. Only do this to the @@ -1781,6 +1935,9 @@ int kmsg_dump_unregister(struct kmsg_dumper *dumper) return err; } EXPORT_SYMBOL_GPL(kmsg_dump_unregister); +static bool always_kmsg_dump; +module_param_named(always_kmsg_dump, always_kmsg_dump, bool, S_IRUGO | S_IWUSR); + /** * kmsg_dump - dump kernel log to kernel message dumpers. @@ -1791,8 +1948,7 @@ EXPORT_SYMBOL_GPL(kmsg_dump_unregister); */ void kmsg_dump(enum kmsg_dump_reason reason) { - unsigned long end; - unsigned chars; + u64 idx; struct kmsg_dumper *dumper; const char *s1, *s2; unsigned long l1, l2; @@ -1805,23 +1961,24 @@ void kmsg_dump(enum kmsg_dump_reason reason) there's not a lot we can do about that. The new messages will overwrite the start of what we dump. */ raw_spin_lock_irqsave(&logbuf_lock, flags); - end = log_end & LOG_BUF_MASK; - chars = logged_chars; - raw_spin_unlock_irqrestore(&logbuf_lock, flags); + if (syslog_seq < log_first_seq) + idx = syslog_idx; + else + idx = log_first_idx; - if (chars > end) { - s1 = log_buf + log_buf_len - chars + end; - l1 = chars - end; - - s2 = log_buf; - l2 = end; + if (idx > log_next_idx) { + s1 = log_buf; + l1 = log_next_idx; + s2 = log_buf + idx; + l2 = log_buf_len - idx; } else { s1 = ""; l1 = 0; - s2 = log_buf + end - chars; - l2 = chars; + s2 = log_buf + idx; + l2 = log_next_idx - idx; } + raw_spin_unlock_irqrestore(&logbuf_lock, flags); rcu_read_lock(); list_for_each_entry_rcu(dumper, &dump_list, list) From 82c4abe1bd414551ad1dbdfb843dc17d54d2b694 Mon Sep 17 00:00:00 2001 From: Kay Sievers Date: Tue, 15 Sep 2015 15:21:13 +0530 Subject: [PATCH 005/168] kmsg: export printk records to the /dev/kmsg interface Support for multiple concurrent readers of /dev/kmsg, with read(), seek(), poll() support. Output of message sequence numbers, to allow userspace log consumers to reliably reconnect and reconstruct their state at any given time. After open("/dev/kmsg"), read() always returns *all* buffered records. If only future messages should be read, SEEK_END can be used. In case records get overwritten while /dev/kmsg is held open, or records get faster overwritten than they are read, the next read() will return -EPIPE and the current reading position gets updated to the next available record. The passed sequence numbers allow the log consumer to calculate the amount of lost messages. [root@mop ~]# cat /dev/kmsg 5,0,0;Linux version 3.4.0-rc1+ (kay@mop) (gcc version 4.7.0 20120315 ... 6,159,423091;ACPI: PCI Root Bridge [PCI0] (domain 0000 [bus 00-ff]) 7,160,424069;pci_root PNP0A03:00: host bridge window [io 0x0000-0x0cf7] (ignored) SUBSYSTEM=acpi DEVICE=+acpi:PNP0A03:00 6,339,5140900;NET: Registered protocol family 10 30,340,5690716;udevd[80]: starting version 181 6,341,6081421;FDC 0 is a S82078B 6,345,6154686;microcode: CPU0 sig=0x623, pf=0x0, revision=0x0 7,346,6156968;sr 1:0:0:0: Attached scsi CD-ROM sr0 SUBSYSTEM=scsi DEVICE=+scsi:1:0:0:0 6,347,6289375;microcode: CPU1 sig=0x623, pf=0x0, revision=0x0 Change-Id: I2334e6a445b774696bec0f008ef0464311648162 Cc: Karel Zak Tested-by: William Douglas Signed-off-by: Kay Sievers Signed-off-by: Greg Kroah-Hartman Git-commit: e11fea92e13fb91c50bacca799a6131c81929986 Git-repo: https://android.googlesource.com/kernel/common/ [schikk@codeaurora.org: Resolved merge conflicts ] Signed-off-by: Swetha Chikkaboraiah --- drivers/char/mem.c | 61 +------- include/linux/printk.h | 2 + kernel/printk.c | 312 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 315 insertions(+), 60 deletions(-) diff --git a/drivers/char/mem.c b/drivers/char/mem.c index 48d794084f9..e2396937568 100644 --- a/drivers/char/mem.c +++ b/drivers/char/mem.c @@ -822,65 +822,6 @@ static const struct file_operations oldmem_fops = { }; #endif -static ssize_t kmsg_writev(struct kiocb *iocb, const struct iovec *iv, - unsigned long count, loff_t pos) -{ - char *buf, *line; - int i; - int level = default_message_loglevel; - int facility = 1; /* LOG_USER */ - size_t len = iov_length(iv, count); - ssize_t ret = len; - - if (len > 1024) - return -EINVAL; - buf = kmalloc(len+1, GFP_KERNEL); - if (buf == NULL) - return -ENOMEM; - - line = buf; - for (i = 0; i < count; i++) { - if (copy_from_user(line, iv[i].iov_base, iv[i].iov_len)) - goto out; - line += iv[i].iov_len; - } - - /* - * Extract and skip the syslog prefix <[0-9]*>. Coming from userspace - * the decimal value represents 32bit, the lower 3 bit are the log - * level, the rest are the log facility. - * - * If no prefix or no userspace facility is specified, we - * enforce LOG_USER, to be able to reliably distinguish - * kernel-generated messages from userspace-injected ones. - */ - line = buf; - if (line[0] == '<') { - char *endp = NULL; - - i = simple_strtoul(line+1, &endp, 10); - if (endp && endp[0] == '>') { - level = i & 7; - if (i >> 3) - facility = i >> 3; - endp++; - len -= endp - line; - line = endp; - } - } - line[len] = '\0'; - - printk_emit(facility, level, NULL, 0, "%s", line); -out: - kfree(buf); - return ret; -} - -static const struct file_operations kmsg_fops = { - .aio_write = kmsg_writev, - .llseek = noop_llseek, -}; - static const struct memdev { const char *name; umode_t mode; @@ -901,7 +842,7 @@ static const struct memdev { [7] = { "full", 0666, &full_fops, NULL }, [8] = { "random", 0666, &random_fops, NULL }, [9] = { "urandom", 0666, &urandom_fops, NULL }, - [11] = { "kmsg", 0, &kmsg_fops, NULL }, + [11] = { "kmsg", 0644, &kmsg_fops, NULL }, #ifdef CONFIG_CRASH_DUMP [12] = { "oldmem", 0, &oldmem_fops, NULL }, #endif diff --git a/include/linux/printk.h b/include/linux/printk.h index 2442de11cd1..dd4627f7d6d 100644 --- a/include/linux/printk.h +++ b/include/linux/printk.h @@ -200,6 +200,8 @@ extern void dump_stack(void) __cold; no_printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__) #endif +extern const struct file_operations kmsg_fops; + /* If you are writing a driver, please use dev_dbg instead */ #if defined(CONFIG_DYNAMIC_DEBUG) /* dynamic_pr_debug() uses pr_fmt() internally so we don't need it here */ diff --git a/kernel/printk.c b/kernel/printk.c index 11cee7f386e..6d994aa0505 100644 --- a/kernel/printk.c +++ b/kernel/printk.c @@ -41,6 +41,7 @@ #include #include #include +#include #include @@ -151,6 +152,48 @@ static int console_may_schedule; * length of the message text is stored in the header, the stored message * is not terminated. * + * Optionally, a message can carry a dictionary of properties (key/value pairs), + * to provide userspace with a machine-readable message context. + * + * Examples for well-defined, commonly used property names are: + * DEVICE=b12:8 device identifier + * b12:8 block dev_t + * c127:3 char dev_t + * n8 netdev ifindex + * +sound:card0 subsystem:devname + * SUBSYSTEM=pci driver-core subsystem name + * + * Valid characters in property names are [a-zA-Z0-9.-_]. The plain text value + * follows directly after a '=' character. Every property is terminated by + * a '\0' character. The last property is not terminated. + * + * Example of a message structure: + * 0000 ff 8f 00 00 00 00 00 00 monotonic time in nsec + * 0008 34 00 record is 52 bytes long + * 000a 0b 00 text is 11 bytes long + * 000c 1f 00 dictionary is 23 bytes long + * 000e 03 00 LOG_KERN (facility) LOG_ERR (level) + * 0010 69 74 27 73 20 61 20 6c "it's a l" + * 69 6e 65 "ine" + * 001b 44 45 56 49 43 "DEVIC" + * 45 3d 62 38 3a 32 00 44 "E=b8:2\0D" + * 52 49 56 45 52 3d 62 75 "RIVER=bu" + * 67 "g" + * 0032 00 00 00 padding to next message header + * + * The 'struct log' buffer header must never be directly exported to + * userspace, it is a kernel-private implementation detail that might + * need to be changed in the future, when the requirements change. + * + * /dev/kmsg exports the structured data in the following line format: + * "level,sequnum,timestamp;\n" + * + * The optional key/value pairs are attached as continuation lines starting + * with a space character and terminated by a newline. All possible + * non-prinatable characters are escaped in the "\xff" notation. + * + * Users of the export format should ignore possible additional values + * separated by ',', and find the message after the ';' character. */ struct log { @@ -299,6 +342,275 @@ static void log_store(int facility, int level, log_next_seq++; } +/* /dev/kmsg - userspace message inject/listen interface */ +struct devkmsg_user { + u64 seq; + u32 idx; + struct mutex lock; + char buf[8192]; +}; + +static ssize_t devkmsg_writev(struct kiocb *iocb, const struct iovec *iv, + unsigned long count, loff_t pos) +{ + char *buf, *line; + int i; + int level = default_message_loglevel; + int facility = 1; /* LOG_USER */ + size_t len = iov_length(iv, count); + ssize_t ret = len; + + if (len > LOG_LINE_MAX) + return -EINVAL; + buf = kmalloc(len+1, GFP_KERNEL); + if (buf == NULL) + return -ENOMEM; + + line = buf; + for (i = 0; i < count; i++) { + if (copy_from_user(line, iv[i].iov_base, iv[i].iov_len)) + goto out; + line += iv[i].iov_len; + } + /* + * Extract and skip the syslog prefix <[0-9]*>. Coming from userspace + * the decimal value represents 32bit, the lower 3 bit are the log + level, the rest are the log facility. + * + * If no prefix or no userspace facility is specified, we + * enforce LOG_USER, to be able to reliably distinguish + * kernel-generated messages from userspace-injected ones. + */ + line = buf; + if (line[0] == '<') { + char *endp = NULL; + + i = simple_strtoul(line+1, &endp, 10); + if (endp && endp[0] == '>') { + level = i & 7; + if (i >> 3) + facility = i >> 3; + endp++; + len -= endp - line; + line = endp; + } + } + line[len] = '\0'; + + printk_emit(facility, level, NULL, 0, "%s", line); +out: + kfree(buf); + return ret; +} + +static ssize_t devkmsg_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct devkmsg_user *user = file->private_data; + struct log *msg; + size_t i; + size_t len; + ssize_t ret; + + if (!user) + return -EBADF; + + mutex_lock(&user->lock); + raw_spin_lock(&logbuf_lock); + while (user->seq == log_next_seq) { + if (file->f_flags & O_NONBLOCK) { + ret = -EAGAIN; + raw_spin_unlock(&logbuf_lock); + goto out; + } + + raw_spin_unlock(&logbuf_lock); + ret = wait_event_interruptible(log_wait, + user->seq != log_next_seq); + if (ret) + goto out; + raw_spin_lock(&logbuf_lock); + } + + if (user->seq < log_first_seq) { + /* our last seen message is gone, return error and reset */ + user->idx = log_first_idx; + user->seq = log_first_seq; + ret = -EPIPE; + raw_spin_unlock(&logbuf_lock); + goto out; + } + + msg = log_from_idx(user->idx); + len = sprintf(user->buf, "%u,%llu,%llu;", + msg->level, user->seq, msg->ts_nsec / 1000); + + /* escape non-printable characters */ + for (i = 0; i < msg->text_len; i++) { + char c = log_text(msg)[i]; + + if (c < ' ' || c >= 128) + len += sprintf(user->buf + len, "\\x%02x", c); + else + user->buf[len++] = c; + } + user->buf[len++] = '\n'; + if (msg->dict_len) { + bool line = true; + + for (i = 0; i < msg->dict_len; i++) { + char c = log_dict(msg)[i]; + + if (line) { + user->buf[len++] = ' '; + line = false; + } + + if (c == '\0') { + user->buf[len++] = '\n'; + line = true; + continue; + } + + if (c < ' ' || c >= 128) { + len += sprintf(user->buf + len, "\\x%02x", c); + continue; + } + + user->buf[len++] = c; + } + user->buf[len++] = '\n'; + } + + user->idx = log_next(user->idx); + user->seq++; + raw_spin_unlock(&logbuf_lock); + + if (len > count) { + ret = -EINVAL; + goto out; + } + + if (copy_to_user(buf, user->buf, len)) { + ret = -EFAULT; + goto out; + } + ret = len; +out: + mutex_unlock(&user->lock); + return ret; +} + +static loff_t devkmsg_llseek(struct file *file, loff_t offset, int whence) +{ + struct devkmsg_user *user = file->private_data; + loff_t ret = 0; + + if (!user) + return -EBADF; + if (offset) + return -ESPIPE; + + raw_spin_lock(&logbuf_lock); + switch (whence) { + case SEEK_SET: + /* the first record */ + user->idx = log_first_idx; + user->seq = log_first_seq; + break; + case SEEK_DATA: + /* + * The first record after the last SYSLOG_ACTION_CLEAR, + * like issued by 'dmesg -c'. Reading /dev/kmsg itself + * changes no global state, and does not clear anything. + */ + user->idx = clear_idx; + user->seq = clear_seq; + break; + case SEEK_END: + /* after the last record */ + user->idx = log_next_idx; + user->seq = log_next_seq; + break; + default: + ret = -EINVAL; + } + raw_spin_unlock(&logbuf_lock); + return ret; +} + +static unsigned int devkmsg_poll(struct file *file, poll_table *wait) +{ + struct devkmsg_user *user = file->private_data; + int ret = 0; + + if (!user) + return POLLERR|POLLNVAL; + + poll_wait(file, &log_wait, wait); + + raw_spin_lock(&logbuf_lock); + if (user->seq < log_next_seq) { + /* return error when data has vanished underneath us */ + if (user->seq < log_first_seq) + ret = POLLIN|POLLRDNORM|POLLERR|POLLPRI; + ret = POLLIN|POLLRDNORM; + } + raw_spin_unlock(&logbuf_lock); + + return ret; +} + +static int devkmsg_open(struct inode *inode, struct file *file) +{ + struct devkmsg_user *user; + int err; + + /* write-only does not need any file context */ + if ((file->f_flags & O_ACCMODE) == O_WRONLY) + return 0; + + err = security_syslog(SYSLOG_ACTION_READ_ALL); + if (err) + return err; + + user = kmalloc(sizeof(struct devkmsg_user), GFP_KERNEL); + if (!user) + return -ENOMEM; + + mutex_init(&user->lock); + + raw_spin_lock(&logbuf_lock); + user->idx = log_first_idx; + user->seq = log_first_seq; + raw_spin_unlock(&logbuf_lock); + + file->private_data = user; + return 0; +} + +static int devkmsg_release(struct inode *inode, struct file *file) +{ + struct devkmsg_user *user = file->private_data; + + if (!user) + return 0; + + mutex_destroy(&user->lock); + kfree(user); + return 0; +} + +const struct file_operations kmsg_fops = { + .open = devkmsg_open, + .read = devkmsg_read, + .aio_write = devkmsg_writev, + .llseek = devkmsg_llseek, + .poll = devkmsg_poll, + .release = devkmsg_release, +}; + + #ifdef CONFIG_KEXEC /* * This appends the listed symbols to /proc/vmcoreinfo From 162ca0016f70a5fd50026f3edb59167f6b71c014 Mon Sep 17 00:00:00 2001 From: Kay Sievers Date: Tue, 15 Sep 2015 16:02:20 +0530 Subject: [PATCH 006/168] kmsg: use do_div() to divide 64bit integer On Tue, May 8, 2012 at 10:02 AM, Stephen Rothwell wrote: > kernel/built-in.o: In function `devkmsg_read': > printk.c:(.text+0x27e8): undefined reference to `__udivdi3' > Most probably the "msg->ts_nsec / 1000" since > ts_nsec is a u64 and this is a 32 bit build ... Change-Id: Ia0469fb1d7e8dfa1b60ba13761c2f4c48907a98d Reported-by: Stephen Rothwell Signed-off-by: Kay Sievers Signed-off-by: Greg Kroah-Hartman Git-commit: 5fc3249068c1ed87c6fd485f42ced24132405629 Git-repo: https://android.googlesource.com/kernel/common/ [schikk@codeaurora.org: Resolved merge conflicts ] Signed-off-by: Swetha Chikkaboraiah --- kernel/printk.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kernel/printk.c b/kernel/printk.c index 6d994aa0505..5e678458709 100644 --- a/kernel/printk.c +++ b/kernel/printk.c @@ -408,6 +408,7 @@ static ssize_t devkmsg_read(struct file *file, char __user *buf, { struct devkmsg_user *user = file->private_data; struct log *msg; + u64 ts_usec; size_t i; size_t len; ssize_t ret; @@ -442,8 +443,10 @@ static ssize_t devkmsg_read(struct file *file, char __user *buf, } msg = log_from_idx(user->idx); + ts_usec = msg->ts_nsec; + do_div(ts_usec, 1000); len = sprintf(user->buf, "%u,%llu,%llu;", - msg->level, user->seq, msg->ts_nsec / 1000); + msg->level, user->seq, ts_usec); /* escape non-printable characters */ for (i = 0; i < msg->text_len; i++) { From 8ec811307bb1e69f09760be40c2504f8745f7325 Mon Sep 17 00:00:00 2001 From: Alan Stern Date: Thu, 24 Sep 2015 10:15:46 +0530 Subject: [PATCH 007/168] printk: fix regression in SYSLOG_ACTION_CLEAR Commit 7ff9554bb578ba02166071d2d487b7fc7d860d62 (printk: convert byte-buffer to variable-length record buffer) introduced a regression by accidentally removing a "break" statement from inside the big switch in printk's do_syslog(). The symptom of this bug is that the "dmesg -C" command doesn't only clear the kernel's log buffer; it also disables console logging. This patch (as1561) fixes the regression by adding the missing "break". Change-Id: Id16704a3ca70355454c7413221da83f81f907da7 Signed-off-by: Alan Stern CC: Kay Sievers Signed-off-by: Greg Kroah-Hartman Git-commit: 4661e3568a7d14a93d4e428d246cdb86f4bac6e7 Git-repo: https://android.googlesource.com/kernel/common/ Signed-off-by: Swetha Chikkaboraiah --- kernel/printk.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/printk.c b/kernel/printk.c index 5e678458709..8f94984a136 100644 --- a/kernel/printk.c +++ b/kernel/printk.c @@ -992,6 +992,7 @@ int do_syslog(int type, char __user *buf, int len, bool from_file) /* Clear ring buffer */ case SYSLOG_ACTION_CLEAR: syslog_print_all(NULL, 0, true); + break; /* Disable logging to console */ case SYSLOG_ACTION_CONSOLE_OFF: if (saved_console_loglevel == -1) From 432bc4d3680212d984f5d5455470de5d8f667e9a Mon Sep 17 00:00:00 2001 From: Jane Zhou Date: Mon, 24 Nov 2014 11:44:08 -0800 Subject: [PATCH 008/168] net/ping: handle protocol mismatching scenario ping_lookup() may return a wrong sock if sk_buff's and sock's protocols dont' match. For example, sk_buff's protocol is ETH_P_IPV6, but sock's sk_family is AF_INET, in that case, if sk->sk_bound_dev_if is zero, a wrong sock will be returned. the fix is to "continue" the searching, if no matching, return NULL. Change-Id: I0edf397538e67657ea64917705ebff828c7aea4f Cc: "David S. Miller" Cc: Alexey Kuznetsov Cc: James Morris Cc: Hideaki YOSHIFUJI Cc: Patrick McHardy Cc: netdev@vger.kernel.org Cc: stable@vger.kernel.org Signed-off-by: Jane Zhou Signed-off-by: Yiwei Zhao Signed-off-by: David S. Miller Git-commit: 91a0b603469069cdcce4d572b7525ffc9fd352a6 Git-repo: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git Signed-off-by: Trilok Soni Signed-off-by: Srinivasarao P --- net/ipv4/ping.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/net/ipv4/ping.c b/net/ipv4/ping.c index fac0668da37..a81c33b221d 100644 --- a/net/ipv4/ping.c +++ b/net/ipv4/ping.c @@ -214,6 +214,8 @@ static struct sock *ping_lookup(struct net *net, struct sk_buff *skb, u16 ident) &ipv6_hdr(skb)->daddr)) continue; #endif + } else { + continue; } if (sk->sk_bound_dev_if && sk->sk_bound_dev_if != dif) From fd42b74e054ea7467446bee6c010627d62991725 Mon Sep 17 00:00:00 2001 From: Lorenzo Colitti Date: Tue, 3 Mar 2015 23:16:16 +0900 Subject: [PATCH 009/168] net: ping: Return EAFNOSUPPORT when appropriate. 1. For an IPv4 ping socket, ping_check_bind_addr does not check the family of the socket address that's passed in. Instead, make it behave like inet_bind, which enforces either that the address family is AF_INET, or that the family is AF_UNSPEC and the address is 0.0.0.0. 2. For an IPv6 ping socket, ping_check_bind_addr returns EINVAL if the socket family is not AF_INET6. Return EAFNOSUPPORT instead, for consistency with inet6_bind. 3. Make ping_v4_sendmsg and ping_v6_sendmsg return EAFNOSUPPORT instead of EINVAL if an incorrect socket address structure is passed in. 4. Make IPv6 ping sockets be IPv6-only. The code does not support IPv4, and it cannot easily be made to support IPv4 because the protocol numbers for ICMP and ICMPv6 are different. This makes connect(::ffff:192.0.2.1) fail with EAFNOSUPPORT instead of making the socket unusable Among other things, this fixes an oops that can be triggered by: int s = socket(AF_INET, SOCK_DGRAM, IPPROTO_ICMP); struct sockaddr_in6 sin6 = { .sin6_family = AF_INET6, .sin6_addr = in6addr_any, }; bind(s, (struct sockaddr *) &sin6, sizeof(sin6)); [backport of net 9145736d4862145684009d6a72a6e61324a9439e] Change-Id: If06ca86d9f1e4593c0d6df174caca3487c57a241 Signed-off-by: Lorenzo Colitti Signed-off-by: David S. Miller Git-commit: 428e6d675d83323d05649d901cc0fa0069f8e825 Git-repo: https://android.googlesource.com/kernel/common.git Signed-off-by: Ian Maund Signed-off-by: Srinivasarao P --- net/ipv4/ping.c | 13 ++++++++++++- net/ipv6/ping.c | 5 +++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/net/ipv4/ping.c b/net/ipv4/ping.c index a81c33b221d..1613a8159bc 100644 --- a/net/ipv4/ping.c +++ b/net/ipv4/ping.c @@ -255,6 +255,9 @@ int ping_init_sock(struct sock *sk) int i, j, count ; int ret = 0; + if (sk->sk_family == AF_INET6) + inet6_sk(sk)->ipv6only = 1; + inet_get_ping_group_range_net(net, range, range+1); if (range[0] <= group && group <= range[1]) return 0; @@ -302,6 +305,11 @@ int ping_check_bind_addr(struct sock *sk, struct inet_sock *isk, if (addr_len < sizeof(*addr)) return -EINVAL; + if (addr->sin_family != AF_INET && + !(addr->sin_family == AF_UNSPEC && + addr->sin_addr.s_addr == htonl(INADDR_ANY))) + return -EAFNOSUPPORT; + pr_debug("ping_check_bind_addr(sk=%p,addr=%pI4,port=%d)\n", sk, &addr->sin_addr.s_addr, ntohs(addr->sin_port)); @@ -326,6 +334,9 @@ int ping_check_bind_addr(struct sock *sk, struct inet_sock *isk, if (addr_len < sizeof(*addr)) return -EINVAL; + if (addr->sin6_family != AF_INET6) + return -EAFNOSUPPORT; + pr_debug("ping_check_bind_addr(sk=%p,addr=%pI6c,port=%d)\n", sk, addr->sin6_addr.s6_addr, ntohs(addr->sin6_port)); @@ -712,7 +723,7 @@ int ping_v4_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, if (msg->msg_namelen < sizeof(*usin)) return -EINVAL; if (usin->sin_family != AF_INET) - return -EINVAL; + return -EAFNOSUPPORT; daddr = usin->sin_addr.s_addr; /* no remote port */ } else { diff --git a/net/ipv6/ping.c b/net/ipv6/ping.c index 75599664f13..89d20ecd843 100644 --- a/net/ipv6/ping.c +++ b/net/ipv6/ping.c @@ -127,9 +127,10 @@ int ping_v6_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, if (msg->msg_name) { struct sockaddr_in6 *u = (struct sockaddr_in6 *) msg->msg_name; - if (msg->msg_namelen < sizeof(struct sockaddr_in6) || - u->sin6_family != AF_INET6) { + if (msg->msg_namelen < sizeof(*u)) return -EINVAL; + if (u->sin6_family != AF_INET6) { + return -EAFNOSUPPORT; } if (sk->sk_bound_dev_if && sk->sk_bound_dev_if != u->sin6_scope_id) { From c177c662df8565396ebf655d72d2b9b2d3e39c60 Mon Sep 17 00:00:00 2001 From: Richard Weinberger Date: Fri, 19 Jul 2013 11:31:36 +0200 Subject: [PATCH 010/168] um: siginfo cleanup Currently we use both struct siginfo and siginfo_t. Let's use struct siginfo internally to avoid ongoing compiler warning. We are allowed to do so because struct siginfo and siginfo_t are equivalent [cherry-pick of upstream 9a8c1359571c5d5e2fbc43cf457a6486b70a70cb] Change-Id: I564775c9ed515c39ffff7d7d600a85d50291f31d Signed-off-by: Richard Weinberger Signed-off-by: Lorenzo Colitti Git-commit: dfca41467fa4570189f76e0fb9e0f911bc88ceb8 Git-repo: https://android.googlesource.com/kernel/common.git Signed-off-by: Ian Maund Signed-off-by: Srinivasarao P --- arch/um/include/shared/frame_kern.h | 6 +++--- arch/um/kernel/signal.c | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/um/include/shared/frame_kern.h b/arch/um/include/shared/frame_kern.h index 76078490c25..fe653443aa7 100644 --- a/arch/um/include/shared/frame_kern.h +++ b/arch/um/include/shared/frame_kern.h @@ -11,11 +11,11 @@ extern int setup_signal_stack_sc(unsigned long stack_top, int sig, struct k_sigaction *ka, - struct pt_regs *regs, + struct pt_regs *regs, sigset_t *mask); -extern int setup_signal_stack_si(unsigned long stack_top, int sig, +extern int setup_signal_stack_si(unsigned long stack_top, int sig, struct k_sigaction *ka, - struct pt_regs *regs, siginfo_t *info, + struct pt_regs *regs, struct siginfo *info, sigset_t *mask); #endif diff --git a/arch/um/kernel/signal.c b/arch/um/kernel/signal.c index fb12f4c5e64..4a9462061fe 100644 --- a/arch/um/kernel/signal.c +++ b/arch/um/kernel/signal.c @@ -23,7 +23,7 @@ EXPORT_SYMBOL(unblock_signals); * OK, we're invoking a handler */ static int handle_signal(struct pt_regs *regs, unsigned long signr, - struct k_sigaction *ka, siginfo_t *info, + struct k_sigaction *ka, struct siginfo *info, sigset_t *oldset) { unsigned long sp; @@ -76,7 +76,7 @@ static int handle_signal(struct pt_regs *regs, unsigned long signr, static int kern_do_signal(struct pt_regs *regs) { struct k_sigaction ka_copy; - siginfo_t info; + struct siginfo info; sigset_t *oldset; int sig, handled_sig = 0; From e5170374f1abdc09089e0f9439a2cfb277d29e28 Mon Sep 17 00:00:00 2001 From: Harout Hedeshian Date: Mon, 26 Jan 2015 15:54:18 -0700 Subject: [PATCH 011/168] net: ipv6: Add sysctl entry to disable MTU updates from RA The kernel forcefully applies MTU values received in router advertisements provided the new MTU is less than the current. This behavior is undesirable when the user space is managing the MTU. Instead a sysctl flag 'accept_ra_mtu' is introduced such that the user space can control whether or not RA provided MTU updates should be applied. The default behavior is unchanged; user space must explicitly set this flag to 0 for RA MTUs to be ignored. Change-Id: I9b2672d7c7804b6d5394516f451888d3ac8d7803 Signed-off-by: Harout Hedeshian Signed-off-by: David S. Miller Signed-off-by: Srinivasarao P --- Documentation/networking/ip-sysctl.txt | 7 +++++++ include/linux/ipv6.h | 2 ++ net/ipv6/addrconf.c | 10 ++++++++++ net/ipv6/ndisc.c | 2 +- 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/Documentation/networking/ip-sysctl.txt b/Documentation/networking/ip-sysctl.txt index 4af594604e9..e37b645f646 100644 --- a/Documentation/networking/ip-sysctl.txt +++ b/Documentation/networking/ip-sysctl.txt @@ -1117,6 +1117,13 @@ accept_ra_prefix_route - BOOLEAN Functional default: enabled +accept_ra_mtu - BOOLEAN + Apply the MTU value specified in RA option 5 (RFC4861). If + disabled, the MTU specified in the RA will be ignored. + + Functional default: enabled if accept_ra is enabled. + disabled if accept_ra is disabled. + accept_redirects - BOOLEAN Accept Redirects. diff --git a/include/linux/ipv6.h b/include/linux/ipv6.h index 07f506310ab..e5bf08e8ba1 100644 --- a/include/linux/ipv6.h +++ b/include/linux/ipv6.h @@ -174,6 +174,7 @@ struct ipv6_devconf { __s32 accept_dad; __s32 force_tllao; __s32 accept_ra_prefix_route; + __s32 accept_ra_mtu; void *sysctl; }; @@ -217,6 +218,7 @@ enum { DEVCONF_FORCE_TLLAO, DEVCONF_ACCEPT_RA_PREFIX_ROUTE, DEVCONF_ACCEPT_RA_RT_TABLE, + DEVCONF_ACCEPT_RA_MTU, DEVCONF_MAX }; diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index dffd854ca3b..e104eb454ea 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -201,6 +201,7 @@ static struct ipv6_devconf ipv6_devconf __read_mostly = { .disable_ipv6 = 0, .accept_dad = 1, .accept_ra_prefix_route = 1, + .accept_ra_mtu = 1, }; static struct ipv6_devconf ipv6_devconf_dflt __read_mostly = { @@ -237,6 +238,7 @@ static struct ipv6_devconf ipv6_devconf_dflt __read_mostly = { .disable_ipv6 = 0, .accept_dad = 1, .accept_ra_prefix_route = 1, + .accept_ra_mtu = 1, }; /* IPv6 Wildcard Address and Loopback Address defined by RFC2553 */ @@ -3962,6 +3964,7 @@ static inline void ipv6_store_devconf(struct ipv6_devconf *cnf, array[DEVCONF_DISABLE_IPV6] = cnf->disable_ipv6; array[DEVCONF_ACCEPT_DAD] = cnf->accept_dad; array[DEVCONF_FORCE_TLLAO] = cnf->force_tllao; + array[DEVCONF_ACCEPT_RA_MTU] = cnf->accept_ra_mtu; } static inline size_t inet6_ifla6_size(void) @@ -4646,6 +4649,13 @@ static struct addrconf_sysctl_table .mode = 0644, .proc_handler = proc_dointvec, }, + { + .procname = "accept_ra_mtu", + .data = &ipv6_devconf.accept_ra_mtu, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, { /* sentinel */ } diff --git a/net/ipv6/ndisc.c b/net/ipv6/ndisc.c index 176b469322a..e32c3b93783 100644 --- a/net/ipv6/ndisc.c +++ b/net/ipv6/ndisc.c @@ -1379,7 +1379,7 @@ skip_routeinfo: } } - if (ndopts.nd_opts_mtu) { + if (ndopts.nd_opts_mtu && in6_dev->cnf.accept_ra_mtu) { __be32 n; u32 mtu; From 0ec3c288f22e11776930c7c86b36ed54487d11bd Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Tue, 15 Apr 2014 16:25:34 -0700 Subject: [PATCH 012/168] ipv4, fib: pass LOOPBACK_IFINDEX instead of 0 to flowi4_iif MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As suggested by Julian: Simply, flowi4_iif must not contain 0, it does not look logical to ignore all ip rules with specified iif. because in fib_rule_match() we do: if (rule->iifindex && (rule->iifindex != fl->flowi_iif)) goto out; flowi4_iif should be LOOPBACK_IFINDEX by default . We need to move LOOPBACK_IFINDEX to include/net/flow.h: 1) It is mostly used by flowi_iif 2) Fix the following compile error if we use it in flow.h by the patches latter: In file included from include/linux/netfilter.h:277:0, from include/net/netns/netfilter.h:5, from include/net/net_namespace.h:21, from include/linux/netdevice.h:43, from include/linux/icmpv6.h:12, from include/linux/ipv6.h:61, from include/net/ipv6.h:16, from include/linux/sunrpc/clnt.h:27, from include/linux/nfs_fs.h:30, from init/do_mounts.c:32: include/net/flow.h: In function ‘flowi4_init_output’: include/net/flow.h:84:32: error: ‘LOOPBACK_IFINDEX’ undeclared (first use in this function) [Backport of net-next 6a662719c9868b3d6c7d26b3a085f0cd3cc15e64] Change-Id: Ib7a0a08d78c03800488afa1b2c170cb70e34cfd9 Cc: Eric Biederman Cc: Julian Anastasov Cc: David S. Miller Signed-off-by: Cong Wang Signed-off-by: Cong Wang Signed-off-by: David S. Miller Signed-off-by: Lorenzo Colitti Git-commit: 9c086b4cf266e9ac1afabb86ff9ef54407b344e2 Git-repo: https://android.googlesource.com/kernel/common.git Signed-off-by: Ian Maund Signed-off-by: Srinivasarao P --- include/net/flow.h | 10 +++++++++- include/net/net_namespace.h | 2 +- net/ipv4/fib_frontend.c | 2 +- net/ipv4/fib_semantics.c | 1 + net/ipv4/ipmr.c | 2 +- net/ipv4/netfilter/ipt_rpfilter.c | 5 +---- net/ipv6/ip6mr.c | 2 +- 7 files changed, 15 insertions(+), 9 deletions(-) diff --git a/include/net/flow.h b/include/net/flow.h index 3fe9261baac..db5523460a5 100644 --- a/include/net/flow.h +++ b/include/net/flow.h @@ -11,6 +11,14 @@ #include #include +/* + * ifindex generation is per-net namespace, and loopback is + * always the 1st device in ns (see net_dev_init), thus any + * loopback device should get ifindex 1 + */ + +#define LOOPBACK_IFINDEX 1 + struct flowi_common { int flowic_oif; int flowic_iif; @@ -84,7 +92,7 @@ static inline void flowi4_init_output(struct flowi4 *fl4, int oif, uid_t uid) { fl4->flowi4_oif = oif; - fl4->flowi4_iif = 0; + fl4->flowi4_iif = LOOPBACK_IFINDEX; fl4->flowi4_mark = mark; fl4->flowi4_tos = tos; fl4->flowi4_scope = scope; diff --git a/include/net/net_namespace.h b/include/net/net_namespace.h index b1cd8b6916d..2633715fac3 100644 --- a/include/net/net_namespace.h +++ b/include/net/net_namespace.h @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -105,7 +106,6 @@ struct net { struct netns_ipvs *ipvs; }; - #include /* Init's network namespace */ diff --git a/net/ipv4/fib_frontend.c b/net/ipv4/fib_frontend.c index 0a24199ff84..3cf4b01e406 100644 --- a/net/ipv4/fib_frontend.c +++ b/net/ipv4/fib_frontend.c @@ -201,7 +201,7 @@ int fib_validate_source(struct sk_buff *skb, __be32 src, __be32 dst, u8 tos, struct net *net; fl4.flowi4_oif = 0; - fl4.flowi4_iif = oif; + fl4.flowi4_iif = oif ? : LOOPBACK_IFINDEX; fl4.daddr = src; fl4.saddr = dst; fl4.flowi4_tos = tos; diff --git a/net/ipv4/fib_semantics.c b/net/ipv4/fib_semantics.c index 8861f91a07c..e990845735d 100644 --- a/net/ipv4/fib_semantics.c +++ b/net/ipv4/fib_semantics.c @@ -562,6 +562,7 @@ static int fib_check_nh(struct fib_config *cfg, struct fib_info *fi, .daddr = nh->nh_gw, .flowi4_scope = cfg->fc_scope + 1, .flowi4_oif = nh->nh_oif, + .flowi4_iif = LOOPBACK_IFINDEX, }; /* It is not necessary, but requires a bit of thinking */ diff --git a/net/ipv4/ipmr.c b/net/ipv4/ipmr.c index 960fbfc3e97..cda81dfa6ed 100644 --- a/net/ipv4/ipmr.c +++ b/net/ipv4/ipmr.c @@ -438,7 +438,7 @@ static netdev_tx_t reg_vif_xmit(struct sk_buff *skb, struct net_device *dev) struct mr_table *mrt; struct flowi4 fl4 = { .flowi4_oif = dev->ifindex, - .flowi4_iif = skb->skb_iif, + .flowi4_iif = skb->skb_iif ? : LOOPBACK_IFINDEX, .flowi4_mark = skb->mark, }; int err; diff --git a/net/ipv4/netfilter/ipt_rpfilter.c b/net/ipv4/netfilter/ipt_rpfilter.c index 31371be8174..10192e8a8f4 100644 --- a/net/ipv4/netfilter/ipt_rpfilter.c +++ b/net/ipv4/netfilter/ipt_rpfilter.c @@ -83,11 +83,8 @@ static bool rpfilter_mt(const struct sk_buff *skb, struct xt_action_param *par) if (ipv4_is_multicast(iph->daddr)) { if (ipv4_is_zeronet(iph->saddr)) return ipv4_is_local_multicast(iph->daddr) ^ invert; - flow.flowi4_iif = 0; - } else { - flow.flowi4_iif = dev_net(par->in)->loopback_dev->ifindex; } - + flow.flowi4_iif = dev_net(par->in)->loopback_dev->ifindex; flow.daddr = iph->saddr; flow.saddr = rpfilter_get_saddr(iph->daddr); flow.flowi4_oif = 0; diff --git a/net/ipv6/ip6mr.c b/net/ipv6/ip6mr.c index 8110362e0af..568ee525326 100644 --- a/net/ipv6/ip6mr.c +++ b/net/ipv6/ip6mr.c @@ -690,7 +690,7 @@ static netdev_tx_t reg_vif_xmit(struct sk_buff *skb, struct mr6_table *mrt; struct flowi6 fl6 = { .flowi6_oif = dev->ifindex, - .flowi6_iif = skb->skb_iif, + .flowi6_iif = skb->skb_iif ? : LOOPBACK_IFINDEX, .flowi6_mark = skb->mark, }; int err; From 401c6a89ca474ed870c4a0461f4f8f53ee98d7e4 Mon Sep 17 00:00:00 2001 From: Sabrina Dubroca Date: Wed, 10 Sep 2014 23:23:02 +0200 Subject: [PATCH 013/168] ipv6: clean up anycast when an interface is destroyed If we try to rmmod the driver for an interface while sockets with setsockopt(JOIN_ANYCAST) are alive, some refcounts aren't cleaned up and we get stuck on : unregister_netdevice: waiting for ens3 to become free. Usage count = 1 If we LEAVE_ANYCAST/close everything before rmmod'ing, there is no problem. We need to perform a cleanup similar to the one for multicast in addrconf_ifdown(how == 1). BUG: 18902601 Change-Id: I6d51aed5755eb5738fcba91950e7773a1c985d2e Signed-off-by: Sabrina Dubroca Acked-by: Hannes Frederic Sowa Signed-off-by: David S. Miller Git-commit: 86a47ad60de5221c3869821d3552dcd1c89199f5 Git-repo: https://android.googlesource.com/kernel/common.git Signed-off-by: Ian Maund Signed-off-by: Srinivasarao P --- include/net/addrconf.h | 1 + net/ipv6/addrconf.c | 8 +++++--- net/ipv6/anycast.c | 21 +++++++++++++++++++++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/include/net/addrconf.h b/include/net/addrconf.h index f08044178bc..93993992e50 100644 --- a/include/net/addrconf.h +++ b/include/net/addrconf.h @@ -167,6 +167,7 @@ extern int ipv6_chk_acast_addr(struct net *net, struct net_device *dev, const struct in6_addr *addr); u32 addrconf_rt_table(const struct net_device *dev, u32 default_table); +extern void ipv6_ac_destroy_dev(struct inet6_dev *idev); /* Device notifier */ diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index e104eb454ea..774e3f81daf 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -2889,11 +2889,13 @@ static int addrconf_ifdown(struct net_device *dev, int how) write_unlock_bh(&idev->lock); - /* Step 5: Discard multicast list */ - if (how) + /* Step 5: Discard anycast and multicast list */ + if (how) { + ipv6_ac_destroy_dev(idev); ipv6_mc_destroy_dev(idev); - else + } else { ipv6_mc_down(idev); + } idev->tstamp = jiffies; diff --git a/net/ipv6/anycast.c b/net/ipv6/anycast.c index db00d27ffb1..f08e0ef4c4f 100644 --- a/net/ipv6/anycast.c +++ b/net/ipv6/anycast.c @@ -338,6 +338,27 @@ static int ipv6_dev_ac_dec(struct net_device *dev, const struct in6_addr *addr) return __ipv6_dev_ac_dec(idev, addr); } +void ipv6_ac_destroy_dev(struct inet6_dev *idev) +{ + struct ifacaddr6 *aca; + + write_lock_bh(&idev->lock); + while ((aca = idev->ac_list) != NULL) { + idev->ac_list = aca->aca_next; + write_unlock_bh(&idev->lock); + + addrconf_leave_solict(idev, &aca->aca_addr); + + dst_hold(&aca->aca_rt->dst); + ip6_del_rt(aca->aca_rt); + + aca_put(aca); + + write_lock_bh(&idev->lock); + } + write_unlock_bh(&idev->lock); +} + /* * check if the interface has this anycast address * called with rcu_read_lock() From 469dbc9d909dd583a1762064f7f9f54eee6f6f01 Mon Sep 17 00:00:00 2001 From: Swetha Chikkaboraiah Date: Wed, 14 Oct 2015 17:58:17 +0530 Subject: [PATCH 014/168] defconfig: Disable CONFIG_DEVMEM and CONFIG_DEVKMEM testDevMemSane & testDevkmemSane test cases are introduced in latest Android-6.0 version of CTS. These testcases expect mem and kmem nodes should not be present in /dev. CRs-fixed: 907542 Change-Id: Icd798c885b6045e889ff720aa8655351d1376353 Signed-off-by: Swetha Chikkaboraiah --- arch/arm/configs/msm8226-perf_defconfig | 2 ++ arch/arm/configs/msm8226_defconfig | 2 ++ arch/arm/configs/msm8974-perf_defconfig | 2 ++ arch/arm/configs/msm8974_defconfig | 2 ++ 4 files changed, 8 insertions(+) diff --git a/arch/arm/configs/msm8226-perf_defconfig b/arch/arm/configs/msm8226-perf_defconfig index d98fd53c873..51ee11d2b4f 100644 --- a/arch/arm/configs/msm8226-perf_defconfig +++ b/arch/arm/configs/msm8226-perf_defconfig @@ -17,6 +17,8 @@ CONFIG_NAMESPACES=y # CONFIG_IPC_NS is not set # CONFIG_USER_NS is not set # CONFIG_PID_NS is not set +# CONFIG_DEVMEM is not set +# CONFIG_DEVKMEM is not set CONFIG_RELAY=y CONFIG_BLK_DEV_INITRD=y CONFIG_RD_BZIP2=y diff --git a/arch/arm/configs/msm8226_defconfig b/arch/arm/configs/msm8226_defconfig index 85789f7dfed..52c6ca170e5 100644 --- a/arch/arm/configs/msm8226_defconfig +++ b/arch/arm/configs/msm8226_defconfig @@ -17,6 +17,8 @@ CONFIG_NAMESPACES=y # CONFIG_IPC_NS is not set # CONFIG_USER_NS is not set # CONFIG_PID_NS is not set +# CONFIG_DEVMEM is not set +# CONFIG_DEVKMEM is not set CONFIG_RELAY=y CONFIG_BLK_DEV_INITRD=y CONFIG_RD_BZIP2=y diff --git a/arch/arm/configs/msm8974-perf_defconfig b/arch/arm/configs/msm8974-perf_defconfig index 8bc7e431897..fb9537e821e 100755 --- a/arch/arm/configs/msm8974-perf_defconfig +++ b/arch/arm/configs/msm8974-perf_defconfig @@ -18,6 +18,8 @@ CONFIG_NAMESPACES=y # CONFIG_IPC_NS is not set # CONFIG_USER_NS is not set # CONFIG_PID_NS is not set +# CONFIG_DEVMEM is not set +# CONFIG_DEVKMEM is not set CONFIG_RELAY=y CONFIG_BLK_DEV_INITRD=y CONFIG_RD_BZIP2=y diff --git a/arch/arm/configs/msm8974_defconfig b/arch/arm/configs/msm8974_defconfig index 5c2df00f2d4..b3931679c09 100755 --- a/arch/arm/configs/msm8974_defconfig +++ b/arch/arm/configs/msm8974_defconfig @@ -17,6 +17,8 @@ CONFIG_NAMESPACES=y # CONFIG_IPC_NS is not set # CONFIG_USER_NS is not set # CONFIG_PID_NS is not set +# CONFIG_DEVMEM is not set +# CONFIG_DEVKMEM is not set CONFIG_RELAY=y CONFIG_BLK_DEV_INITRD=y CONFIG_RD_BZIP2=y From 0f10d578d4c278eb3268ce66ff889cf9ac8a85e1 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 27 Oct 2015 17:44:32 +0530 Subject: [PATCH 015/168] vfs: umount_tree() might be called on subtree that had never made it __mnt_make_shortterm() in there undoes the effect of __mnt_make_longterm() we'd done back when we set ->mnt_ns non-NULL; it should not be done to vfsmounts that had never gone through commit_tree() and friends. Kudos to lczerner for catching that one... CR's-Fixed: 914867 Change-Id: I6022d463213413d3849e75f57f552b4cff0a6c57 Cc: stable@vger.kernel.org Signed-off-by: Al Viro Git-commit: 63d37a84ab6004c235314ffd7a76c5eb28c2fae0 Git-repo: https://android.googlesource.com/kernel/common/ Signed-off-by: Swetha Chikkaboraiah --- fs/namespace.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/namespace.c b/fs/namespace.c index 34d97cab0ee..be55bbbaaf9 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -1076,8 +1076,9 @@ void umount_tree(struct mount *mnt, int propagate, struct list_head *kill) list_del_init(&p->mnt_expire); list_del_init(&p->mnt_list); __touch_mnt_namespace(p->mnt_ns); + if (p->mnt_ns) + __mnt_make_shortterm(p); p->mnt_ns = NULL; - __mnt_make_shortterm(p); list_del_init(&p->mnt_child); if (mnt_has_parent(p)) { p->mnt_parent->mnt_ghosts++; From 0e702ac537243b9fcc69c876c385851ce6580263 Mon Sep 17 00:00:00 2001 From: Srinivasarao P Date: Wed, 28 Oct 2015 15:28:38 +0530 Subject: [PATCH 016/168] Failed to load kernel modules After upgrading to GCC-4.9 version, loading of kernel modules failed with error "Exec format error". GCC compiler option '-fno-pic' is added to fix this error. Change-Id: I2372e4f3813ef04557f63bfa32948c6f5a3c56bd Signed-off-by: Srinivasarao P --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0c11d690a32..f9c396670f2 100644 --- a/Makefile +++ b/Makefile @@ -377,7 +377,7 @@ KBUILD_AFLAGS_KERNEL := KBUILD_CFLAGS_KERNEL := KBUILD_AFLAGS := -D__ASSEMBLY__ KBUILD_AFLAGS_MODULE := -DMODULE -KBUILD_CFLAGS_MODULE := -DMODULE +KBUILD_CFLAGS_MODULE := -DMODULE -fno-pic KBUILD_LDFLAGS_MODULE := -T $(srctree)/scripts/module-common.lds # Read KERNELRELEASE from include/config/kernel.release (if it exists) From d560b4e3fa96b8abc170aa077e64f3ce3e9ed648 Mon Sep 17 00:00:00 2001 From: Balamurugan Alagarsamy Date: Tue, 27 Oct 2015 17:51:48 +0530 Subject: [PATCH 017/168] msm: vidc: fix null pointer crash in sys error handler When the iteration pointer of list_for_each_entry is used to evaluate, it can have a garbage value. So,instead, using a inst pointer, using core to get the hdev. Also, flushing the core work queue in unload firmware leads to deadlock, when there is a SYS error is received from core, because handle_sys_error is also a part of core work handler, so flushing itself by its own execution, will lead to wait indefinitely, i.e, a deadlock. So, skipping it when there is a error condition. Change-Id: Ic4e2df0db2341c974cbba19ddd55ecdc5c832124 Signed-off-by: Balamurugan Alagarsamy --- drivers/media/platform/msm/vidc/msm_vidc_common.c | 9 +++++++-- drivers/media/platform/msm/vidc/venus_hfi.c | 3 ++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/drivers/media/platform/msm/vidc/msm_vidc_common.c b/drivers/media/platform/msm/vidc/msm_vidc_common.c index 4ba0076fdf9..13f6bf3abb9 100644 --- a/drivers/media/platform/msm/vidc/msm_vidc_common.c +++ b/drivers/media/platform/msm/vidc/msm_vidc_common.c @@ -936,8 +936,13 @@ static void handle_sys_error(enum command_response cmd, void *data) V4L2_EVENT_MSM_VIDC_SYS_ERROR); } - if (inst->core) - hdev = inst->core->device; + hdev = core->device; + if (!hdev) { + dprintk(VIDC_ERR, "hdev in core is NULL\n"); + mutex_unlock(&core->lock); + return; + } + if (core->state == VIDC_CORE_INVALID) { dprintk(VIDC_DBG, "Calling core_release\n"); rc = call_hfi_op(hdev, core_release, diff --git a/drivers/media/platform/msm/vidc/venus_hfi.c b/drivers/media/platform/msm/vidc/venus_hfi.c index fdc61c74ffc..5e00b409d21 100644 --- a/drivers/media/platform/msm/vidc/venus_hfi.c +++ b/drivers/media/platform/msm/vidc/venus_hfi.c @@ -3761,7 +3761,8 @@ static void venus_hfi_unload_fw(void *dev) return; } if (device->resources.fw.cookie) { - flush_workqueue(device->vidc_workq); + if (device->state != VENUS_STATE_DEINIT) + flush_workqueue(device->vidc_workq); flush_workqueue(device->venus_pm_workq); subsystem_put(device->resources.fw.cookie); venus_hfi_interface_queues_release(dev); From 56d62647adca2229d4cffd55dce0544326803b82 Mon Sep 17 00:00:00 2001 From: Zhen Kong Date: Tue, 10 Mar 2015 16:29:53 -0700 Subject: [PATCH 018/168] qseecom: Change to work with appsbl qseecom Do not send app regin notification if qseecom in appsbl has already done it Change-Id: I81ae9a991a5d8ec582d9320e18be1b6f8e8ee7fd Signed-off-by: Zhen Kong --- Documentation/devicetree/bindings/qseecom/qseecom.txt | 2 ++ drivers/misc/qseecom.c | 10 +++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/qseecom/qseecom.txt b/Documentation/devicetree/bindings/qseecom/qseecom.txt index c0426a7b082..fcca2e5c834 100644 --- a/Documentation/devicetree/bindings/qseecom/qseecom.txt +++ b/Documentation/devicetree/bindings/qseecom/qseecom.txt @@ -16,6 +16,7 @@ Optional properties: - qcom,support-bus-scaling : indicates if driver support scaling the bus for crypto operation. - qcom,support-fde : indicates if driver support key managing for full disk encryption feature. - qcom,support-pfe : indicates if driver support key managing for per file encryption feature. + - qcom,appsbl-qseecom-support : indicates if there is qseecom support in appsbootloader Example: qcom,qseecom@fe806000 { @@ -32,6 +33,7 @@ Example: qcom,msm_bus,num_cases = <4>; qcom,msm_bus,active_only = <0>; qcom,msm_bus,num_paths = <1>; + qcom,appsbl-qseecom-support; qcom,msm_bus,vectors = <55 512 0 0>, <55 512 3936000000 393600000>, diff --git a/drivers/misc/qseecom.c b/drivers/misc/qseecom.c index c9bf7f8a7a4..42b725f650f 100644 --- a/drivers/misc/qseecom.c +++ b/drivers/misc/qseecom.c @@ -177,6 +177,7 @@ struct qseecom_control { struct work_struct bw_inactive_req_ws; struct cdev cdev; bool timer_running; + bool appsbl_qseecom_support; }; struct qseecom_client_handle { @@ -4444,6 +4445,12 @@ static int __devinit qseecom_probe(struct platform_device *pdev) qseecom.ce_info.qsee_ce_hw_instance); } + qseecom.appsbl_qseecom_support = + of_property_read_bool((&pdev->dev)->of_node, + "qcom,appsbl-qseecom-support"); + pr_info("qseecom.appsbl_qseecom_support = 0x%x", + qseecom.appsbl_qseecom_support); + qseecom.qsee.instance = qseecom.ce_info.qsee_ce_hw_instance; qseecom.ce_drv.instance = qseecom.ce_info.hlos_ce_hw_instance; @@ -4470,7 +4477,8 @@ static int __devinit qseecom_probe(struct platform_device *pdev) qseecom_platform_support = (struct msm_bus_scale_pdata *) msm_bus_cl_get_pdata(pdev); - if (qseecom.qsee_version >= (QSEE_VERSION_02)) { + if (qseecom.qsee_version >= (QSEE_VERSION_02) && + !qseecom.appsbl_qseecom_support) { struct resource *resource = NULL; struct qsee_apps_region_info_ireq req; struct qseecom_command_scm_resp resp; From 15f5132aac88d0473f6c36e815d7145c3561adef Mon Sep 17 00:00:00 2001 From: Balamurugan Alagarsamy Date: Tue, 27 Oct 2015 17:51:48 +0530 Subject: [PATCH 019/168] msm: vidc: fix null pointer crash in sys error handler When the iteration pointer of list_for_each_entry is used to evaluate, it can have a garbage value. So,instead, using a inst pointer, using core to get the hdev. Also, flushing the core work queue in unload firmware leads to deadlock, when there is a SYS error is received from core, because handle_sys_error is also a part of core work handler, so flushing itself by its own execution, will lead to wait indefinitely, i.e, a deadlock. So, skipping it when there is a error condition. Change-Id: Ic4e2df0db2341c974cbba19ddd55ecdc5c832124 Signed-off-by: Balamurugan Alagarsamy --- drivers/media/platform/msm/vidc/msm_vidc_common.c | 9 +++++++-- drivers/media/platform/msm/vidc/venus_hfi.c | 3 ++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/drivers/media/platform/msm/vidc/msm_vidc_common.c b/drivers/media/platform/msm/vidc/msm_vidc_common.c index 4ba0076fdf9..13f6bf3abb9 100644 --- a/drivers/media/platform/msm/vidc/msm_vidc_common.c +++ b/drivers/media/platform/msm/vidc/msm_vidc_common.c @@ -936,8 +936,13 @@ static void handle_sys_error(enum command_response cmd, void *data) V4L2_EVENT_MSM_VIDC_SYS_ERROR); } - if (inst->core) - hdev = inst->core->device; + hdev = core->device; + if (!hdev) { + dprintk(VIDC_ERR, "hdev in core is NULL\n"); + mutex_unlock(&core->lock); + return; + } + if (core->state == VIDC_CORE_INVALID) { dprintk(VIDC_DBG, "Calling core_release\n"); rc = call_hfi_op(hdev, core_release, diff --git a/drivers/media/platform/msm/vidc/venus_hfi.c b/drivers/media/platform/msm/vidc/venus_hfi.c index fdc61c74ffc..5e00b409d21 100644 --- a/drivers/media/platform/msm/vidc/venus_hfi.c +++ b/drivers/media/platform/msm/vidc/venus_hfi.c @@ -3761,7 +3761,8 @@ static void venus_hfi_unload_fw(void *dev) return; } if (device->resources.fw.cookie) { - flush_workqueue(device->vidc_workq); + if (device->state != VENUS_STATE_DEINIT) + flush_workqueue(device->vidc_workq); flush_workqueue(device->venus_pm_workq); subsystem_put(device->resources.fw.cookie); venus_hfi_interface_queues_release(dev); From 66652f969bd92e458be64cbbae5610949f676f3d Mon Sep 17 00:00:00 2001 From: Balamurugan Alagarsamy Date: Tue, 17 Nov 2015 21:33:52 +0530 Subject: [PATCH 020/168] msm: vidc: use dedicated pendingq.lock for pendingq when the release buffer reference event is received the sync_lock is acquired in handle_event_change() and if the instance state is not START_DONE then the same thread tries to acquire the sync_lock again in msm_comm_qbuf() leading to deadlock issues. So use dedicated pending queue lock instead of sync_lock for pending queue buffer list to avoid the issues due to sync_lock. CRs-Fixed: 931529 Change-Id: I1037f5e0671dfebd8fe0701b5539908daecb1b5f Signed-off-by: Balamurugan Alagarsamy --- drivers/media/platform/msm/vidc/msm_vdec.c | 25 +++++---- drivers/media/platform/msm/vidc/msm_venc.c | 23 ++++---- drivers/media/platform/msm/vidc/msm_vidc.c | 20 ++++--- .../media/platform/msm/vidc/msm_vidc_common.c | 54 +++++++++---------- .../platform/msm/vidc/msm_vidc_internal.h | 13 ++++- 5 files changed, 71 insertions(+), 64 deletions(-) diff --git a/drivers/media/platform/msm/vidc/msm_vdec.c b/drivers/media/platform/msm/vidc/msm_vdec.c index 25bee1628b4..a30aa42cdd4 100644 --- a/drivers/media/platform/msm/vidc/msm_vdec.c +++ b/drivers/media/platform/msm/vidc/msm_vdec.c @@ -1311,21 +1311,20 @@ static inline int start_streaming(struct msm_vidc_inst *inst) goto fail_start; } } - mutex_lock(&inst->sync_lock); - if (!list_empty(&inst->pendingq)) { - list_for_each_safe(ptr, next, &inst->pendingq) { - temp = list_entry(ptr, struct vb2_buf_entry, list); - rc = msm_comm_qbuf(temp->vb); - if (rc) { - dprintk(VIDC_ERR, - "Failed to qbuf to hardware\n"); - break; - } - list_del(&temp->list); - kfree(temp); + + mutex_lock(&inst->pendingq.lock); + list_for_each_safe(ptr, next, &inst->pendingq.list) { + temp = list_entry(ptr, struct vb2_buf_entry, list); + rc = msm_comm_qbuf(temp->vb); + if (rc) { + dprintk(VIDC_ERR, + "Failed to qbuf to hardware\n"); + break; } + list_del(&temp->list); + kfree(temp); } - mutex_unlock(&inst->sync_lock); + mutex_unlock(&inst->pendingq.lock); return rc; fail_start: return rc; diff --git a/drivers/media/platform/msm/vidc/msm_venc.c b/drivers/media/platform/msm/vidc/msm_venc.c index 520ad9fd6cf..814d161e052 100644 --- a/drivers/media/platform/msm/vidc/msm_venc.c +++ b/drivers/media/platform/msm/vidc/msm_venc.c @@ -1103,21 +1103,20 @@ static inline int start_streaming(struct msm_vidc_inst *inst) "Failed to move inst: %p to start done state\n", inst); goto fail_start; } - mutex_lock(&inst->sync_lock); - if (!list_empty(&inst->pendingq)) { - list_for_each_safe(ptr, next, &inst->pendingq) { - temp = list_entry(ptr, struct vb2_buf_entry, list); - rc = msm_comm_qbuf(temp->vb); - if (rc) { - dprintk(VIDC_ERR, + + mutex_lock(&inst->pendingq.lock); + list_for_each_safe(ptr, next, &inst->pendingq.list) { + temp = list_entry(ptr, struct vb2_buf_entry, list); + rc = msm_comm_qbuf(temp->vb); + if (rc) { + dprintk(VIDC_ERR, "Failed to qbuf to hardware\n"); - break; - } - list_del(&temp->list); - kfree(temp); + break; } + list_del(&temp->list); + kfree(temp); } - mutex_unlock(&inst->sync_lock); + mutex_unlock(&inst->pendingq.lock); return rc; fail_start: return rc; diff --git a/drivers/media/platform/msm/vidc/msm_vidc.c b/drivers/media/platform/msm/vidc/msm_vidc.c index bbbfe3eab52..0a4a4660530 100644 --- a/drivers/media/platform/msm/vidc/msm_vidc.c +++ b/drivers/media/platform/msm/vidc/msm_vidc.c @@ -1223,7 +1223,7 @@ void *msm_vidc_open(int core_id, int session_type) mutex_init(&inst->bufq[OUTPUT_PORT].lock); mutex_init(&inst->lock); inst->session_type = session_type; - INIT_LIST_HEAD(&inst->pendingq); + INIT_MSM_VIDC_LIST(&inst->pendingq); INIT_LIST_HEAD(&inst->internalbufs); INIT_LIST_HEAD(&inst->persistbufs); INIT_LIST_HEAD(&inst->registered_bufs); @@ -1306,18 +1306,16 @@ err_invalid_core: static void cleanup_instance(struct msm_vidc_inst *inst) { - struct list_head *ptr, *next; - struct vb2_buf_entry *entry; + struct vb2_buf_entry *entry, *dummy; if (inst) { - mutex_lock(&inst->lock); - if (!list_empty(&inst->pendingq)) { - list_for_each_safe(ptr, next, &inst->pendingq) { - entry = list_entry(ptr, struct vb2_buf_entry, - list); - list_del(&entry->list); - kfree(entry); - } + mutex_lock(&inst->pendingq.lock); + list_for_each_entry_safe(entry, dummy, &inst->pendingq.list, + list) { + list_del(&entry->list); + kfree(entry); } + mutex_unlock(&inst->pendingq.lock); + mutex_lock(&inst->lock); if (!list_empty(&inst->internalbufs)) { mutex_unlock(&inst->lock); if (msm_comm_release_scratch_buffers(inst)) diff --git a/drivers/media/platform/msm/vidc/msm_vidc_common.c b/drivers/media/platform/msm/vidc/msm_vidc_common.c index 4ba0076fdf9..84663d51c57 100644 --- a/drivers/media/platform/msm/vidc/msm_vidc_common.c +++ b/drivers/media/platform/msm/vidc/msm_vidc_common.c @@ -2536,9 +2536,9 @@ int msm_comm_qbuf(struct vb2_buffer *vb) goto err_no_mem; } entry->vb = vb; - mutex_lock(&inst->sync_lock); - list_add_tail(&entry->list, &inst->pendingq); - mutex_unlock(&inst->sync_lock); + mutex_lock(&inst->pendingq.lock); + list_add_tail(&entry->list, &inst->pendingq.list); + mutex_unlock(&inst->pendingq.lock); } else { int64_t time_usec = timeval_to_ns(&vb->v4l2_buf.timestamp); do_div(time_usec, NSEC_PER_USEC); @@ -3105,7 +3105,8 @@ void msm_comm_flush_pending_dynamic_buffers(struct msm_vidc_inst *inst) if (inst->buffer_mode_set[CAPTURE_PORT] != HAL_BUFFER_MODE_DYNAMIC) return; - if (list_empty(&inst->pendingq) || list_empty(&inst->registered_bufs)) + if (list_empty(&inst->pendingq.list) || + list_empty(&inst->registered_bufs)) return; list = &inst->registered_bufs; @@ -3175,9 +3176,9 @@ int msm_comm_flush(struct msm_vidc_inst *inst, u32 flags) return 0; } - mutex_lock(&inst->sync_lock); if (inst->in_reconfig && !ip_flush && op_flush) { - if (!list_empty(&inst->pendingq)) { + mutex_lock(&inst->pendingq.lock); + if (!list_empty(&inst->pendingq.list)) { /*Execution can never reach here since port reconfig * wont happen unless pendingq is emptied out * (both pendingq and flush being secured with same @@ -3185,6 +3186,7 @@ int msm_comm_flush(struct msm_vidc_inst *inst, u32 flags) dprintk(VIDC_WARN, "FLUSH BUG: Pending q not empty! It should be empty\n"); } + mutex_unlock(&inst->pendingq.lock); rc = call_hfi_op(hdev, session_flush, inst->session, HAL_FLUSH_OUTPUT); if (!rc && (msm_comm_get_stream_output_mode(inst) == @@ -3193,35 +3195,33 @@ int msm_comm_flush(struct msm_vidc_inst *inst, u32 flags) HAL_FLUSH_OUTPUT2); } else { - if (!list_empty(&inst->pendingq)) { - msm_comm_flush_pending_dynamic_buffers(inst); + msm_comm_flush_pending_dynamic_buffers(inst); - /*If flush is called after queueing buffers but before - * streamon driver should flush the pending queue*/ - list_for_each_safe(ptr, next, &inst->pendingq) { - temp = - list_entry(ptr, struct vb2_buf_entry, list); - if (temp->vb->v4l2_buf.type == - V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) - lock = &inst->bufq[CAPTURE_PORT].lock; - else - lock = &inst->bufq[OUTPUT_PORT].lock; - temp->vb->v4l2_planes[0].bytesused = 0; - mutex_lock(lock); - vb2_buffer_done(temp->vb, VB2_BUF_STATE_DONE); - mutex_unlock(lock); - list_del(&temp->list); - kfree(temp); - } + /*If flush is called after queueing buffers but before + * streamon driver should flush the pending queue*/ + mutex_lock(&inst->pendingq.lock); + list_for_each_safe(ptr, next, &inst->pendingq.list) { + temp = + list_entry(ptr, struct vb2_buf_entry, list); + if (temp->vb->v4l2_buf.type == + V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) + lock = &inst->bufq[CAPTURE_PORT].lock; + else + lock = &inst->bufq[OUTPUT_PORT].lock; + temp->vb->v4l2_planes[0].bytesused = 0; + mutex_lock(lock); + vb2_buffer_done(temp->vb, VB2_BUF_STATE_DONE); + mutex_unlock(lock); + list_del(&temp->list); + kfree(temp); } - + mutex_unlock(&inst->pendingq.lock); /*Do not send flush in case of session_error */ if (!(inst->state == MSM_VIDC_CORE_INVALID && core->state != VIDC_CORE_INVALID)) rc = call_hfi_op(hdev, session_flush, inst->session, HAL_FLUSH_ALL); } - mutex_unlock(&inst->sync_lock); return rc; } diff --git a/drivers/media/platform/msm/vidc/msm_vidc_internal.h b/drivers/media/platform/msm/vidc/msm_vidc_internal.h index 58f5f2f7c3e..9968e77b302 100644 --- a/drivers/media/platform/msm/vidc/msm_vidc_internal.h +++ b/drivers/media/platform/msm/vidc/msm_vidc_internal.h @@ -101,6 +101,17 @@ struct buf_info { struct vb2_buffer *buf; }; +struct msm_vidc_list { + struct list_head list; + struct mutex lock; +}; + +static inline void INIT_MSM_VIDC_LIST(struct msm_vidc_list *mlist) +{ + mutex_init(&mlist->lock); + INIT_LIST_HEAD(&mlist->list); +} + enum buffer_owner { DRIVER, FIRMWARE, @@ -227,7 +238,7 @@ struct msm_vidc_inst { int state; struct msm_vidc_format *fmts[MAX_PORT_NUM]; struct buf_queue bufq[MAX_PORT_NUM]; - struct list_head pendingq; + struct msm_vidc_list pendingq; struct list_head internalbufs; struct list_head persistbufs; struct list_head outputbufs; From 499ff6accb90ac30147f61fea2c64a869ff038da Mon Sep 17 00:00:00 2001 From: Katish Paran Date: Tue, 17 Nov 2015 17:36:52 +0530 Subject: [PATCH 021/168] diag: Cleanup client information on deinit Ioctl Clean up the client pid information from the client table on lsm_deinit Ioctl call. Also mutex protect the deinitialization steps for any diag client. Change-Id: I13410b0b1382de7b026726a3736f397dee1092fa Signed-off-by: Sreelakshmi Gownipalli Signed-off-by: Katish Paran --- drivers/char/diag/diagchar.h | 3 ++- drivers/char/diag/diagchar_core.c | 37 ++++++++++++++++++++++++------- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/drivers/char/diag/diagchar.h b/drivers/char/diag/diagchar.h index 0141a632e97..c148fe175f5 100644 --- a/drivers/char/diag/diagchar.h +++ b/drivers/char/diag/diagchar.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2008-2014, The Linux Foundation. All rights reserved. +/* Copyright (c) 2008-2015, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -312,6 +312,7 @@ struct diagchar_dev { struct device *diag_dev; int ref_count; struct mutex diagchar_mutex; + struct mutex diag_file_mutex; wait_queue_head_t wait_q; wait_queue_head_t smd_wait_q; struct diag_client_map *client_map; diff --git a/drivers/char/diag/diagchar_core.c b/drivers/char/diag/diagchar_core.c index 7ef47686e66..ea39b5dbe5c 100644 --- a/drivers/char/diag/diagchar_core.c +++ b/drivers/char/diag/diagchar_core.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2008-2014, The Linux Foundation. All rights reserved. +/* Copyright (c) 2008-2015, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -261,20 +261,30 @@ fail: return -ENOMEM; } -static int diagchar_close(struct inode *inode, struct file *file) +static int diag_remove_client_entry(struct file *file) { int i = -1; - struct diagchar_priv *diagpriv_data = file->private_data; + struct diagchar_priv *diagpriv_data = NULL; pr_debug("diag: process exit %s\n", current->comm); - if (!(file->private_data)) { - pr_alert("diag: Invalid file pointer"); - return -ENOMEM; - } if (!driver) return -ENOMEM; + mutex_lock(&driver->diag_file_mutex); + if (!file) { + pr_debug("diag: Invalid file pointer\n"); + mutex_unlock(&driver->diag_file_mutex); + return -ENOENT; + } + if (!(file->private_data)) { + pr_alert("diag: Invalid private data"); + mutex_unlock(&driver->diag_file_mutex); + return -ENOMEM; + } + + diagpriv_data = file->private_data; + /* clean up any DCI registrations, if this is a DCI client * This will specially help in case of ungraceful exit of any DCI client * This call will remove any pending registrations of such client @@ -329,13 +339,21 @@ static int diagchar_close(struct inode *inode, struct file *file) driver->client_map[i].pid = 0; kfree(diagpriv_data); diagpriv_data = NULL; + file->private_data = 0; break; } } mutex_unlock(&driver->diagchar_mutex); + mutex_unlock(&driver->diag_file_mutex); return 0; } +static int diagchar_close(struct inode *inode, struct file *file) +{ + pr_debug("diag: process exit %s\n", current->comm); + return diag_remove_client_entry(file); +} + int diag_find_polling_reg(int i) { uint16_t subsys_id, cmd_code_lo, cmd_code_hi; @@ -1327,7 +1345,9 @@ drop: data_type = driver->data_ready[index] & DEINIT_TYPE; COPY_USER_SPACE_OR_EXIT(buf, data_type, 4); driver->data_ready[index] ^= DEINIT_TYPE; - goto exit; + mutex_unlock(&driver->diagchar_mutex); + diag_remove_client_entry(file); + return ret; } if (driver->data_ready[index] & MSG_MASKS_TYPE) { @@ -2161,6 +2181,7 @@ static int __init diagchar_init(void) driver->in_busy_pktdata = 0; driver->in_busy_dcipktdata = 0; mutex_init(&driver->diagchar_mutex); + mutex_init(&driver->diag_file_mutex); init_waitqueue_head(&driver->wait_q); init_waitqueue_head(&driver->smd_wait_q); INIT_WORK(&(driver->diag_drain_work), diag_drain_work_fn); From e122d7afbe2d401e4b64db695faa2666148daab6 Mon Sep 17 00:00:00 2001 From: Sachin Shah Date: Tue, 10 Jun 2014 19:16:35 -0700 Subject: [PATCH 022/168] msm: vidc: Fix timestamp for EOS buffer Timestamp for zero filled length FBD is not being set correctly even if the timestamp from FW is valid. This causes certain decode usecase to not exit correctly. CRs-fixed: 664086 Change-Id: Ieb0f274efa5d94c9294b021e061bca6d9b45de87 Signed-off-by: Sachin Shah --- drivers/media/platform/msm/vidc/msm_vidc_common.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/media/platform/msm/vidc/msm_vidc_common.c b/drivers/media/platform/msm/vidc/msm_vidc_common.c index 3065eddf969..dc925d82b1e 100644 --- a/drivers/media/platform/msm/vidc/msm_vidc_common.c +++ b/drivers/media/platform/msm/vidc/msm_vidc_common.c @@ -1293,9 +1293,17 @@ static void handle_fbd(enum command_response cmd, void *data) time_usec = fill_buf_done->timestamp_hi; time_usec = (time_usec << 32) | fill_buf_done->timestamp_lo; - vb->v4l2_buf.timestamp = - ns_to_timeval(time_usec * NSEC_PER_USEC); + } else { + time_usec = 0; + dprintk(VIDC_DBG, + "Set zero timestamp for buffer 0x%pa, filled: %d, (hi:%u, lo:%u)\n", + &fill_buf_done->packet_buffer1, + fill_buf_done->filled_len1, + fill_buf_done->timestamp_hi, + fill_buf_done->timestamp_lo); } + vb->v4l2_buf.timestamp = + ns_to_timeval(time_usec * NSEC_PER_USEC); vb->v4l2_buf.flags = 0; extra_idx = EXTRADATA_IDX(inst->fmts[CAPTURE_PORT]->num_planes); From 38dfdc1a74ce4313f48cc2491f2d43099b0b9ad4 Mon Sep 17 00:00:00 2001 From: Ruchi Kandoi Date: Fri, 14 Mar 2014 18:27:20 -0700 Subject: [PATCH 023/168] cpufreq: Persist cpufreq time in state data across hotplug Cpufreq time_in_state data for all CPUs is made persistent across hotplug and exposed to userspace via sysfs file /sys/devices/system/cpu/cpufreq/all_time_in_state Change-Id: I97cb5de24b6de16189bf8b5df9592d0a6e6ddf32 Signed-off-by: Ruchi Kandoi Git-commit: 3de804108a1fa1ba9d2095045e2f8e9a4823332b Git-repo: https://android.googlesource.com/kernel/msm/ Signed-off-by: Srinivasarao P --- drivers/cpufreq/cpufreq_stats.c | 206 +++++++++++++++++++++++++++++++- 1 file changed, 205 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/cpufreq_stats.c b/drivers/cpufreq/cpufreq_stats.c index 1a9a6a5eb57..ea7cddbf5d3 100644 --- a/drivers/cpufreq/cpufreq_stats.c +++ b/drivers/cpufreq/cpufreq_stats.c @@ -20,6 +20,8 @@ #include #include #include +#include +#include #include static spinlock_t cpufreq_stats_lock; @@ -44,6 +46,20 @@ struct cpufreq_stats { #endif }; +struct all_cpufreq_stats { + unsigned int state_num; + cputime64_t *time_in_state; + unsigned int *freq_table; +}; + +struct all_freq_table { + unsigned int *freq_table; + unsigned int table_size; +}; + +static struct all_freq_table *all_freq_table; + +static DEFINE_PER_CPU(struct all_cpufreq_stats *, all_cpufreq_stats); static DEFINE_PER_CPU(struct cpufreq_stats *, cpufreq_stats_table); struct cpufreq_stats_attribute { @@ -54,14 +70,20 @@ struct cpufreq_stats_attribute { static int cpufreq_stats_update(unsigned int cpu) { struct cpufreq_stats *stat; + struct all_cpufreq_stats *all_stat; unsigned long long cur_time; cur_time = get_jiffies_64(); spin_lock(&cpufreq_stats_lock); stat = per_cpu(cpufreq_stats_table, cpu); - if (stat->time_in_state) + all_stat = per_cpu(all_cpufreq_stats, cpu); + if (stat->time_in_state) { stat->time_in_state[stat->last_index] += cur_time - stat->last_time; + if (all_stat) + all_stat->time_in_state[stat->last_index] += + cur_time - stat->last_time; + } stat->last_time = cur_time; spin_unlock(&cpufreq_stats_lock); return 0; @@ -92,6 +114,56 @@ static ssize_t show_time_in_state(struct cpufreq_policy *policy, char *buf) return len; } +static int get_index_all_cpufreq_stat(struct all_cpufreq_stats *all_stat, + unsigned int freq) +{ + int i; + if (!all_stat) + return -1; + for (i = 0; i < all_stat->state_num; i++) { + if (all_stat->freq_table[i] == freq) + return i; + } + return -1; +} + +static ssize_t show_all_time_in_state(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + ssize_t len = 0; + unsigned int i, cpu, freq, index; + struct all_cpufreq_stats *all_stat; + + len += scnprintf(buf + len, PAGE_SIZE - len, "freq\t\t"); + for_each_possible_cpu(cpu) { + len += scnprintf(buf + len, PAGE_SIZE - len, "cpu%d\t\t", cpu); + if (cpu_online(cpu)) + cpufreq_stats_update(cpu); + } + + if (!all_freq_table) + goto out; + for (i = 0; i < all_freq_table->table_size; i++) { + freq = all_freq_table->freq_table[i]; + len += scnprintf(buf + len, PAGE_SIZE - len, "\n%u\t\t", freq); + for_each_possible_cpu(cpu) { + all_stat = per_cpu(all_cpufreq_stats, cpu); + index = get_index_all_cpufreq_stat(all_stat, freq); + if (index != -1) { + len += scnprintf(buf + len, PAGE_SIZE - len, + "%llu\t\t", (unsigned long long) + cputime64_to_clock_t(all_stat->time_in_state[index])); + continue; + } + len += scnprintf(buf + len, PAGE_SIZE - len, "N/A\t\t"); + } + } + +out: + len += scnprintf(buf + len, PAGE_SIZE - len, "\n"); + return len; +} + #ifdef CONFIG_CPU_FREQ_STAT_DETAILS static ssize_t show_trans_table(struct cpufreq_policy *policy, char *buf) { @@ -155,6 +227,9 @@ static struct attribute_group stats_attr_group = { .name = "stats" }; +static struct kobj_attribute _attr_all_time_in_state = __ATTR(all_time_in_state, + 0444, show_all_time_in_state, NULL); + static int freq_table_get_index(struct cpufreq_stats *stat, unsigned int freq) { int index; @@ -189,6 +264,29 @@ static void cpufreq_stats_free_sysfs(unsigned int cpu) cpufreq_cpu_put(policy); } +static void cpufreq_allstats_free(void) +{ + int i; + struct all_cpufreq_stats *all_stat; + + sysfs_remove_file(cpufreq_global_kobject, + &_attr_all_time_in_state.attr); + + for (i = 0; i < total_cpus; i++) { + all_stat = per_cpu(all_cpufreq_stats, i); + if (!all_stat) + continue; + kfree(all_stat->time_in_state); + kfree(all_stat); + per_cpu(all_cpufreq_stats, i) = NULL; + } + if (all_freq_table) { + kfree(all_freq_table->freq_table); + kfree(all_freq_table); + all_freq_table = NULL; + } +} + static int cpufreq_stats_create_table(struct cpufreq_policy *policy, struct cpufreq_frequency_table *table) { @@ -262,6 +360,87 @@ error_get_fail: return ret; } +static bool check_all_freq_table(unsigned int freq) +{ + int i; + for (i = 0; i < all_freq_table->table_size; i++) { + if (freq == all_freq_table->freq_table[i]) + return true; + } + return false; +} + +static void create_all_freq_table(void) +{ + all_freq_table = kzalloc(sizeof(struct all_freq_table), + GFP_KERNEL); + if (!all_freq_table) + pr_warn("could not allocate memory for all_freq_table\n"); + return; +} + +static void add_all_freq_table(unsigned int freq) +{ + unsigned int size; + size = sizeof(unsigned int) * (all_freq_table->table_size + 1); + all_freq_table->freq_table = krealloc(all_freq_table->freq_table, + size, GFP_KERNEL); + if (IS_ERR(all_freq_table->freq_table)) { + pr_warn("Could not reallocate memory for freq_table\n"); + all_freq_table->freq_table = NULL; + return; + } + all_freq_table->freq_table[all_freq_table->table_size++] = freq; +} + +static void cpufreq_allstats_create(unsigned int cpu) +{ + int i , j = 0; + unsigned int alloc_size, count = 0; + struct cpufreq_frequency_table *table = cpufreq_frequency_get_table(cpu); + struct all_cpufreq_stats *all_stat; + + if (!table) + return; + + for (i = 0; table[i].frequency != CPUFREQ_TABLE_END; i++) { + unsigned int freq = table[i].frequency; + if (freq == CPUFREQ_ENTRY_INVALID) + continue; + count++; + } + + all_stat = kzalloc(sizeof(struct all_cpufreq_stats), + GFP_KERNEL); + if (!all_stat) { + pr_warn("Cannot allocate memory for cpufreq stats\n"); + return; + } + + /*Allocate memory for freq table per cpu as well as clockticks per freq*/ + alloc_size = count * sizeof(int) + count * sizeof(cputime64_t); + all_stat->time_in_state = kzalloc(alloc_size, GFP_KERNEL); + if (!all_stat->time_in_state) { + pr_warn("Cannot allocate memory for cpufreq time_in_state\n"); + kfree(all_stat); + all_stat = NULL; + return; + } + all_stat->freq_table = (unsigned int *) + (all_stat->time_in_state + count); + + for (i = 0; table[i].frequency != CPUFREQ_TABLE_END; i++) { + unsigned int freq = table[i].frequency; + if (freq == CPUFREQ_ENTRY_INVALID) + continue; + all_stat->freq_table[j++] = freq; + if (all_freq_table && !check_all_freq_table(freq)) + add_all_freq_table(freq); + } + all_stat->state_num = j; + per_cpu(all_cpufreq_stats, cpu) = all_stat; +} + static int cpufreq_stat_notifier_policy(struct notifier_block *nb, unsigned long val, void *data) { @@ -378,6 +557,17 @@ static struct notifier_block notifier_trans_block = { .notifier_call = cpufreq_stat_notifier_trans }; +static int compare_for_sort(const void *lhs_ptr, const void *rhs_ptr) +{ + unsigned int lhs = *(const unsigned int *)(lhs_ptr); + unsigned int rhs = *(const unsigned int *)(rhs_ptr); + if (lhs < rhs) + return -1; + if (lhs > rhs) + return 1; + return 0; +} + static int __init cpufreq_stats_init(void) { int ret; @@ -401,6 +591,19 @@ static int __init cpufreq_stats_init(void) for_each_online_cpu(cpu) { cpufreq_update_policy(cpu); } + + create_all_freq_table(); + for_each_possible_cpu(cpu) { + cpufreq_allstats_create(cpu); + } + if (all_freq_table && all_freq_table->freq_table) + sort(all_freq_table->freq_table, all_freq_table->table_size, + sizeof(unsigned int), &compare_for_sort, NULL); + ret = sysfs_create_file(cpufreq_global_kobject, + &_attr_all_time_in_state.attr); + if (ret) + pr_warn("Error creating sysfs file for cpufreq stats\n"); + return 0; } static void __exit cpufreq_stats_exit(void) @@ -416,6 +619,7 @@ static void __exit cpufreq_stats_exit(void) cpufreq_stats_free_table(cpu); cpufreq_stats_free_sysfs(cpu); } + cpufreq_allstats_free(); } MODULE_AUTHOR("Zou Nan hai "); From 6851444a2879991332cb71411d08973f20c1f7e6 Mon Sep 17 00:00:00 2001 From: Minsung Kim Date: Wed, 25 Jun 2014 19:44:50 +0900 Subject: [PATCH 024/168] cpufreq: fix sleeping in atomic context when realloc freq_table for all_time_in_state Commit 40cf2f8 (cpufreq: Persist cpufreq time in state data across hotplug) causes the following call trace to be spit on boot: BUG: sleeping function called from invalid context at mm/slub.c:936 in_atomic(): 1, irqs_disabled(): 0, pid: 1, name: swapper/0 CPU: 6 PID: 1 Comm: swapper/0 Not tainted 3.10.9-20140624.172707-eng-gd6c0f69-dirty #50 Backtrace: [] (dump_backtrace+0x0/0x10c) from [] (show_stack+0x18/0x1c) r6:ffff1788 r5:c0c020c0 r4:e609c000 r3:00000000 [] (show_stack+0x0/0x1c) from [] (dump_stack+0x20/0x28) [] (dump_stack+0x0/0x28) from [] (__might_sleep+0x104/0x120) [] (__might_sleep+0x0/0x120) from [] (__kmalloc_track_caller+0x144/0x274) r6:00000000 r5:e609c000 r4:e6802140 [] (__kmalloc_track_caller+0x0/0x274) from [] (krealloc+0x58/0xb0) [] (krealloc+0x0/0xb0) from [] (cpufreq_allstats_create+0x120/0x204) r8:e4c4ff00 r7:c0d266b8 r6:0013d620 r5:e4c4e600 r4:00000001 r3:e535d6d0 [] (cpufreq_allstats_create+0x0/0x204) from [] (cpufreq_stat_notifier_policy+0xb8/0xd0) [] (cpufreq_stat_notifier_policy+0x0/0xd0) from [] (notifier_call_chain+0x4c/0x8c) r5:00000000 r4:fffffffe [] (notifier_call_chain+0x0/0x8c) from [] (__blocking_notifier_call_chain+0x50/0x68) r8:c0cd4d00 r7:00000002 r6:e609dd7c r5:ffffffff r4:c0d25a4c r3:ffffffff [] (__blocking_notifier_call_chain+0x0/0x68) from [] (blocking_notifier_call_chain+0x20/0x28) r7:c0e24f30 r6:00000000 r5:e53e1e00 r4:e609dd7c [] (blocking_notifier_call_chain+0x0/0x28) from [] (__cpufreq_set_policy+0xc0/0x1d0) [] (__cpufreq_set_policy+0x0/0x1d0) from [] (cpufreq_add_dev_interface+0x20c/0x270) r7:00000008 r6:00000000 r5:e53e1e00 r4:e53e1e58 [] (cpufreq_add_dev_interface+0x0/0x270) from [] (cpufreq_add_dev+0x33c/0x420) [] (cpufreq_add_dev+0x0/0x420) from [] (subsys_interface_register+0x80/0xbc) [] (subsys_interface_register+0x0/0xbc) from [] (cpufreq_register_driver+0x8c/0x194) Change-Id: If77a656d0ea60a8fc4083283d104509fa6c07f8f Signed-off-by: Minsung Kim Git-commit: d7052164059dd0447cc8f00a2692161a81644417 Git-repo: https://android.googlesource.com/kernel/msm/ Signed-off-by: Nirmal Abraham Signed-off-by: Srinivasarao P --- drivers/cpufreq/cpufreq_stats.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cpufreq/cpufreq_stats.c b/drivers/cpufreq/cpufreq_stats.c index ea7cddbf5d3..a78c20bb808 100644 --- a/drivers/cpufreq/cpufreq_stats.c +++ b/drivers/cpufreq/cpufreq_stats.c @@ -384,7 +384,7 @@ static void add_all_freq_table(unsigned int freq) unsigned int size; size = sizeof(unsigned int) * (all_freq_table->table_size + 1); all_freq_table->freq_table = krealloc(all_freq_table->freq_table, - size, GFP_KERNEL); + size, GFP_ATOMIC); if (IS_ERR(all_freq_table->freq_table)) { pr_warn("Could not reallocate memory for freq_table\n"); all_freq_table->freq_table = NULL; From ffbb01df9e42b234cfb8703195e17d757f6fe582 Mon Sep 17 00:00:00 2001 From: Ruchi Kandoi Date: Tue, 21 Oct 2014 13:55:04 -0700 Subject: [PATCH 025/168] cpufreq: Avoid using global variable total_cpus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The change is to compile on kernels where cpufreq stats are compiled as a module (CONFIG_CPU_FREQ_STAT=m), because total_cpus is not exported for module use. Reported-By: Emilio López Signed-off-by: Ruchi Kandoi Change-Id: I4f3c74f0fac5e8d9449655b26bf3b407b0fe4290 Git-commit: c8468201e5509fd4a942247d32d07904e15a57df Git-repo: https://android.googlesource.com/kernel/msm/ Signed-off-by: Nirmal Abraham Signed-off-by: Srinivasarao P --- drivers/cpufreq/cpufreq_stats.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/cpufreq/cpufreq_stats.c b/drivers/cpufreq/cpufreq_stats.c index a78c20bb808..5e7ce2b5c08 100644 --- a/drivers/cpufreq/cpufreq_stats.c +++ b/drivers/cpufreq/cpufreq_stats.c @@ -266,19 +266,19 @@ static void cpufreq_stats_free_sysfs(unsigned int cpu) static void cpufreq_allstats_free(void) { - int i; + int cpu; struct all_cpufreq_stats *all_stat; sysfs_remove_file(cpufreq_global_kobject, &_attr_all_time_in_state.attr); - for (i = 0; i < total_cpus; i++) { - all_stat = per_cpu(all_cpufreq_stats, i); + for_each_possible_cpu(cpu) { + all_stat = per_cpu(all_cpufreq_stats, cpu); if (!all_stat) continue; kfree(all_stat->time_in_state); kfree(all_stat); - per_cpu(all_cpufreq_stats, i) = NULL; + per_cpu(all_cpufreq_stats, cpu) = NULL; } if (all_freq_table) { kfree(all_freq_table->freq_table); From 25b8b2472b61bee74e0b55da8af2affa2df80035 Mon Sep 17 00:00:00 2001 From: Ruchi Kandoi Date: Tue, 8 Apr 2014 19:06:14 -0700 Subject: [PATCH 026/168] cpufreq: prevents NULL pointer reference while all_time_in_state stats is collected. Platform cpufreq driver may not be initiated at the time cpufreq_stats_init() and therefore allocation of all_cpufreq_stats is moved. Also makes sure that all_cpufreq_stats is created appropriately for the devices which doesn't support hotplugging of CPUs. Change-Id: Iaf033688cf0e6cfdb3d605edbcbb984946f8ed45 Signed-off-by: Ruchi Kandoi Git-commit: 2b9dc8f2245b829087a3f61d113cff0f42295083 Git-repo: https://android.googlesource.com/kernel/msm/ Signed-off-by: Srinivasarao P --- drivers/cpufreq/cpufreq_stats.c | 57 +++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/drivers/cpufreq/cpufreq_stats.c b/drivers/cpufreq/cpufreq_stats.c index 5e7ce2b5c08..fc0e0e42045 100644 --- a/drivers/cpufreq/cpufreq_stats.c +++ b/drivers/cpufreq/cpufreq_stats.c @@ -77,6 +77,10 @@ static int cpufreq_stats_update(unsigned int cpu) spin_lock(&cpufreq_stats_lock); stat = per_cpu(cpufreq_stats_table, cpu); all_stat = per_cpu(all_cpufreq_stats, cpu); + if (!stat) { + spin_unlock(&cpufreq_stats_lock); + return 0; + } if (stat->time_in_state) { stat->time_in_state[stat->last_index] += cur_time - stat->last_time; @@ -133,6 +137,7 @@ static ssize_t show_all_time_in_state(struct kobject *kobj, ssize_t len = 0; unsigned int i, cpu, freq, index; struct all_cpufreq_stats *all_stat; + struct cpufreq_policy *policy; len += scnprintf(buf + len, PAGE_SIZE - len, "freq\t\t"); for_each_possible_cpu(cpu) { @@ -147,15 +152,20 @@ static ssize_t show_all_time_in_state(struct kobject *kobj, freq = all_freq_table->freq_table[i]; len += scnprintf(buf + len, PAGE_SIZE - len, "\n%u\t\t", freq); for_each_possible_cpu(cpu) { - all_stat = per_cpu(all_cpufreq_stats, cpu); + policy = cpufreq_cpu_get(cpu); + if (policy == NULL) + continue; + all_stat = per_cpu(all_cpufreq_stats, policy->cpu); index = get_index_all_cpufreq_stat(all_stat, freq); if (index != -1) { len += scnprintf(buf + len, PAGE_SIZE - len, "%llu\t\t", (unsigned long long) cputime64_to_clock_t(all_stat->time_in_state[index])); - continue; + } else { + len += scnprintf(buf + len, PAGE_SIZE - len, + "N/A\t\t"); } - len += scnprintf(buf + len, PAGE_SIZE - len, "N/A\t\t"); + cpufreq_cpu_put(policy); } } @@ -360,6 +370,13 @@ error_get_fail: return ret; } +static int compare_for_sort(const void *lhs_ptr, const void *rhs_ptr) +{ + unsigned int lhs = *(const unsigned int *)(lhs_ptr); + unsigned int rhs = *(const unsigned int *)(rhs_ptr); + return (lhs - rhs); +} + static bool check_all_freq_table(unsigned int freq) { int i; @@ -399,6 +416,7 @@ static void cpufreq_allstats_create(unsigned int cpu) unsigned int alloc_size, count = 0; struct cpufreq_frequency_table *table = cpufreq_frequency_get_table(cpu); struct all_cpufreq_stats *all_stat; + bool sort_needed = false; if (!table) return; @@ -429,16 +447,23 @@ static void cpufreq_allstats_create(unsigned int cpu) all_stat->freq_table = (unsigned int *) (all_stat->time_in_state + count); + spin_lock(&cpufreq_stats_lock); for (i = 0; table[i].frequency != CPUFREQ_TABLE_END; i++) { unsigned int freq = table[i].frequency; if (freq == CPUFREQ_ENTRY_INVALID) continue; all_stat->freq_table[j++] = freq; - if (all_freq_table && !check_all_freq_table(freq)) + if (all_freq_table && !check_all_freq_table(freq)) { add_all_freq_table(freq); + sort_needed = true; + } } + if (sort_needed) + sort(all_freq_table->freq_table, all_freq_table->table_size, + sizeof(unsigned int), &compare_for_sort, NULL); all_stat->state_num = j; per_cpu(all_cpufreq_stats, cpu) = all_stat; + spin_unlock(&cpufreq_stats_lock); } static int cpufreq_stat_notifier_policy(struct notifier_block *nb, @@ -453,6 +478,10 @@ static int cpufreq_stat_notifier_policy(struct notifier_block *nb, table = cpufreq_frequency_get_table(cpu); if (!table) return 0; + + if (!per_cpu(all_cpufreq_stats, cpu)) + cpufreq_allstats_create(cpu); + ret = cpufreq_stats_create_table(policy, table); if (ret) return ret; @@ -509,6 +538,9 @@ static int cpufreq_stats_create_table_cpu(unsigned int cpu) if (!table) goto out; + if (!per_cpu(all_cpufreq_stats, cpu)) + cpufreq_allstats_create(cpu); + ret = cpufreq_stats_create_table(policy, table); out: @@ -557,17 +589,6 @@ static struct notifier_block notifier_trans_block = { .notifier_call = cpufreq_stat_notifier_trans }; -static int compare_for_sort(const void *lhs_ptr, const void *rhs_ptr) -{ - unsigned int lhs = *(const unsigned int *)(lhs_ptr); - unsigned int rhs = *(const unsigned int *)(rhs_ptr); - if (lhs < rhs) - return -1; - if (lhs > rhs) - return 1; - return 0; -} - static int __init cpufreq_stats_init(void) { int ret; @@ -593,12 +614,6 @@ static int __init cpufreq_stats_init(void) } create_all_freq_table(); - for_each_possible_cpu(cpu) { - cpufreq_allstats_create(cpu); - } - if (all_freq_table && all_freq_table->freq_table) - sort(all_freq_table->freq_table, all_freq_table->table_size, - sizeof(unsigned int), &compare_for_sort, NULL); ret = sysfs_create_file(cpufreq_global_kobject, &_attr_all_time_in_state.attr); if (ret) From 80aecb90313f51cedad18dc654698ea7c1a6a06a Mon Sep 17 00:00:00 2001 From: Ruchi Kandoi Date: Thu, 16 Apr 2015 16:32:02 -0700 Subject: [PATCH 027/168] cpufreq_stats: Adds the fucntionality to load current values for each frequency for all the cores. The current values for the cpu cores needs to be added to the device tree for this functionaly to work. It loads the current values for each frequecy in uA for all the cores. Bug: 21498425 Change-Id: If03311aaeb3e4c09375dd0beb9ad4fbb254b5c08 Signed-off-by: Ruchi Kandoi Git-commit: 0d6489d36f29c17c32e82be51f53e073c9382412 Git-repo: https://android.googlesource.com/kernel/msm/ Signed-off-by: Srinivasarao P --- drivers/cpufreq/cpufreq_stats.c | 173 ++++++++++++++++++++++++++------ 1 file changed, 142 insertions(+), 31 deletions(-) diff --git a/drivers/cpufreq/cpufreq_stats.c b/drivers/cpufreq/cpufreq_stats.c index fc0e0e42045..347a2bdc94b 100644 --- a/drivers/cpufreq/cpufreq_stats.c +++ b/drivers/cpufreq/cpufreq_stats.c @@ -22,6 +22,7 @@ #include #include #include +#include #include static spinlock_t cpufreq_stats_lock; @@ -52,6 +53,12 @@ struct all_cpufreq_stats { unsigned int *freq_table; }; +struct cpufreq_power_stats { + unsigned int state_num; + unsigned int *curr; + unsigned int *freq_table; +}; + struct all_freq_table { unsigned int *freq_table; unsigned int table_size; @@ -61,6 +68,7 @@ static struct all_freq_table *all_freq_table; static DEFINE_PER_CPU(struct all_cpufreq_stats *, all_cpufreq_stats); static DEFINE_PER_CPU(struct cpufreq_stats *, cpufreq_stats_table); +static DEFINE_PER_CPU(struct cpufreq_power_stats *, cpufreq_power_stats); struct cpufreq_stats_attribute { struct attribute attr; @@ -131,6 +139,29 @@ static int get_index_all_cpufreq_stat(struct all_cpufreq_stats *all_stat, return -1; } +static ssize_t show_current_in_state(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + ssize_t len = 0; + unsigned int i, cpu; + struct cpufreq_power_stats *powerstats; + + spin_lock(&cpufreq_stats_lock); + for_each_possible_cpu(cpu) { + powerstats = per_cpu(cpufreq_power_stats, cpu); + if (!powerstats) + continue; + len += scnprintf(buf + len, PAGE_SIZE - len, "CPU%d:", cpu); + for (i = 0; i < powerstats->state_num; i++) + len += scnprintf(buf + len, PAGE_SIZE - len, + "%d=%d ", powerstats->freq_table[i], + powerstats->curr[i]); + len += scnprintf(buf + len, PAGE_SIZE - len, "\n"); + } + spin_unlock(&cpufreq_stats_lock); + return len; +} + static ssize_t show_all_time_in_state(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { @@ -240,6 +271,9 @@ static struct attribute_group stats_attr_group = { static struct kobj_attribute _attr_all_time_in_state = __ATTR(all_time_in_state, 0444, show_all_time_in_state, NULL); +static struct kobj_attribute _attr_current_in_state = __ATTR(current_in_state, + 0444, show_current_in_state, NULL); + static int freq_table_get_index(struct cpufreq_stats *stat, unsigned int freq) { int index; @@ -297,10 +331,27 @@ static void cpufreq_allstats_free(void) } } -static int cpufreq_stats_create_table(struct cpufreq_policy *policy, - struct cpufreq_frequency_table *table) +static void cpufreq_powerstats_free(void) { - unsigned int i, j, count = 0, ret = 0; + int cpu; + struct cpufreq_power_stats *powerstats; + + sysfs_remove_file(cpufreq_global_kobject, &_attr_current_in_state.attr); + + for_each_possible_cpu(cpu) { + powerstats = per_cpu(cpufreq_power_stats, cpu); + if (!powerstats) + continue; + kfree(powerstats->curr); + kfree(powerstats); + per_cpu(cpufreq_power_stats, cpu) = NULL; + } +} + +static int cpufreq_stats_create_table(struct cpufreq_policy *policy, + struct cpufreq_frequency_table *table, int count) +{ + unsigned int i, j, ret = 0; struct cpufreq_stats *stat; struct cpufreq_policy *data; unsigned int alloc_size; @@ -324,12 +375,6 @@ static int cpufreq_stats_create_table(struct cpufreq_policy *policy, stat->cpu = cpu; per_cpu(cpufreq_stats_table, cpu) = stat; - for (i = 0; table[i].frequency != CPUFREQ_TABLE_END; i++) { - unsigned int freq = table[i].frequency; - if (freq == CPUFREQ_ENTRY_INVALID) - continue; - count++; - } alloc_size = count * sizeof(int) + count * sizeof(cputime64_t); @@ -370,6 +415,54 @@ error_get_fail: return ret; } +static void cpufreq_powerstats_create(unsigned int cpu, + struct cpufreq_frequency_table *table, int count) { + unsigned int alloc_size, i = 0, j = 0, ret = 0; + struct cpufreq_power_stats *powerstats; + struct device_node *cpu_node; + char device_path[16]; + + powerstats = kzalloc(sizeof(struct cpufreq_power_stats), + GFP_KERNEL); + if (!powerstats) + return; + + /* Allocate memory for freq table per cpu as well as clockticks per + * freq*/ + alloc_size = count * sizeof(unsigned int) + + count * sizeof(unsigned int); + powerstats->curr = kzalloc(alloc_size, GFP_KERNEL); + if (!powerstats->curr) { + kfree(powerstats); + return; + } + powerstats->freq_table = powerstats->curr + count; + + spin_lock(&cpufreq_stats_lock); + for (i = 0; table[i].frequency != CPUFREQ_TABLE_END && j < count; i++) { + unsigned int freq = table[i].frequency; + + if (freq == CPUFREQ_ENTRY_INVALID) + continue; + powerstats->freq_table[j++] = freq; + } + powerstats->state_num = j; + + snprintf(device_path, sizeof(device_path), "/cpus/cpu@%d", cpu); + cpu_node = of_find_node_by_path(device_path); + if (cpu_node) { + ret = of_property_read_u32_array(cpu_node, "current", + powerstats->curr, count); + if (ret) { + kfree(powerstats->curr); + kfree(powerstats); + powerstats = NULL; + } + } + per_cpu(cpufreq_power_stats, cpu) = powerstats; + spin_unlock(&cpufreq_stats_lock); +} + static int compare_for_sort(const void *lhs_ptr, const void *rhs_ptr) { unsigned int lhs = *(const unsigned int *)(lhs_ptr); @@ -410,24 +503,14 @@ static void add_all_freq_table(unsigned int freq) all_freq_table->freq_table[all_freq_table->table_size++] = freq; } -static void cpufreq_allstats_create(unsigned int cpu) +static void cpufreq_allstats_create(unsigned int cpu, + struct cpufreq_frequency_table *table, int count) { int i , j = 0; - unsigned int alloc_size, count = 0; - struct cpufreq_frequency_table *table = cpufreq_frequency_get_table(cpu); + unsigned int alloc_size; struct all_cpufreq_stats *all_stat; bool sort_needed = false; - if (!table) - return; - - for (i = 0; table[i].frequency != CPUFREQ_TABLE_END; i++) { - unsigned int freq = table[i].frequency; - if (freq == CPUFREQ_ENTRY_INVALID) - continue; - count++; - } - all_stat = kzalloc(sizeof(struct all_cpufreq_stats), GFP_KERNEL); if (!all_stat) { @@ -469,7 +552,7 @@ static void cpufreq_allstats_create(unsigned int cpu) static int cpufreq_stat_notifier_policy(struct notifier_block *nb, unsigned long val, void *data) { - int ret; + int ret, count = 0, i; struct cpufreq_policy *policy = data; struct cpufreq_frequency_table *table; unsigned int cpu = policy->cpu; @@ -479,10 +562,21 @@ static int cpufreq_stat_notifier_policy(struct notifier_block *nb, if (!table) return 0; - if (!per_cpu(all_cpufreq_stats, cpu)) - cpufreq_allstats_create(cpu); + for (i = 0; table[i].frequency != CPUFREQ_TABLE_END; i++) { + unsigned int freq = table[i].frequency; - ret = cpufreq_stats_create_table(policy, table); + if (freq == CPUFREQ_ENTRY_INVALID) + continue; + count++; + } + + if (!per_cpu(all_cpufreq_stats, cpu)) + cpufreq_allstats_create(cpu, table, count); + + if (!per_cpu(cpufreq_power_stats, cpu)) + cpufreq_powerstats_create(cpu, table, count); + + ret = cpufreq_stats_create_table(policy, table, count); if (ret) return ret; return 0; @@ -528,7 +622,7 @@ static int cpufreq_stats_create_table_cpu(unsigned int cpu) { struct cpufreq_policy *policy; struct cpufreq_frequency_table *table; - int ret = -ENODEV; + int ret = -ENODEV, i, count = 0; policy = cpufreq_cpu_get(cpu); if (!policy) @@ -538,10 +632,21 @@ static int cpufreq_stats_create_table_cpu(unsigned int cpu) if (!table) goto out; - if (!per_cpu(all_cpufreq_stats, cpu)) - cpufreq_allstats_create(cpu); + for (i = 0; table[i].frequency != CPUFREQ_TABLE_END; i++) { + unsigned int freq = table[i].frequency; - ret = cpufreq_stats_create_table(policy, table); + if (freq == CPUFREQ_ENTRY_INVALID) + continue; + count++; + } + + if (!per_cpu(all_cpufreq_stats, cpu)) + cpufreq_allstats_create(cpu, table, count); + + if (!per_cpu(cpufreq_power_stats, cpu)) + cpufreq_powerstats_create(cpu, table, count); + + ret = cpufreq_stats_create_table(policy, table, count); out: cpufreq_cpu_put(policy); @@ -617,7 +722,12 @@ static int __init cpufreq_stats_init(void) ret = sysfs_create_file(cpufreq_global_kobject, &_attr_all_time_in_state.attr); if (ret) - pr_warn("Error creating sysfs file for cpufreq stats\n"); + pr_warn("Cannot create sysfs file for cpufreq stats\n"); + + ret = sysfs_create_file(cpufreq_global_kobject, + &_attr_current_in_state.attr); + if (ret) + pr_warn("Cannot create sysfs file for cpufreq current stats\n"); return 0; } @@ -635,6 +745,7 @@ static void __exit cpufreq_stats_exit(void) cpufreq_stats_free_sysfs(cpu); } cpufreq_allstats_free(); + cpufreq_powerstats_free(); } MODULE_AUTHOR("Zou Nan hai "); From c997826cefe8acca510afb5670a333c761e8dccb Mon Sep 17 00:00:00 2001 From: Ruchi Kandoi Date: Fri, 17 Apr 2015 16:33:29 -0700 Subject: [PATCH 028/168] sched: cpufreq: Adds a field cpu_power in the task_struct cpu_power has been added to keep track of amount of power each task is consuming. cpu_power is updated whenever stime and utime are updated for a task. power is computed by taking into account the frequency at which the current core was running and the current for cpu actively running at hat frequency. Bug: 21498425 Change-Id: Ic535941e7b339aab5cae9081a34049daeb44b248 Signed-off-by: Ruchi Kandoi Git-commit: d0c5706bc96517885051d4eacde519d748c1f737 Git-repo: https://android.googlesource.com/kernel/msm/ Signed-off-by: Srinivasarao P --- drivers/cpufreq/cpufreq_stats.c | 19 +++++++++++++++++++ include/linux/cpufreq.h | 7 +++++++ include/linux/sched.h | 2 ++ kernel/fork.c | 1 + kernel/sched/core.c | 7 +++++++ 5 files changed, 36 insertions(+) diff --git a/drivers/cpufreq/cpufreq_stats.c b/drivers/cpufreq/cpufreq_stats.c index 347a2bdc94b..fcb1a68df4f 100644 --- a/drivers/cpufreq/cpufreq_stats.c +++ b/drivers/cpufreq/cpufreq_stats.c @@ -23,6 +23,7 @@ #include #include #include +#include #include static spinlock_t cpufreq_stats_lock; @@ -139,6 +140,24 @@ static int get_index_all_cpufreq_stat(struct all_cpufreq_stats *all_stat, return -1; } +void acct_update_power(struct task_struct *task, cputime_t cputime) { + struct cpufreq_power_stats *powerstats; + struct cpufreq_stats *stats; + unsigned int cpu_num, curr; + + if (!task) + return; + cpu_num = task_cpu(task); + powerstats = per_cpu(cpufreq_power_stats, cpu_num); + stats = per_cpu(cpufreq_stats_table, cpu_num); + if (!powerstats || !stats) + return; + + curr = powerstats->curr[stats->last_index]; + task->cpu_power += curr * cputime_to_usecs(cputime); +} +EXPORT_SYMBOL_GPL(acct_update_power); + static ssize_t show_current_in_state(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 6723f48f2d4..58442f6b277 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -20,6 +20,7 @@ #include #include #include +#include #define CPUFREQ_NAME_LEN 16 @@ -417,4 +418,10 @@ void cpufreq_frequency_table_get_attr(struct cpufreq_frequency_table *table, void cpufreq_frequency_table_put_attr(unsigned int cpu); +/********************************************************************* + * CPUFREQ STATS * + *********************************************************************/ + +void acct_update_power(struct task_struct *p, cputime_t cputime); + #endif /* _LINUX_CPUFREQ_H */ diff --git a/include/linux/sched.h b/include/linux/sched.h index 952bcb1d86f..b000bcc90bf 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1393,6 +1393,8 @@ struct task_struct { cputime_t utime, stime, utimescaled, stimescaled; cputime_t gtime; + unsigned long long cpu_power; + #ifndef CONFIG_VIRT_CPU_ACCOUNTING cputime_t prev_utime, prev_stime; #endif diff --git a/kernel/fork.c b/kernel/fork.c index a8bf7218bbf..b706730e1e8 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1219,6 +1219,7 @@ static struct task_struct *copy_process(unsigned long clone_flags, p->utime = p->stime = p->gtime = 0; p->utimescaled = p->stimescaled = 0; + p->cpu_power = 0; #ifndef CONFIG_VIRT_CPU_ACCOUNTING p->prev_utime = p->prev_stime = 0; #endif diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 780ac9b0013..f8a1cbecec0 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -2719,6 +2720,9 @@ void account_user_time(struct task_struct *p, cputime_t cputime, /* Account for user time used */ acct_update_integrals(p); + + /* Account power usage for user time */ + acct_update_power(p, cputime); } /* @@ -2769,6 +2773,9 @@ void __account_system_time(struct task_struct *p, cputime_t cputime, /* Account for system time used */ acct_update_integrals(p); + + /* Account power usage for system time */ + acct_update_power(p, cputime); } /* From f49d0f7761a54aabe00cd7e754adfb54541aa9cf Mon Sep 17 00:00:00 2001 From: Ruchi Kandoi Date: Fri, 17 Apr 2015 16:52:54 -0700 Subject: [PATCH 029/168] uid_cputime: Extends the cputime functionality to report power per uid /proc/uid_cputime/show_uid_stats shows a third field power for each of the uids. It represents the power in the units (uAusec) Bug: 21498425 Change-Id: I52fdc5e59647e9dc97561a26d56f462a2689ba9c Signed-off-by: Ruchi Kandoi Git-commit: d17e7e2771a3998c2514de082be588522625618e Git-repo: https://android.googlesource.com/kernel/msm/ Signed-off-by: Srinivasarao P --- drivers/misc/uid_cputime.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/misc/uid_cputime.c b/drivers/misc/uid_cputime.c index 50f10290b3f..130ac641fd4 100644 --- a/drivers/misc/uid_cputime.c +++ b/drivers/misc/uid_cputime.c @@ -38,6 +38,8 @@ struct uid_entry { cputime_t stime; cputime_t active_utime; cputime_t active_stime; + unsigned long long active_power; + unsigned long long power; struct hlist_node hash; }; @@ -83,6 +85,7 @@ static int uid_stat_show(struct seq_file *m, void *v) hash_for_each(hash_table, bkt, node, uid_entry, hash) { uid_entry->active_stime = 0; uid_entry->active_utime = 0; + uid_entry->active_power = 0; } read_lock(&tasklist_lock); @@ -97,6 +100,7 @@ static int uid_stat_show(struct seq_file *m, void *v) } uid_entry->active_utime += task->utime; uid_entry->active_stime += task->stime; + uid_entry->active_power += task->cpu_power; } read_unlock(&tasklist_lock); @@ -105,9 +109,12 @@ static int uid_stat_show(struct seq_file *m, void *v) uid_entry->active_utime; cputime_t total_stime = uid_entry->stime + uid_entry->active_stime; - seq_printf(m, "%d: %u %u\n", uid_entry->uid, + unsigned long long total_power = uid_entry->power + + uid_entry->active_power; + seq_printf(m, "%d: %u %u %llu\n", uid_entry->uid, cputime_to_usecs(total_utime), - cputime_to_usecs(total_stime)); + cputime_to_usecs(total_stime), + total_power); } mutex_unlock(&uid_lock); @@ -199,6 +206,7 @@ static int process_notifier(struct notifier_block *self, uid_entry->utime += task->utime; uid_entry->stime += task->stime; + uid_entry->power += task->cpu_power; exit: mutex_unlock(&uid_lock); From 1fee576ec8dde10c579dc6b6234c3944df9f9842 Mon Sep 17 00:00:00 2001 From: Ruchi Kandoi Date: Fri, 26 Jun 2015 14:19:21 -0700 Subject: [PATCH 030/168] uid_cputime: Fixes double accounting race condition on task exit. This avoids the race where a particular process is terminating and we read the show_uid_stats. At this time since the task_struct still exists and we will account for the terminating process as one of the active task, where as the stats would have been added in the task exit callback. Bug: 22064385 Change-Id: Id2ae04b33fcd230eda9683a41b6019d4dd8f5d85 Signed-off-by: Jin Qian Signed-off-by: Ruchi Kandoi Git-commit: 42ec52850c400aca1c67da166c11a7ea256c7884 Git-repo: https://android.googlesource.com/kernel/msm/ Signed-off-by: Srinivasarao P --- drivers/misc/uid_cputime.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/misc/uid_cputime.c b/drivers/misc/uid_cputime.c index 130ac641fd4..e8edfe9182f 100644 --- a/drivers/misc/uid_cputime.c +++ b/drivers/misc/uid_cputime.c @@ -98,6 +98,10 @@ static int uid_stat_show(struct seq_file *m, void *v) __func__, task_uid(task)); return -ENOMEM; } + /* if this task is exiting, we have already accounted for the + * time and power. */ + if (task->cpu_power == ULLONG_MAX) + continue; uid_entry->active_utime += task->utime; uid_entry->active_stime += task->stime; uid_entry->active_power += task->cpu_power; @@ -207,6 +211,7 @@ static int process_notifier(struct notifier_block *self, uid_entry->utime += task->utime; uid_entry->stime += task->stime; uid_entry->power += task->cpu_power; + task->cpu_power = ULLONG_MAX; exit: mutex_unlock(&uid_lock); From f4b41b75e3b545212e863571c40ea872185370ab Mon Sep 17 00:00:00 2001 From: Jin Qian Date: Mon, 13 Jul 2015 18:16:55 -0700 Subject: [PATCH 031/168] uid_cputime: fix cputime overflow Converting cputime_t to usec caused overflow when the value is greater than 1 hour. Use msec and convert to unsigned long long to support bigger range. Bug: 22461683 Change-Id: I853fe3e8e7dbf0d3e2cc5c6f9688a5a6e1f1fb3e Signed-off-by: Jin Qian Git-commit: 73f7982c2bc8590fa63b9f5e308b9dda27701bf9 Git-repo: https://android.googlesource.com/kernel/msm/ Signed-off-by: Srinivasarao P --- drivers/misc/uid_cputime.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/misc/uid_cputime.c b/drivers/misc/uid_cputime.c index e8edfe9182f..18096f679fe 100644 --- a/drivers/misc/uid_cputime.c +++ b/drivers/misc/uid_cputime.c @@ -115,10 +115,12 @@ static int uid_stat_show(struct seq_file *m, void *v) uid_entry->active_stime; unsigned long long total_power = uid_entry->power + uid_entry->active_power; - seq_printf(m, "%d: %u %u %llu\n", uid_entry->uid, - cputime_to_usecs(total_utime), - cputime_to_usecs(total_stime), - total_power); + seq_printf(m, "%d: %llu %llu %llu\n", uid_entry->uid, + (unsigned long long)jiffies_to_msecs( + cputime_to_jiffies(total_utime)) * USEC_PER_MSEC, + (unsigned long long)jiffies_to_msecs( + cputime_to_jiffies(total_stime)) * USEC_PER_MSEC, + total_power); } mutex_unlock(&uid_lock); From e4dd583628ffece778e7f7a499f6ca5d914d4145 Mon Sep 17 00:00:00 2001 From: Ruchi Kandoi Date: Fri, 31 Jul 2015 10:17:54 -0700 Subject: [PATCH 032/168] uid_cputime: Iterates over all the threads instead of processes. Bug: 22833116 Change-Id: I775a18f61bd2f4df2bec23d01bd49421d0969f87 Signed-off-by: Ruchi Kandoi Git-commit: 20dcde6cbc0cd4a17c4d4fd3aa68dc5b03f520d1 Git-repo: https://android.googlesource.com/kernel/msm/ Signed-off-by: Srinivasarao P --- drivers/misc/uid_cputime.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/misc/uid_cputime.c b/drivers/misc/uid_cputime.c index 18096f679fe..98de3ce62e3 100644 --- a/drivers/misc/uid_cputime.c +++ b/drivers/misc/uid_cputime.c @@ -76,7 +76,7 @@ static struct uid_entry *find_or_register_uid(uid_t uid) static int uid_stat_show(struct seq_file *m, void *v) { struct uid_entry *uid_entry; - struct task_struct *task; + struct task_struct *task, *temp; unsigned long bkt; struct hlist_node *node; @@ -89,7 +89,7 @@ static int uid_stat_show(struct seq_file *m, void *v) } read_lock(&tasklist_lock); - for_each_process(task) { + do_each_thread(temp, task) { uid_entry = find_or_register_uid(task_uid(task)); if (!uid_entry) { read_unlock(&tasklist_lock); @@ -105,7 +105,7 @@ static int uid_stat_show(struct seq_file *m, void *v) uid_entry->active_utime += task->utime; uid_entry->active_stime += task->stime; uid_entry->active_power += task->cpu_power; - } + } while_each_thread(temp, task); read_unlock(&tasklist_lock); hash_for_each(hash_table, bkt, node, uid_entry, hash) { From 9d219c0bc6505b538d68bc56df07d69264ba91dc Mon Sep 17 00:00:00 2001 From: Ruchi Kandoi Date: Fri, 26 Jun 2015 19:02:08 -0700 Subject: [PATCH 033/168] cpu_power: Avoids race condition when the task exits. When the task is terminated, the cpu_power for that particular task is added to the terminated tasks. cpu_power is set to ULLONG_MAX at this point to avoid double accounting of the power. It is possible that before the task releases all the resources, cpu reschedules the task or a timer interrupt is fired. At this point we will try to add the additional time to the process, which will cause the accounting to be skewed. This avoids the case where we change the cpu_power when it is already set to ULLONG_MAX. Bug: 22064385 Change-Id: I405733725d535b0a864088516bf52fa3638ee6aa Signed-off-by: Jin Qian Signed-off-by: Ruchi Kandoi Git-commit: 51a8b4487f5e302e4919bfd06b855521a31d9885 Git-repo: https://android.googlesource.com/kernel/msm/ Signed-off-by: Srinivasarao P --- drivers/cpufreq/cpufreq_stats.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/cpufreq_stats.c b/drivers/cpufreq/cpufreq_stats.c index fcb1a68df4f..3aa1ec2c5e5 100644 --- a/drivers/cpufreq/cpufreq_stats.c +++ b/drivers/cpufreq/cpufreq_stats.c @@ -154,7 +154,8 @@ void acct_update_power(struct task_struct *task, cputime_t cputime) { return; curr = powerstats->curr[stats->last_index]; - task->cpu_power += curr * cputime_to_usecs(cputime); + if (task->cpu_power != ULLONG_MAX) + task->cpu_power += curr * cputime_to_usecs(cputime); } EXPORT_SYMBOL_GPL(acct_update_power); From ac4d1adc0ca22c12ea31303a2ccff61b6a47dd19 Mon Sep 17 00:00:00 2001 From: Ruchi Kandoi Date: Tue, 2 Jun 2015 18:18:43 -0700 Subject: [PATCH 034/168] boot:dtsi: Add the active current values in uA for the cpus in device tree. Bug: 21498425 Change-Id: I791a8368698fefc44d1a9445f40825bf571256a5 Signed-off-by: Ruchi Kandoi Git-commit: a059562a77c7a2f9f9c18f94fb3b102b61b9f7ed Git-repo: https://android.googlesource.com/kernel/msm/ Signed-off-by: Srinivasarao P --- arch/arm/boot/dts/msm8974.dtsi | 64 ++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/arch/arm/boot/dts/msm8974.dtsi b/arch/arm/boot/dts/msm8974.dtsi index 895071dee60..db2191e9765 100644 --- a/arch/arm/boot/dts/msm8974.dtsi +++ b/arch/arm/boot/dts/msm8974.dtsi @@ -47,24 +47,88 @@ device_type = "cpu"; compatible = "qcom,krait"; reg = <0x0>; + // The currents(uA) correspond to the frequencies in the + // frequency table. + current = < 57900 //300000 kHz + 88200 //422400 kHz + 99600 //652800 kHz + 138800 //729600 kHz + 149600 //883200 kHz + 170200 //960000 kHz + 178300 //1036800 kHz + 189100 //1190400 kHz + 232100 //1267200 kHz + 256500 //1497600 kHz + 266400 //1574000 kHz + 287700 //1728000 kHz + 325700 //1958400 kHz + 386200>; //2265600 kHz }; CPU1: cpu@1 { device_type = "cpu"; compatible = "qcom,krait"; reg = <0x1>; + // The currents(uA) correspond to the frequencies in the + // frequency table. + current = < 23739 //300000 kHz + 36162 //422400 kHz + 40836 //652800 kHz + 56908 //729600 kHz + 61335 //883200 kHz + 69782 //960000 kHz + 73103 //1036800 kHz + 77531 //1190400 kHz + 95161 //1267200 kHz + 105165 //1497600 kHz + 109224 //1574000 kHz + 117957 //1728000 kHz + 133537 //1958400 kHz + 158342>; //2265600 kHz }; CPU2: cpu@2 { device_type = "cpu"; compatible = "qcom,krait"; reg = <0x2>; + // The currents(uA) correspond to the frequencies in the + // frequency table. + current = < 27213 //300000 kHz + 41454 //422400 kHz + 46812 //652800 kHz + 65235 //729600 kHz + 70312 //883200 kHz + 79994 //960000 kHz + 83801 //1036800 kHz + 88877 //1190400 kHz + 109087 //1267200 kHz + 120555 //1497600 kHz + 125208 //1574000 kHz + 135219 //1728000 kHz + 153079 //1958400 kHz + 181514>; //2265600 kHz }; CPU3: cpu@3 { device_type = "cpu"; compatible = "qcom,krait"; reg = <0x3>; + // The currents(uA) correspond to the frequencies in the + // frequency table. + current = < 31266 //300000 kHz + 47628 //422400 kHz + 53784 //652800 kHz + 74952 //729600 kHz + 80784 //883200 kHz + 91908 //960000 kHz + 96282 //1036800 kHz + 102114 //1190400 kHz + 125334 //1267200 kHz + 138510 //1497600 kHz + 143856 //1574000 kHz + 155358 //1728000 kHz + 175878 //1958400 kHz + 208548>; //2265600 kHz }; }; From cfe1568b934a53a5f6dd4173783af1dede22a139 Mon Sep 17 00:00:00 2001 From: Nirmal Abraham Date: Wed, 7 Oct 2015 16:23:43 +0530 Subject: [PATCH 035/168] cpufreq: Correct the data reported in all_time_in_state Commit bd9474e059bbb2bb62f7e93894cfc3d3ba473fb2 (cpufreq_stats: Adds the fucntionality to load current values for each frequency for all the cores) introduced a change by which 'cpufreq_allstats_create' gets called at initialization (from 'cpufreq_stats_init' instead of 'cpufreq_stat_notifier_policy'). This causes 'cpufreq_allstate_create' to be called before the freq_table is allocated from 'create_all_freq_table'. Due to this, the data for cpu's which are online at boot are not added to the 'all_freq_table' leading to the incorrect reporting of data when the below sysfs command is run - 'cat sys/devices/system/cpu/cpufreq/all_time_in_state'. Correct this behaviour by altering the cpufreq_stats init sequence by which the memory for 'all_freq_table' is allocated before the 'cpufreq_allstats_create' function is called. Change-Id: I2232dacdc0deec4d1987c418e833fe28f74623fc Signed-off-by: Nirmal Abraham Signed-off-by: Srinivasarao P --- drivers/cpufreq/cpufreq_stats.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/cpufreq_stats.c b/drivers/cpufreq/cpufreq_stats.c index 3aa1ec2c5e5..e2599926990 100644 --- a/drivers/cpufreq/cpufreq_stats.c +++ b/drivers/cpufreq/cpufreq_stats.c @@ -509,6 +509,18 @@ static void create_all_freq_table(void) return; } +static void free_all_freq_table(void) +{ + if (all_freq_table) { + if (all_freq_table->freq_table) { + kfree(all_freq_table->freq_table); + all_freq_table->freq_table = NULL; + } + kfree(all_freq_table); + all_freq_table = NULL; + } +} + static void add_all_freq_table(unsigned int freq) { unsigned int size; @@ -725,11 +737,14 @@ static int __init cpufreq_stats_init(void) if (ret) return ret; + create_all_freq_table(); + ret = cpufreq_register_notifier(¬ifier_trans_block, CPUFREQ_TRANSITION_NOTIFIER); if (ret) { cpufreq_unregister_notifier(¬ifier_policy_block, CPUFREQ_POLICY_NOTIFIER); + free_all_freq_table(); return ret; } @@ -738,7 +753,6 @@ static int __init cpufreq_stats_init(void) cpufreq_update_policy(cpu); } - create_all_freq_table(); ret = sysfs_create_file(cpufreq_global_kobject, &_attr_all_time_in_state.attr); if (ret) From e6ca25a2eb80c3e95e5cc9b1a9c039b37cec2cbb Mon Sep 17 00:00:00 2001 From: Seungwon Jeon Date: Wed, 26 Dec 2012 10:40:17 +0900 Subject: [PATCH 036/168] mmc: queue: exclude asynchronous transfer for special request Unlike normal r/w request, special requests(discard, flush) is finished with a one-time issue_fn. Request change to mqrq_prev makes unnecessary call. Change-Id: I00b7c428d760560a808c1fb51409c1fd0708bd81 Signed-off-by: Seungwon Jeon Reviewed-by: Konstantin Dorfman Signed-off-by: Chris Ball [asutoshd@codeaurora.org: fixed merge conflicts] Signed-off-by: Asutosh Das --- drivers/mmc/card/queue.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/mmc/card/queue.c b/drivers/mmc/card/queue.c index cceb8d96318..fd57f6e3d84 100644 --- a/drivers/mmc/card/queue.c +++ b/drivers/mmc/card/queue.c @@ -24,6 +24,8 @@ #define MMC_QUEUE_BOUNCESZ 65536 +#define MMC_REQ_SPECIAL_MASK (REQ_DISCARD | REQ_FLUSH) + /* * Based on benchmark tests the default num of requests to trigger the write * packing was determined, to keep the read latency as low as possible and @@ -66,6 +68,7 @@ static int mmc_queue_thread(void *d) do { struct mmc_queue_req *tmp; struct request *req = NULL; + unsigned int cmd_flags = 0; spin_lock_irq(q->queue_lock); set_current_state(TASK_INTERRUPTIBLE); @@ -75,6 +78,7 @@ static int mmc_queue_thread(void *d) if (req || mq->mqrq_prev->req) { set_current_state(TASK_RUNNING); + cmd_flags = req ? req->cmd_flags : 0; mq->issue_fn(mq, req); if (test_bit(MMC_QUEUE_NEW_REQUEST, &mq->flags)) { continue; /* fetch again */ @@ -95,7 +99,13 @@ static int mmc_queue_thread(void *d) /* * Current request becomes previous request * and vice versa. + * In case of special requests, current request + * has been finished. Do not assign it to previous + * request. */ + if (cmd_flags & MMC_REQ_SPECIAL_MASK) + mq->mqrq_cur->req = NULL; + mq->mqrq_prev->brq.mrq.data = NULL; mq->mqrq_prev->req = NULL; tmp = mq->mqrq_prev; From 0fe4ec363c22ca736da710b8d31ba86cbe583dc1 Mon Sep 17 00:00:00 2001 From: Asutosh Das Date: Mon, 7 Dec 2015 10:27:24 +0530 Subject: [PATCH 037/168] mmc: queue: use cached attributes of special requests Special requests like discard and flush would be completed and their flags cleared to 0x6b6b6b6b. Hence, refer to the saved flags further. CRs-fixed: 933468 Change-Id: I1e94c5a743eceabb802ca099c215292d2624eb29 Signed-off-by: Asutosh Das --- drivers/mmc/card/queue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/mmc/card/queue.c b/drivers/mmc/card/queue.c index fd57f6e3d84..4edea18b2b0 100644 --- a/drivers/mmc/card/queue.c +++ b/drivers/mmc/card/queue.c @@ -84,7 +84,7 @@ static int mmc_queue_thread(void *d) continue; /* fetch again */ } else if (test_bit(MMC_QUEUE_URGENT_REQUEST, &mq->flags) && (mq->mqrq_cur->req && - !(mq->mqrq_cur->req->cmd_flags & + !(cmd_flags & MMC_REQ_NOREINSERT_MASK))) { /* * clean current request when urgent request From 59155a76f8046b1f2055ef2b57fef29050d5367d Mon Sep 17 00:00:00 2001 From: Zhen Kong Date: Wed, 20 May 2015 13:50:04 -0700 Subject: [PATCH 038/168] qseecom: Register the existing app if it is loaded by appsbl. Register the app in qseecom driver if it has been already loaded by appsbl before Change-Id: Iec39137a7e18dc703c731e55955ab84d1b9c97f3 Signed-off-by: Zhen Kong Signed-off-by: Shreyas Narayan --- drivers/misc/qseecom.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/drivers/misc/qseecom.c b/drivers/misc/qseecom.c index c9bf7f8a7a4..96b1eb1bf9f 100644 --- a/drivers/misc/qseecom.c +++ b/drivers/misc/qseecom.c @@ -3007,6 +3007,7 @@ static int qseecom_query_app_loaded(struct qseecom_dev_handle *data, struct qseecom_check_app_ireq req; struct qseecom_registered_app_list *entry = NULL; unsigned long flags = 0; + bool found_app = false; /* Copy the relevant information needed for loading the image */ if (copy_from_user(&query_req, @@ -3033,6 +3034,7 @@ static int qseecom_query_app_loaded(struct qseecom_dev_handle *data, &qseecom.registered_app_list_head, list){ if (entry->app_id == ret) { entry->ref_cnt++; + found_app = true; break; } } @@ -3042,6 +3044,29 @@ static int qseecom_query_app_loaded(struct qseecom_dev_handle *data, query_req.app_id = ret; strlcpy(data->client.app_name, query_req.app_name, MAX_APP_NAME_SIZE); + /* + * If app was loaded by appsbl or kernel client before + * and was not registered, regiser this app now. + */ + if (!found_app) { + pr_debug("Register app %d [%s] which was loaded before\n", + ret, (char *)query_req.app_name); + entry = kmalloc(sizeof(*entry), GFP_KERNEL); + if (!entry) { + pr_err("kmalloc for app entry failed\n"); + return -ENOMEM; + } + entry->app_id = ret; + entry->ref_cnt = 1; + strlcpy(entry->app_name, data->client.app_name, + MAX_APP_NAME_SIZE); + spin_lock_irqsave(&qseecom.registered_app_list_lock, + flags); + list_add_tail(&entry->list, + &qseecom.registered_app_list_head); + spin_unlock_irqrestore( + &qseecom.registered_app_list_lock, flags); + } if (copy_to_user(argp, &query_req, sizeof(query_req))) { pr_err("copy_to_user failed\n"); return -EFAULT; From dd99e80e0c8a7270faed35dd35f5bb65888269eb Mon Sep 17 00:00:00 2001 From: Vijayavardhan Vennapusa Date: Wed, 16 Dec 2015 12:42:53 +0530 Subject: [PATCH 039/168] usb: gadget: uac1: Fix redefinition conflict for ac interface Fix redefinition conflict for ac interface related descriptors' variable names (ac_interface_desc and ac_header_desc) with f_midi.c and f_uac1.c files by renaming same in f_uac1.c Change-Id: Ie10dba1a9fad2cf76116adc162889e6339736c76 Signed-off-by: Mayank Rana Signed-off-by: Vijayavardhan Vennapusa --- drivers/usb/gadget/f_uac1.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/drivers/usb/gadget/f_uac1.c b/drivers/usb/gadget/f_uac1.c index 883c5fa50b5..29e32538b4d 100644 --- a/drivers/usb/gadget/f_uac1.c +++ b/drivers/usb/gadget/f_uac1.c @@ -79,7 +79,7 @@ static int generic_get_cmd(struct usb_audio_control *con, u8 cmd); #define F_AUDIO_NUM_INTERFACES 2 /* B.3.1 Standard AC Interface Descriptor */ -struct usb_interface_descriptor ac_interface_desc = { +struct usb_interface_descriptor uac1_ac_interface_desc = { .bLength = USB_DT_INTERFACE_SIZE, .bDescriptorType = USB_DT_INTERFACE, .bNumEndpoints = 0, @@ -96,7 +96,7 @@ struct usb_interface_descriptor ac_interface_desc = { ) /* B.3.2 Class-Specific AC Interface Descriptor */ -struct uac1_ac_header_descriptor_2 ac_header_desc = { +struct uac1_ac_header_descriptor_2 uac1_ac_header_desc = { .bLength = UAC_DT_AC_HEADER_SIZE(2), .bDescriptorType = USB_DT_CS_INTERFACE, .bDescriptorSubtype = UAC_HEADER, @@ -361,8 +361,8 @@ static struct usb_audio_control_selector microphone_as_iso_in = { /*--------------------------------- */ static struct usb_descriptor_header *f_audio_desc[] = { - (struct usb_descriptor_header *)&ac_interface_desc, - (struct usb_descriptor_header *)&ac_header_desc, + (struct usb_descriptor_header *)&uac1_ac_interface_desc, + (struct usb_descriptor_header *)&uac1_ac_header_desc, (struct usb_descriptor_header *)µphone_input_terminal_desc, (struct usb_descriptor_header *)µphone_output_terminal_desc, @@ -887,9 +887,9 @@ static int f_audio_get_alt(struct usb_function *f, unsigned intf) { struct f_audio *audio = func_to_audio(f); - if (intf == ac_header_desc.baInterfaceNr[0]) + if (intf == uac1_ac_header_desc.baInterfaceNr[0]) return audio->alt_intf[0]; - if (intf == ac_header_desc.baInterfaceNr[1]) + if (intf == uac1_ac_header_desc.baInterfaceNr[1]) return audio->alt_intf[1]; return 0; @@ -906,7 +906,7 @@ static int f_audio_set_alt(struct usb_function *f, unsigned intf, unsigned alt) pr_debug("intf %d, alt %d\n", intf, alt); - if (intf == ac_header_desc.baInterfaceNr[0]) { + if (intf == uac1_ac_header_desc.baInterfaceNr[0]) { if (alt == 1) { err = usb_ep_enable(in_ep); if (err) { @@ -953,7 +953,7 @@ static int f_audio_set_alt(struct usb_function *f, unsigned intf, unsigned alt) spin_unlock_irqrestore(&audio->capture_lock, flags); } audio->alt_intf[0] = alt; - } else if (intf == ac_header_desc.baInterfaceNr[1]) { + } else if (intf == uac1_ac_header_desc.baInterfaceNr[1]) { if (alt == 1) { err = usb_ep_enable(out_ep); if (err) { @@ -1071,7 +1071,7 @@ f_audio_bind(struct usb_configuration *c, struct usb_function *f) pr_err("%s: failed to allocate desc interface", __func__); goto fail; } - ac_interface_desc.bInterfaceNumber = status; + uac1_ac_interface_desc.bInterfaceNumber = status; status = -ENOMEM; @@ -1082,7 +1082,7 @@ f_audio_bind(struct usb_configuration *c, struct usb_function *f) } microphone_as_interface_alt_0_desc.bInterfaceNumber = status; microphone_as_interface_alt_1_desc.bInterfaceNumber = status; - ac_header_desc.baInterfaceNr[0] = status; + uac1_ac_header_desc.baInterfaceNr[0] = status; audio->alt_intf[0] = 0; status = -ENODEV; @@ -1094,7 +1094,7 @@ f_audio_bind(struct usb_configuration *c, struct usb_function *f) } speaker_as_interface_alt_0_desc.bInterfaceNumber = status; speaker_as_interface_alt_1_desc.bInterfaceNumber = status; - ac_header_desc.baInterfaceNr[1] = status; + uac1_ac_header_desc.baInterfaceNr[1] = status; audio->alt_intf[1] = 0; status = -ENODEV; From 3b45962f52eb05335ce84c0cc8aff5c4fe9f59bb Mon Sep 17 00:00:00 2001 From: Mike Lockwood Date: Thu, 29 Jan 2015 13:52:51 -0800 Subject: [PATCH 040/168] USB: gadget: android: Integrate f_midi USB MIDI gadget driver Change-Id: I4c4b99f9d54314fc31445cf42b825527ca483af9 Signed-off-by: Mike Lockwood Git-commit: 3f048135f3028e503d9d6e32b602e362bd9f374f Git-repo: https://android.googlesource.com/kernel/common/ [mrana@codeaurora.org: Fix minor merged conflict, compilation error] Signed-off-by: Mayank Rana Signed-off-by: Vijayavardhan Vennapusa --- drivers/usb/gadget/android.c | 66 ++++++++++++++++++++++++++++++++++++ drivers/usb/gadget/f_midi.c | 40 +++++++++++++++------- 2 files changed, 94 insertions(+), 12 deletions(-) diff --git a/drivers/usb/gadget/android.c b/drivers/usb/gadget/android.c index 4de97f56ff8..e80e5a2caa2 100644 --- a/drivers/usb/gadget/android.c +++ b/drivers/usb/gadget/android.c @@ -47,6 +47,9 @@ #include "epautoconf.c" #include "composite.c" +#ifdef CONFIG_SND_RAWMIDI +#include "f_midi.c" +#endif #include "f_diag.c" #include "f_qdss.c" #include "f_rmnet_smd.c" @@ -105,6 +108,11 @@ static const char longname[] = "Gadget Android"; #define PRODUCT_ID 0x0001 #define ANDROID_DEVICE_NODE_NAME_LENGTH 11 +/* f_midi configuration */ +#define MIDI_INPUT_PORTS 1 +#define MIDI_OUTPUT_PORTS 1 +#define MIDI_BUFFER_SIZE 256 +#define MIDI_QUEUE_LENGTH 32 struct android_usb_function { char *name; @@ -2029,6 +2037,61 @@ static struct android_usb_function uasp_function = { .bind_config = uasp_function_bind_config, }; +#ifdef CONFIG_SND_RAWMIDI +static int midi_function_init(struct android_usb_function *f, + struct usb_composite_dev *cdev) +{ + struct midi_alsa_config *config; + + config = kzalloc(sizeof(struct midi_alsa_config), GFP_KERNEL); + f->config = config; + if (!config) + return -ENOMEM; + config->card = -1; + config->device = -1; + return 0; +} + +static void midi_function_cleanup(struct android_usb_function *f) +{ + kfree(f->config); +} + +static int midi_function_bind_config(struct android_usb_function *f, + struct usb_configuration *c) +{ + struct midi_alsa_config *config = f->config; + + return f_midi_bind_config(c, SNDRV_DEFAULT_IDX1, SNDRV_DEFAULT_STR1, + MIDI_INPUT_PORTS, MIDI_OUTPUT_PORTS, MIDI_BUFFER_SIZE, + MIDI_QUEUE_LENGTH, config); +} + +static ssize_t midi_alsa_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct android_usb_function *f = dev_get_drvdata(dev); + struct midi_alsa_config *config = f->config; + + /* print ALSA card and device numbers */ + return sprintf(buf, "%d %d\n", config->card, config->device); +} + +static DEVICE_ATTR(alsa, S_IRUGO, midi_alsa_show, NULL); + +static struct device_attribute *midi_function_attributes[] = { + &dev_attr_alsa, + NULL +}; + +static struct android_usb_function midi_function = { + .name = "midi", + .init = midi_function_init, + .cleanup = midi_function_cleanup, + .bind_config = midi_function_bind_config, + .attributes = midi_function_attributes, +}; +#endif static struct android_usb_function *supported_functions[] = { &mbim_function, &ecm_qc_function, @@ -2058,6 +2121,9 @@ static struct android_usb_function *supported_functions[] = { &audio_source_function, #endif &uasp_function, +#ifdef CONFIG_SND_RAWMIDI + &midi_function, +#endif NULL }; diff --git a/drivers/usb/gadget/f_midi.c b/drivers/usb/gadget/f_midi.c index 2f7e8f2930c..68370485ea5 100644 --- a/drivers/usb/gadget/f_midi.c +++ b/drivers/usb/gadget/f_midi.c @@ -66,6 +66,11 @@ struct gmidi_in_port { uint8_t data[2]; }; +struct midi_alsa_config { + int card; + int device; +}; + struct f_midi { struct usb_function func; struct usb_gadget *gadget; @@ -98,7 +103,7 @@ DECLARE_USB_MIDI_OUT_JACK_DESCRIPTOR(1); DECLARE_USB_MS_ENDPOINT_DESCRIPTOR(16); /* B.3.1 Standard AC Interface Descriptor */ -static struct usb_interface_descriptor ac_interface_desc __initdata = { +static struct usb_interface_descriptor ac_interface_desc /* __initdata */ = { .bLength = USB_DT_INTERFACE_SIZE, .bDescriptorType = USB_DT_INTERFACE, /* .bInterfaceNumber = DYNAMIC */ @@ -109,7 +114,7 @@ static struct usb_interface_descriptor ac_interface_desc __initdata = { }; /* B.3.2 Class-Specific AC Interface Descriptor */ -static struct uac1_ac_header_descriptor_1 ac_header_desc __initdata = { +static struct uac1_ac_header_descriptor_1 ac_header_desc /* __initdata */ = { .bLength = UAC_DT_AC_HEADER_SIZE(1), .bDescriptorType = USB_DT_CS_INTERFACE, .bDescriptorSubtype = USB_MS_HEADER, @@ -120,7 +125,7 @@ static struct uac1_ac_header_descriptor_1 ac_header_desc __initdata = { }; /* B.4.1 Standard MS Interface Descriptor */ -static struct usb_interface_descriptor ms_interface_desc __initdata = { +static struct usb_interface_descriptor ms_interface_desc /* __initdata */ = { .bLength = USB_DT_INTERFACE_SIZE, .bDescriptorType = USB_DT_INTERFACE, /* .bInterfaceNumber = DYNAMIC */ @@ -131,7 +136,7 @@ static struct usb_interface_descriptor ms_interface_desc __initdata = { }; /* B.4.2 Class-Specific MS Interface Descriptor */ -static struct usb_ms_header_descriptor ms_header_desc __initdata = { +static struct usb_ms_header_descriptor ms_header_desc /* __initdata */ = { .bLength = USB_DT_MS_HEADER_SIZE, .bDescriptorType = USB_DT_CS_INTERFACE, .bDescriptorSubtype = USB_MS_HEADER, @@ -734,7 +739,7 @@ fail: /* MIDI function driver setup/binding */ -static int __init +static int /* __init */ f_midi_bind(struct usb_configuration *c, struct usb_function *f) { struct usb_descriptor_header **midi_function; @@ -918,16 +923,22 @@ fail: * * Returns zero on success, else negative errno. */ -int __init f_midi_bind_config(struct usb_configuration *c, +int /* __init */ f_midi_bind_config(struct usb_configuration *c, int index, char *id, unsigned int in_ports, unsigned int out_ports, unsigned int buflen, - unsigned int qlen) + unsigned int qlen, + struct midi_alsa_config* config) { struct f_midi *midi; int status, i; + if (config) { + config->card = -1; + config->device = -1; + } + /* sanity check */ if (in_ports > MAX_PORTS || out_ports > MAX_PORTS) return -EINVAL; @@ -956,6 +967,10 @@ int __init f_midi_bind_config(struct usb_configuration *c, tasklet_init(&midi->tasklet, f_midi_in_tasklet, (unsigned long) midi); /* set up ALSA midi devices */ + midi->id = kstrdup(id, GFP_KERNEL); + midi->index = index; + midi->buflen = buflen; + midi->qlen = qlen; midi->in_ports = in_ports; midi->out_ports = out_ports; status = f_midi_register_card(midi); @@ -969,15 +984,16 @@ int __init f_midi_bind_config(struct usb_configuration *c, midi->func.set_alt = f_midi_set_alt; midi->func.disable = f_midi_disable; - midi->id = kstrdup(id, GFP_KERNEL); - midi->index = index; - midi->buflen = buflen; - midi->qlen = qlen; - status = usb_add_function(c, &midi->func); if (status) goto setup_fail; + + if (config) { + config->card = midi->rmidi->card->number; + config->device = midi->rmidi->device; + } + return 0; setup_fail: From d19cd47d2b57f60decde37c89ccaa3da3472ffbc Mon Sep 17 00:00:00 2001 From: Mark Salyzyn Date: Tue, 24 Mar 2015 12:42:35 -0700 Subject: [PATCH 041/168] USB: gadget: android: Integrate f_midi USB MIDI gadget driver part deux free_ep_req and alloc_ep_req are static open coded and conflicting in f_midi.c. The exported versions are present in f_sourcesink.c. Changed names to protect the innocent. Change-Id: I4aee40054b5715d0532d433d23dea2bccff7ec30 Signed-off-by: Mark Salyzyn Git-commit: ae2019d83ecf14454315240c16dc3136212f0da2 Git-repo: https://android.googlesource.com/kernel/common/ Signed-off-by: Mayank Rana --- drivers/usb/gadget/f_midi.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/usb/gadget/f_midi.c b/drivers/usb/gadget/f_midi.c index 68370485ea5..e736767ddd7 100644 --- a/drivers/usb/gadget/f_midi.c +++ b/drivers/usb/gadget/f_midi.c @@ -197,7 +197,7 @@ static struct usb_gadget_strings *midi_strings[] = { NULL, }; -static struct usb_request *alloc_ep_req(struct usb_ep *ep, unsigned length) +static struct usb_request *midi_alloc_ep_req(struct usb_ep *ep, unsigned length) { struct usb_request *req; @@ -213,7 +213,7 @@ static struct usb_request *alloc_ep_req(struct usb_ep *ep, unsigned length) return req; } -static void free_ep_req(struct usb_ep *ep, struct usb_request *req) +static void midi_free_ep_req(struct usb_ep *ep, struct usb_request *req) { kfree(req->buf); usb_ep_free_request(ep, req); @@ -284,7 +284,7 @@ f_midi_complete(struct usb_ep *ep, struct usb_request *req) if (ep == midi->out_ep) f_midi_handle_out_data(ep, req); - free_ep_req(ep, req); + midi_free_ep_req(ep, req); return; case -EOVERFLOW: /* buffer overrun on read means that @@ -371,7 +371,7 @@ static int f_midi_set_alt(struct usb_function *f, unsigned intf, unsigned alt) /* allocate a bunch of read buffers and queue them all at once. */ for (i = 0; i < midi->qlen && err == 0; i++) { struct usb_request *req = - alloc_ep_req(midi->out_ep, midi->buflen); + midi_alloc_ep_req(midi->out_ep, midi->buflen); if (req == NULL) return -ENOMEM; @@ -552,10 +552,10 @@ static void f_midi_transmit(struct f_midi *midi, struct usb_request *req) return; if (!req) - req = alloc_ep_req(ep, midi->buflen); + req = midi_alloc_ep_req(ep, midi->buflen); if (!req) { - ERROR(midi, "gmidi_transmit: alloc_ep_request failed\n"); + ERROR(midi, "gmidi_transmit: midi_alloc_ep_request failed\n"); return; } req->length = 0; @@ -581,7 +581,7 @@ static void f_midi_transmit(struct f_midi *midi, struct usb_request *req) if (req->length > 0) usb_ep_queue(ep, req, GFP_ATOMIC); else - free_ep_req(ep, req); + midi_free_ep_req(ep, req); } static void f_midi_in_tasklet(unsigned long data) From 72bc44e06a0fd3eca93829e8d6dd55fdec996bd9 Mon Sep 17 00:00:00 2001 From: Mayank Rana Date: Mon, 24 Aug 2015 18:08:39 -0700 Subject: [PATCH 042/168] USB: gadget: android: Fix checkpatch related errors Fix checkpatch error by using snprintf() instead of sprintf(). Also fix checkpatch error by using struct midi_alsa_config *config instead of struct midi_alsa_config* config. Change-Id: I093a90971de0a5482a80bc0e5206ac325f11bc92 Signed-off-by: Mayank Rana --- drivers/usb/gadget/android.c | 3 ++- drivers/usb/gadget/f_midi.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/usb/gadget/android.c b/drivers/usb/gadget/android.c index e80e5a2caa2..ce8f12420e6 100644 --- a/drivers/usb/gadget/android.c +++ b/drivers/usb/gadget/android.c @@ -1971,7 +1971,8 @@ static ssize_t audio_source_pcm_show(struct device *dev, struct audio_source_config *config = f->config; /* print PCM card and device numbers */ - return sprintf(buf, "%d %d\n", config->card, config->device); + return snprintf(buf, PAGE_SIZE, + "%d %d\n", config->card, config->device); } static DEVICE_ATTR(pcm, S_IRUGO | S_IWUSR, audio_source_pcm_show, NULL); diff --git a/drivers/usb/gadget/f_midi.c b/drivers/usb/gadget/f_midi.c index e736767ddd7..18381897858 100644 --- a/drivers/usb/gadget/f_midi.c +++ b/drivers/usb/gadget/f_midi.c @@ -929,7 +929,7 @@ int /* __init */ f_midi_bind_config(struct usb_configuration *c, unsigned int out_ports, unsigned int buflen, unsigned int qlen, - struct midi_alsa_config* config) + struct midi_alsa_config *config) { struct f_midi *midi; int status, i; From dbd15a7b8ab36b8293851ff67725b6bf6c68ac5c Mon Sep 17 00:00:00 2001 From: Mayank Rana Date: Mon, 31 Aug 2015 12:36:13 -0700 Subject: [PATCH 043/168] usb: gadget: android: Use 1024 bytes as MIDI buffer size Some UDC driver doesn't allow or throw errors when USB request buffer queued is unaligned i.e. buffer size is not equal to wMaxPacketSize or multiple of wMaxPacketSize. Fix this issue by using 1024 bytes as MIDI buffer size instead of 256 bytes. Change-Id: Ib3493f219de242be56be41d129d128facfe4a8a5 Signed-off-by: Mayank Rana --- drivers/usb/gadget/android.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/usb/gadget/android.c b/drivers/usb/gadget/android.c index ce8f12420e6..13564d40f00 100644 --- a/drivers/usb/gadget/android.c +++ b/drivers/usb/gadget/android.c @@ -111,7 +111,7 @@ static const char longname[] = "Gadget Android"; /* f_midi configuration */ #define MIDI_INPUT_PORTS 1 #define MIDI_OUTPUT_PORTS 1 -#define MIDI_BUFFER_SIZE 256 +#define MIDI_BUFFER_SIZE 1024 #define MIDI_QUEUE_LENGTH 32 struct android_usb_function { From 84b408625d3a946a1900caca3cdcbd4c86e4617f Mon Sep 17 00:00:00 2001 From: Mayank Rana Date: Tue, 1 Sep 2015 11:55:51 -0700 Subject: [PATCH 044/168] gadget: composite: Fix crash seen when SS descriptor is not available In some cases, composite device may have multiple interfaces and one of interface may not support USB super speed functionality. When host is trying to enumerate this composite device into super speed mode by sending SET_CONFIG(1), it results into crash due to ss descriptors are not available for particular interface. Hence add check against ss descriptors and fail SET_CONFIG(1). This results into some of host try to re-enumerate USB composite device with only interface which support super speed mode. Change-Id: Ib909f3d451e247c0415863689711e66b15fa2fa5 Signed-off-by: Mayank Rana --- drivers/usb/gadget/composite.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/usb/gadget/composite.c b/drivers/usb/gadget/composite.c index fe0b20fa617..b86a9e6b4a4 100644 --- a/drivers/usb/gadget/composite.c +++ b/drivers/usb/gadget/composite.c @@ -661,6 +661,11 @@ static int set_config(struct usb_composite_dev *cdev, */ switch (gadget->speed) { case USB_SPEED_SUPER: + if (!f->ss_descriptors) { + pr_err("%s(): No SS desc for function:%s\n", + __func__, f->name); + return -EINVAL; + } descriptors = f->ss_descriptors; break; case USB_SPEED_HIGH: From 2bc2bf8ac2e0934d55e42efba6a55d1d96b962ad Mon Sep 17 00:00:00 2001 From: Ravinder Konka Date: Wed, 16 Dec 2015 20:37:59 +0530 Subject: [PATCH 045/168] msm8974_defconfig: Enable NFLOG target support StrictModeTest#testCleartextNetwork CTS testcase is failing due to missing NFLOG target support related defconfig. Enabling the defconfig for 8974. Change-Id: Idb20e33bfb553c5aa4e648d8264f7185fa6f8547 CRs-Fixed: 916129 Signed-off-by: Ravinder Konka --- arch/arm/configs/msm8974-perf_defconfig | 1 + arch/arm/configs/msm8974_defconfig | 1 + 2 files changed, 2 insertions(+) diff --git a/arch/arm/configs/msm8974-perf_defconfig b/arch/arm/configs/msm8974-perf_defconfig index fb9537e821e..5e51adb38fa 100755 --- a/arch/arm/configs/msm8974-perf_defconfig +++ b/arch/arm/configs/msm8974-perf_defconfig @@ -167,6 +167,7 @@ CONFIG_NETFILTER_XT_TARGET_IDLETIMER=y CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=y CONFIG_NETFILTER_XT_TARGET_LOG=y CONFIG_NETFILTER_XT_TARGET_MARK=y +CONFIG_NETFILTER_XT_TARGET_NFLOG=y CONFIG_NETFILTER_XT_TARGET_NFQUEUE=y CONFIG_NETFILTER_XT_TARGET_NOTRACK=y CONFIG_NETFILTER_XT_TARGET_SECMARK=y diff --git a/arch/arm/configs/msm8974_defconfig b/arch/arm/configs/msm8974_defconfig index b3931679c09..82791a332bb 100755 --- a/arch/arm/configs/msm8974_defconfig +++ b/arch/arm/configs/msm8974_defconfig @@ -173,6 +173,7 @@ CONFIG_NETFILTER_XT_TARGET_CONNMARK=y CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=y CONFIG_NETFILTER_XT_TARGET_LOG=y CONFIG_NETFILTER_XT_TARGET_MARK=y +CONFIG_NETFILTER_XT_TARGET_NFLOG=y CONFIG_NETFILTER_XT_TARGET_NFQUEUE=y CONFIG_NETFILTER_XT_TARGET_NOTRACK=y CONFIG_NETFILTER_XT_TARGET_SECMARK=y From 274f0b63bc3e6d7a7d1b6e3af015221830fd574b Mon Sep 17 00:00:00 2001 From: Ravinder Konka Date: Mon, 25 Jun 2012 15:42:26 +0000 Subject: [PATCH 046/168] net/ipv6/route.c: packets originating on device match lo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix to allow IPv6 packets originating locally to match rules with the "iff" set to "lo". This allows IPv6 rule matching work the same as it does for IPv4. From the iproute2 man page: iif NAME select the incoming device to match. If the interface is loop‐ back, the rule only matches packets originating from this host. This means that you may create separate routing tables for for‐ warded and local packets and, hence, completely segregate them. Signed-off-by: David McCullough Signed-off-by: David S. Miller (cherry picked from commit 4dc27d1cf3b3027b9ce654221c559e88b2f41b33) Change-Id: Ic62766e5c2d197446c67a0f225f10f39d70f09ea Signed-off-by: Ravinder Konka --- net/ipv6/route.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/net/ipv6/route.c b/net/ipv6/route.c index 3e6b53b981a..ab71b41e139 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -928,6 +928,8 @@ struct dst_entry * ip6_route_output(struct net *net, const struct sock *sk, { int flags = 0; + fl6->flowi6_iif = net->loopback_dev->ifindex; + if ((sk && sk->sk_bound_dev_if) || rt6_need_strict(&fl6->daddr)) flags |= RT6_LOOKUP_F_IFACE; From c601250d2316a21add670d42bd27170d843d8f1d Mon Sep 17 00:00:00 2001 From: Shreyas Narayan Date: Thu, 17 Dec 2015 21:24:02 -0800 Subject: [PATCH 047/168] Revert "net/ipv6/route.c: packets originating on device match lo" This reverts commit 274f0b63bc3e6d7a7d1b6e3af015221830fd574b Change-Id: I080ab5c017dfca3b55f03ca4784b32369431238e Signed-off-by: Ravinder Konka Signed-off-by: Shreyas Narayan --- net/ipv6/route.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/net/ipv6/route.c b/net/ipv6/route.c index ab71b41e139..3e6b53b981a 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -928,8 +928,6 @@ struct dst_entry * ip6_route_output(struct net *net, const struct sock *sk, { int flags = 0; - fl6->flowi6_iif = net->loopback_dev->ifindex; - if ((sk && sk->sk_bound_dev_if) || rt6_need_strict(&fl6->daddr)) flags |= RT6_LOOKUP_F_IFACE; From d12e596c511e719e4663b960d33431ec55497b2f Mon Sep 17 00:00:00 2001 From: Abhimanyu Kapur Date: Sun, 1 Jun 2014 20:12:43 -0700 Subject: [PATCH 048/168] serial: tty: msm_hs_uart: Fix the compat table The compat tables in the msm_hs and msm_hsl drivers do not have a null terminator structure at the end of the array. Add them. Change-Id: I86bd6f92805328099126ef852fca1eb18cbe4670 Signed-off-by: Abhimanyu Kapur --- drivers/tty/serial/msm_serial_hs.c | 4 ++-- drivers/tty/serial/msm_serial_hs_lite.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/tty/serial/msm_serial_hs.c b/drivers/tty/serial/msm_serial_hs.c index 8545ce9d987..abf7619d9ea 100644 --- a/drivers/tty/serial/msm_serial_hs.c +++ b/drivers/tty/serial/msm_serial_hs.c @@ -253,8 +253,8 @@ struct msm_hs_port { }; static struct of_device_id msm_hs_match_table[] = { - { .compatible = "qcom,msm-hsuart-v14", - }, + { .compatible = "qcom,msm-hsuart-v14"}, + {} }; diff --git a/drivers/tty/serial/msm_serial_hs_lite.c b/drivers/tty/serial/msm_serial_hs_lite.c index 0ff8ad70106..e1bf06c0815 100644 --- a/drivers/tty/serial/msm_serial_hs_lite.c +++ b/drivers/tty/serial/msm_serial_hs_lite.c @@ -152,7 +152,7 @@ static const unsigned int regmap[][UARTDM_LAST] = { static struct of_device_id msm_hsl_match_table[] = { { .compatible = "qcom,msm-lsuart-v14", - .data = (void *)UARTDM_VERSION_14 + .data = (void *)UARTDM_VERSION_14, }, {} }; From eba90218e74bc228e6c956d3e459749bd747fcba Mon Sep 17 00:00:00 2001 From: David McCullough Date: Fri, 7 Sep 2012 04:17:02 +0800 Subject: [PATCH 049/168] arm/crypto: Add optimized AES and SHA1 routines Add assembler versions of AES and SHA1 for ARM platforms. This has provided up to a 50% improvement in IPsec/TCP throughout for tunnels using AES128/SHA1. Platform CPU SPeed Endian Before (bps) After (bps) Improvement IXP425 533 MHz big 11217042 15566294 ~38% KS8695 166 MHz little 3828549 5795373 ~51% Change-Id: I4a1bcedabe2d464de347630292d9d1ec8532b1ab Signed-off-by: David McCullough Signed-off-by: Herbert Xu Git-commit: 37e0ece9c34965ae137573e69ca3dcf0922f77ef Git-repo: https://android.googlesource.com/kernel/common/ Signed-off-by: Samir Mehta --- arch/arm/Makefile | 1 + arch/arm/crypto/Makefile | 9 + arch/arm/crypto/aes-armv4.S | 1112 ++++++++++++++++++++++++++++ arch/arm/crypto/aes_glue.c | 108 +++ arch/arm/crypto/sha1-armv4-large.S | 503 +++++++++++++ arch/arm/crypto/sha1_glue.c | 179 +++++ crypto/Kconfig | 33 + 7 files changed, 1945 insertions(+) create mode 100644 arch/arm/crypto/Makefile create mode 100644 arch/arm/crypto/aes-armv4.S create mode 100644 arch/arm/crypto/aes_glue.c create mode 100644 arch/arm/crypto/sha1-armv4-large.S create mode 100644 arch/arm/crypto/sha1_glue.c diff --git a/arch/arm/Makefile b/arch/arm/Makefile index e4096309609..44e180ec027 100644 --- a/arch/arm/Makefile +++ b/arch/arm/Makefile @@ -264,6 +264,7 @@ core-$(CONFIG_VFP) += arch/arm/vfp/ # If we have a machine-specific directory, then include it in the build. core-y += arch/arm/kernel/ arch/arm/mm/ arch/arm/common/ core-y += arch/arm/net/ +core-y += arch/arm/crypto/ core-y += $(machdirs) $(platdirs) drivers-$(CONFIG_OPROFILE) += arch/arm/oprofile/ diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile new file mode 100644 index 00000000000..a2c83851bc9 --- /dev/null +++ b/arch/arm/crypto/Makefile @@ -0,0 +1,9 @@ +# +# Arch-specific CryptoAPI modules. +# + +obj-$(CONFIG_CRYPTO_AES_ARM) += aes-arm.o +obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o + +aes-arm-y := aes-armv4.o aes_glue.o +sha1-arm-y := sha1-armv4-large.o sha1_glue.o diff --git a/arch/arm/crypto/aes-armv4.S b/arch/arm/crypto/aes-armv4.S new file mode 100644 index 00000000000..e59b1d505d6 --- /dev/null +++ b/arch/arm/crypto/aes-armv4.S @@ -0,0 +1,1112 @@ +#define __ARM_ARCH__ __LINUX_ARM_ARCH__ +@ ==================================================================== +@ Written by Andy Polyakov for the OpenSSL +@ project. The module is, however, dual licensed under OpenSSL and +@ CRYPTOGAMS licenses depending on where you obtain it. For further +@ details see http://www.openssl.org/~appro/cryptogams/. +@ ==================================================================== + +@ AES for ARMv4 + +@ January 2007. +@ +@ Code uses single 1K S-box and is >2 times faster than code generated +@ by gcc-3.4.1. This is thanks to unique feature of ARMv4 ISA, which +@ allows to merge logical or arithmetic operation with shift or rotate +@ in one instruction and emit combined result every cycle. The module +@ is endian-neutral. The performance is ~42 cycles/byte for 128-bit +@ key [on single-issue Xscale PXA250 core]. + +@ May 2007. +@ +@ AES_set_[en|de]crypt_key is added. + +@ July 2010. +@ +@ Rescheduling for dual-issue pipeline resulted in 12% improvement on +@ Cortex A8 core and ~25 cycles per byte processed with 128-bit key. + +@ February 2011. +@ +@ Profiler-assisted and platform-specific optimization resulted in 16% +@ improvement on Cortex A8 core and ~21.5 cycles per byte. + +@ A little glue here to select the correct code below for the ARM CPU +@ that is being targetted. + +.text +.code 32 + +.type AES_Te,%object +.align 5 +AES_Te: +.word 0xc66363a5, 0xf87c7c84, 0xee777799, 0xf67b7b8d +.word 0xfff2f20d, 0xd66b6bbd, 0xde6f6fb1, 0x91c5c554 +.word 0x60303050, 0x02010103, 0xce6767a9, 0x562b2b7d +.word 0xe7fefe19, 0xb5d7d762, 0x4dababe6, 0xec76769a +.word 0x8fcaca45, 0x1f82829d, 0x89c9c940, 0xfa7d7d87 +.word 0xeffafa15, 0xb25959eb, 0x8e4747c9, 0xfbf0f00b +.word 0x41adadec, 0xb3d4d467, 0x5fa2a2fd, 0x45afafea +.word 0x239c9cbf, 0x53a4a4f7, 0xe4727296, 0x9bc0c05b +.word 0x75b7b7c2, 0xe1fdfd1c, 0x3d9393ae, 0x4c26266a +.word 0x6c36365a, 0x7e3f3f41, 0xf5f7f702, 0x83cccc4f +.word 0x6834345c, 0x51a5a5f4, 0xd1e5e534, 0xf9f1f108 +.word 0xe2717193, 0xabd8d873, 0x62313153, 0x2a15153f +.word 0x0804040c, 0x95c7c752, 0x46232365, 0x9dc3c35e +.word 0x30181828, 0x379696a1, 0x0a05050f, 0x2f9a9ab5 +.word 0x0e070709, 0x24121236, 0x1b80809b, 0xdfe2e23d +.word 0xcdebeb26, 0x4e272769, 0x7fb2b2cd, 0xea75759f +.word 0x1209091b, 0x1d83839e, 0x582c2c74, 0x341a1a2e +.word 0x361b1b2d, 0xdc6e6eb2, 0xb45a5aee, 0x5ba0a0fb +.word 0xa45252f6, 0x763b3b4d, 0xb7d6d661, 0x7db3b3ce +.word 0x5229297b, 0xdde3e33e, 0x5e2f2f71, 0x13848497 +.word 0xa65353f5, 0xb9d1d168, 0x00000000, 0xc1eded2c +.word 0x40202060, 0xe3fcfc1f, 0x79b1b1c8, 0xb65b5bed +.word 0xd46a6abe, 0x8dcbcb46, 0x67bebed9, 0x7239394b +.word 0x944a4ade, 0x984c4cd4, 0xb05858e8, 0x85cfcf4a +.word 0xbbd0d06b, 0xc5efef2a, 0x4faaaae5, 0xedfbfb16 +.word 0x864343c5, 0x9a4d4dd7, 0x66333355, 0x11858594 +.word 0x8a4545cf, 0xe9f9f910, 0x04020206, 0xfe7f7f81 +.word 0xa05050f0, 0x783c3c44, 0x259f9fba, 0x4ba8a8e3 +.word 0xa25151f3, 0x5da3a3fe, 0x804040c0, 0x058f8f8a +.word 0x3f9292ad, 0x219d9dbc, 0x70383848, 0xf1f5f504 +.word 0x63bcbcdf, 0x77b6b6c1, 0xafdada75, 0x42212163 +.word 0x20101030, 0xe5ffff1a, 0xfdf3f30e, 0xbfd2d26d +.word 0x81cdcd4c, 0x180c0c14, 0x26131335, 0xc3ecec2f +.word 0xbe5f5fe1, 0x359797a2, 0x884444cc, 0x2e171739 +.word 0x93c4c457, 0x55a7a7f2, 0xfc7e7e82, 0x7a3d3d47 +.word 0xc86464ac, 0xba5d5de7, 0x3219192b, 0xe6737395 +.word 0xc06060a0, 0x19818198, 0x9e4f4fd1, 0xa3dcdc7f +.word 0x44222266, 0x542a2a7e, 0x3b9090ab, 0x0b888883 +.word 0x8c4646ca, 0xc7eeee29, 0x6bb8b8d3, 0x2814143c +.word 0xa7dede79, 0xbc5e5ee2, 0x160b0b1d, 0xaddbdb76 +.word 0xdbe0e03b, 0x64323256, 0x743a3a4e, 0x140a0a1e +.word 0x924949db, 0x0c06060a, 0x4824246c, 0xb85c5ce4 +.word 0x9fc2c25d, 0xbdd3d36e, 0x43acacef, 0xc46262a6 +.word 0x399191a8, 0x319595a4, 0xd3e4e437, 0xf279798b +.word 0xd5e7e732, 0x8bc8c843, 0x6e373759, 0xda6d6db7 +.word 0x018d8d8c, 0xb1d5d564, 0x9c4e4ed2, 0x49a9a9e0 +.word 0xd86c6cb4, 0xac5656fa, 0xf3f4f407, 0xcfeaea25 +.word 0xca6565af, 0xf47a7a8e, 0x47aeaee9, 0x10080818 +.word 0x6fbabad5, 0xf0787888, 0x4a25256f, 0x5c2e2e72 +.word 0x381c1c24, 0x57a6a6f1, 0x73b4b4c7, 0x97c6c651 +.word 0xcbe8e823, 0xa1dddd7c, 0xe874749c, 0x3e1f1f21 +.word 0x964b4bdd, 0x61bdbddc, 0x0d8b8b86, 0x0f8a8a85 +.word 0xe0707090, 0x7c3e3e42, 0x71b5b5c4, 0xcc6666aa +.word 0x904848d8, 0x06030305, 0xf7f6f601, 0x1c0e0e12 +.word 0xc26161a3, 0x6a35355f, 0xae5757f9, 0x69b9b9d0 +.word 0x17868691, 0x99c1c158, 0x3a1d1d27, 0x279e9eb9 +.word 0xd9e1e138, 0xebf8f813, 0x2b9898b3, 0x22111133 +.word 0xd26969bb, 0xa9d9d970, 0x078e8e89, 0x339494a7 +.word 0x2d9b9bb6, 0x3c1e1e22, 0x15878792, 0xc9e9e920 +.word 0x87cece49, 0xaa5555ff, 0x50282878, 0xa5dfdf7a +.word 0x038c8c8f, 0x59a1a1f8, 0x09898980, 0x1a0d0d17 +.word 0x65bfbfda, 0xd7e6e631, 0x844242c6, 0xd06868b8 +.word 0x824141c3, 0x299999b0, 0x5a2d2d77, 0x1e0f0f11 +.word 0x7bb0b0cb, 0xa85454fc, 0x6dbbbbd6, 0x2c16163a +@ Te4[256] +.byte 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5 +.byte 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76 +.byte 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0 +.byte 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0 +.byte 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc +.byte 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15 +.byte 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a +.byte 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75 +.byte 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0 +.byte 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84 +.byte 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b +.byte 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf +.byte 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85 +.byte 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8 +.byte 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5 +.byte 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2 +.byte 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17 +.byte 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73 +.byte 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88 +.byte 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb +.byte 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c +.byte 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79 +.byte 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9 +.byte 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08 +.byte 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6 +.byte 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a +.byte 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e +.byte 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e +.byte 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94 +.byte 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf +.byte 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68 +.byte 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16 +@ rcon[] +.word 0x01000000, 0x02000000, 0x04000000, 0x08000000 +.word 0x10000000, 0x20000000, 0x40000000, 0x80000000 +.word 0x1B000000, 0x36000000, 0, 0, 0, 0, 0, 0 +.size AES_Te,.-AES_Te + +@ void AES_encrypt(const unsigned char *in, unsigned char *out, +@ const AES_KEY *key) { +.global AES_encrypt +.type AES_encrypt,%function +.align 5 +AES_encrypt: + sub r3,pc,#8 @ AES_encrypt + stmdb sp!,{r1,r4-r12,lr} + mov r12,r0 @ inp + mov r11,r2 + sub r10,r3,#AES_encrypt-AES_Te @ Te +#if __ARM_ARCH__<7 + ldrb r0,[r12,#3] @ load input data in endian-neutral + ldrb r4,[r12,#2] @ manner... + ldrb r5,[r12,#1] + ldrb r6,[r12,#0] + orr r0,r0,r4,lsl#8 + ldrb r1,[r12,#7] + orr r0,r0,r5,lsl#16 + ldrb r4,[r12,#6] + orr r0,r0,r6,lsl#24 + ldrb r5,[r12,#5] + ldrb r6,[r12,#4] + orr r1,r1,r4,lsl#8 + ldrb r2,[r12,#11] + orr r1,r1,r5,lsl#16 + ldrb r4,[r12,#10] + orr r1,r1,r6,lsl#24 + ldrb r5,[r12,#9] + ldrb r6,[r12,#8] + orr r2,r2,r4,lsl#8 + ldrb r3,[r12,#15] + orr r2,r2,r5,lsl#16 + ldrb r4,[r12,#14] + orr r2,r2,r6,lsl#24 + ldrb r5,[r12,#13] + ldrb r6,[r12,#12] + orr r3,r3,r4,lsl#8 + orr r3,r3,r5,lsl#16 + orr r3,r3,r6,lsl#24 +#else + ldr r0,[r12,#0] + ldr r1,[r12,#4] + ldr r2,[r12,#8] + ldr r3,[r12,#12] +#ifdef __ARMEL__ + rev r0,r0 + rev r1,r1 + rev r2,r2 + rev r3,r3 +#endif +#endif + bl _armv4_AES_encrypt + + ldr r12,[sp],#4 @ pop out +#if __ARM_ARCH__>=7 +#ifdef __ARMEL__ + rev r0,r0 + rev r1,r1 + rev r2,r2 + rev r3,r3 +#endif + str r0,[r12,#0] + str r1,[r12,#4] + str r2,[r12,#8] + str r3,[r12,#12] +#else + mov r4,r0,lsr#24 @ write output in endian-neutral + mov r5,r0,lsr#16 @ manner... + mov r6,r0,lsr#8 + strb r4,[r12,#0] + strb r5,[r12,#1] + mov r4,r1,lsr#24 + strb r6,[r12,#2] + mov r5,r1,lsr#16 + strb r0,[r12,#3] + mov r6,r1,lsr#8 + strb r4,[r12,#4] + strb r5,[r12,#5] + mov r4,r2,lsr#24 + strb r6,[r12,#6] + mov r5,r2,lsr#16 + strb r1,[r12,#7] + mov r6,r2,lsr#8 + strb r4,[r12,#8] + strb r5,[r12,#9] + mov r4,r3,lsr#24 + strb r6,[r12,#10] + mov r5,r3,lsr#16 + strb r2,[r12,#11] + mov r6,r3,lsr#8 + strb r4,[r12,#12] + strb r5,[r12,#13] + strb r6,[r12,#14] + strb r3,[r12,#15] +#endif +#if __ARM_ARCH__>=5 + ldmia sp!,{r4-r12,pc} +#else + ldmia sp!,{r4-r12,lr} + tst lr,#1 + moveq pc,lr @ be binary compatible with V4, yet + .word 0xe12fff1e @ interoperable with Thumb ISA:-) +#endif +.size AES_encrypt,.-AES_encrypt + +.type _armv4_AES_encrypt,%function +.align 2 +_armv4_AES_encrypt: + str lr,[sp,#-4]! @ push lr + ldmia r11!,{r4-r7} + eor r0,r0,r4 + ldr r12,[r11,#240-16] + eor r1,r1,r5 + eor r2,r2,r6 + eor r3,r3,r7 + sub r12,r12,#1 + mov lr,#255 + + and r7,lr,r0 + and r8,lr,r0,lsr#8 + and r9,lr,r0,lsr#16 + mov r0,r0,lsr#24 +.Lenc_loop: + ldr r4,[r10,r7,lsl#2] @ Te3[s0>>0] + and r7,lr,r1,lsr#16 @ i0 + ldr r5,[r10,r8,lsl#2] @ Te2[s0>>8] + and r8,lr,r1 + ldr r6,[r10,r9,lsl#2] @ Te1[s0>>16] + and r9,lr,r1,lsr#8 + ldr r0,[r10,r0,lsl#2] @ Te0[s0>>24] + mov r1,r1,lsr#24 + + ldr r7,[r10,r7,lsl#2] @ Te1[s1>>16] + ldr r8,[r10,r8,lsl#2] @ Te3[s1>>0] + ldr r9,[r10,r9,lsl#2] @ Te2[s1>>8] + eor r0,r0,r7,ror#8 + ldr r1,[r10,r1,lsl#2] @ Te0[s1>>24] + and r7,lr,r2,lsr#8 @ i0 + eor r5,r5,r8,ror#8 + and r8,lr,r2,lsr#16 @ i1 + eor r6,r6,r9,ror#8 + and r9,lr,r2 + ldr r7,[r10,r7,lsl#2] @ Te2[s2>>8] + eor r1,r1,r4,ror#24 + ldr r8,[r10,r8,lsl#2] @ Te1[s2>>16] + mov r2,r2,lsr#24 + + ldr r9,[r10,r9,lsl#2] @ Te3[s2>>0] + eor r0,r0,r7,ror#16 + ldr r2,[r10,r2,lsl#2] @ Te0[s2>>24] + and r7,lr,r3 @ i0 + eor r1,r1,r8,ror#8 + and r8,lr,r3,lsr#8 @ i1 + eor r6,r6,r9,ror#16 + and r9,lr,r3,lsr#16 @ i2 + ldr r7,[r10,r7,lsl#2] @ Te3[s3>>0] + eor r2,r2,r5,ror#16 + ldr r8,[r10,r8,lsl#2] @ Te2[s3>>8] + mov r3,r3,lsr#24 + + ldr r9,[r10,r9,lsl#2] @ Te1[s3>>16] + eor r0,r0,r7,ror#24 + ldr r7,[r11],#16 + eor r1,r1,r8,ror#16 + ldr r3,[r10,r3,lsl#2] @ Te0[s3>>24] + eor r2,r2,r9,ror#8 + ldr r4,[r11,#-12] + eor r3,r3,r6,ror#8 + + ldr r5,[r11,#-8] + eor r0,r0,r7 + ldr r6,[r11,#-4] + and r7,lr,r0 + eor r1,r1,r4 + and r8,lr,r0,lsr#8 + eor r2,r2,r5 + and r9,lr,r0,lsr#16 + eor r3,r3,r6 + mov r0,r0,lsr#24 + + subs r12,r12,#1 + bne .Lenc_loop + + add r10,r10,#2 + + ldrb r4,[r10,r7,lsl#2] @ Te4[s0>>0] + and r7,lr,r1,lsr#16 @ i0 + ldrb r5,[r10,r8,lsl#2] @ Te4[s0>>8] + and r8,lr,r1 + ldrb r6,[r10,r9,lsl#2] @ Te4[s0>>16] + and r9,lr,r1,lsr#8 + ldrb r0,[r10,r0,lsl#2] @ Te4[s0>>24] + mov r1,r1,lsr#24 + + ldrb r7,[r10,r7,lsl#2] @ Te4[s1>>16] + ldrb r8,[r10,r8,lsl#2] @ Te4[s1>>0] + ldrb r9,[r10,r9,lsl#2] @ Te4[s1>>8] + eor r0,r7,r0,lsl#8 + ldrb r1,[r10,r1,lsl#2] @ Te4[s1>>24] + and r7,lr,r2,lsr#8 @ i0 + eor r5,r8,r5,lsl#8 + and r8,lr,r2,lsr#16 @ i1 + eor r6,r9,r6,lsl#8 + and r9,lr,r2 + ldrb r7,[r10,r7,lsl#2] @ Te4[s2>>8] + eor r1,r4,r1,lsl#24 + ldrb r8,[r10,r8,lsl#2] @ Te4[s2>>16] + mov r2,r2,lsr#24 + + ldrb r9,[r10,r9,lsl#2] @ Te4[s2>>0] + eor r0,r7,r0,lsl#8 + ldrb r2,[r10,r2,lsl#2] @ Te4[s2>>24] + and r7,lr,r3 @ i0 + eor r1,r1,r8,lsl#16 + and r8,lr,r3,lsr#8 @ i1 + eor r6,r9,r6,lsl#8 + and r9,lr,r3,lsr#16 @ i2 + ldrb r7,[r10,r7,lsl#2] @ Te4[s3>>0] + eor r2,r5,r2,lsl#24 + ldrb r8,[r10,r8,lsl#2] @ Te4[s3>>8] + mov r3,r3,lsr#24 + + ldrb r9,[r10,r9,lsl#2] @ Te4[s3>>16] + eor r0,r7,r0,lsl#8 + ldr r7,[r11,#0] + ldrb r3,[r10,r3,lsl#2] @ Te4[s3>>24] + eor r1,r1,r8,lsl#8 + ldr r4,[r11,#4] + eor r2,r2,r9,lsl#16 + ldr r5,[r11,#8] + eor r3,r6,r3,lsl#24 + ldr r6,[r11,#12] + + eor r0,r0,r7 + eor r1,r1,r4 + eor r2,r2,r5 + eor r3,r3,r6 + + sub r10,r10,#2 + ldr pc,[sp],#4 @ pop and return +.size _armv4_AES_encrypt,.-_armv4_AES_encrypt + +.global private_AES_set_encrypt_key +.type private_AES_set_encrypt_key,%function +.align 5 +private_AES_set_encrypt_key: +_armv4_AES_set_encrypt_key: + sub r3,pc,#8 @ AES_set_encrypt_key + teq r0,#0 + moveq r0,#-1 + beq .Labrt + teq r2,#0 + moveq r0,#-1 + beq .Labrt + + teq r1,#128 + beq .Lok + teq r1,#192 + beq .Lok + teq r1,#256 + movne r0,#-1 + bne .Labrt + +.Lok: stmdb sp!,{r4-r12,lr} + sub r10,r3,#_armv4_AES_set_encrypt_key-AES_Te-1024 @ Te4 + + mov r12,r0 @ inp + mov lr,r1 @ bits + mov r11,r2 @ key + +#if __ARM_ARCH__<7 + ldrb r0,[r12,#3] @ load input data in endian-neutral + ldrb r4,[r12,#2] @ manner... + ldrb r5,[r12,#1] + ldrb r6,[r12,#0] + orr r0,r0,r4,lsl#8 + ldrb r1,[r12,#7] + orr r0,r0,r5,lsl#16 + ldrb r4,[r12,#6] + orr r0,r0,r6,lsl#24 + ldrb r5,[r12,#5] + ldrb r6,[r12,#4] + orr r1,r1,r4,lsl#8 + ldrb r2,[r12,#11] + orr r1,r1,r5,lsl#16 + ldrb r4,[r12,#10] + orr r1,r1,r6,lsl#24 + ldrb r5,[r12,#9] + ldrb r6,[r12,#8] + orr r2,r2,r4,lsl#8 + ldrb r3,[r12,#15] + orr r2,r2,r5,lsl#16 + ldrb r4,[r12,#14] + orr r2,r2,r6,lsl#24 + ldrb r5,[r12,#13] + ldrb r6,[r12,#12] + orr r3,r3,r4,lsl#8 + str r0,[r11],#16 + orr r3,r3,r5,lsl#16 + str r1,[r11,#-12] + orr r3,r3,r6,lsl#24 + str r2,[r11,#-8] + str r3,[r11,#-4] +#else + ldr r0,[r12,#0] + ldr r1,[r12,#4] + ldr r2,[r12,#8] + ldr r3,[r12,#12] +#ifdef __ARMEL__ + rev r0,r0 + rev r1,r1 + rev r2,r2 + rev r3,r3 +#endif + str r0,[r11],#16 + str r1,[r11,#-12] + str r2,[r11,#-8] + str r3,[r11,#-4] +#endif + + teq lr,#128 + bne .Lnot128 + mov r12,#10 + str r12,[r11,#240-16] + add r6,r10,#256 @ rcon + mov lr,#255 + +.L128_loop: + and r5,lr,r3,lsr#24 + and r7,lr,r3,lsr#16 + ldrb r5,[r10,r5] + and r8,lr,r3,lsr#8 + ldrb r7,[r10,r7] + and r9,lr,r3 + ldrb r8,[r10,r8] + orr r5,r5,r7,lsl#24 + ldrb r9,[r10,r9] + orr r5,r5,r8,lsl#16 + ldr r4,[r6],#4 @ rcon[i++] + orr r5,r5,r9,lsl#8 + eor r5,r5,r4 + eor r0,r0,r5 @ rk[4]=rk[0]^... + eor r1,r1,r0 @ rk[5]=rk[1]^rk[4] + str r0,[r11],#16 + eor r2,r2,r1 @ rk[6]=rk[2]^rk[5] + str r1,[r11,#-12] + eor r3,r3,r2 @ rk[7]=rk[3]^rk[6] + str r2,[r11,#-8] + subs r12,r12,#1 + str r3,[r11,#-4] + bne .L128_loop + sub r2,r11,#176 + b .Ldone + +.Lnot128: +#if __ARM_ARCH__<7 + ldrb r8,[r12,#19] + ldrb r4,[r12,#18] + ldrb r5,[r12,#17] + ldrb r6,[r12,#16] + orr r8,r8,r4,lsl#8 + ldrb r9,[r12,#23] + orr r8,r8,r5,lsl#16 + ldrb r4,[r12,#22] + orr r8,r8,r6,lsl#24 + ldrb r5,[r12,#21] + ldrb r6,[r12,#20] + orr r9,r9,r4,lsl#8 + orr r9,r9,r5,lsl#16 + str r8,[r11],#8 + orr r9,r9,r6,lsl#24 + str r9,[r11,#-4] +#else + ldr r8,[r12,#16] + ldr r9,[r12,#20] +#ifdef __ARMEL__ + rev r8,r8 + rev r9,r9 +#endif + str r8,[r11],#8 + str r9,[r11,#-4] +#endif + + teq lr,#192 + bne .Lnot192 + mov r12,#12 + str r12,[r11,#240-24] + add r6,r10,#256 @ rcon + mov lr,#255 + mov r12,#8 + +.L192_loop: + and r5,lr,r9,lsr#24 + and r7,lr,r9,lsr#16 + ldrb r5,[r10,r5] + and r8,lr,r9,lsr#8 + ldrb r7,[r10,r7] + and r9,lr,r9 + ldrb r8,[r10,r8] + orr r5,r5,r7,lsl#24 + ldrb r9,[r10,r9] + orr r5,r5,r8,lsl#16 + ldr r4,[r6],#4 @ rcon[i++] + orr r5,r5,r9,lsl#8 + eor r9,r5,r4 + eor r0,r0,r9 @ rk[6]=rk[0]^... + eor r1,r1,r0 @ rk[7]=rk[1]^rk[6] + str r0,[r11],#24 + eor r2,r2,r1 @ rk[8]=rk[2]^rk[7] + str r1,[r11,#-20] + eor r3,r3,r2 @ rk[9]=rk[3]^rk[8] + str r2,[r11,#-16] + subs r12,r12,#1 + str r3,[r11,#-12] + subeq r2,r11,#216 + beq .Ldone + + ldr r7,[r11,#-32] + ldr r8,[r11,#-28] + eor r7,r7,r3 @ rk[10]=rk[4]^rk[9] + eor r9,r8,r7 @ rk[11]=rk[5]^rk[10] + str r7,[r11,#-8] + str r9,[r11,#-4] + b .L192_loop + +.Lnot192: +#if __ARM_ARCH__<7 + ldrb r8,[r12,#27] + ldrb r4,[r12,#26] + ldrb r5,[r12,#25] + ldrb r6,[r12,#24] + orr r8,r8,r4,lsl#8 + ldrb r9,[r12,#31] + orr r8,r8,r5,lsl#16 + ldrb r4,[r12,#30] + orr r8,r8,r6,lsl#24 + ldrb r5,[r12,#29] + ldrb r6,[r12,#28] + orr r9,r9,r4,lsl#8 + orr r9,r9,r5,lsl#16 + str r8,[r11],#8 + orr r9,r9,r6,lsl#24 + str r9,[r11,#-4] +#else + ldr r8,[r12,#24] + ldr r9,[r12,#28] +#ifdef __ARMEL__ + rev r8,r8 + rev r9,r9 +#endif + str r8,[r11],#8 + str r9,[r11,#-4] +#endif + + mov r12,#14 + str r12,[r11,#240-32] + add r6,r10,#256 @ rcon + mov lr,#255 + mov r12,#7 + +.L256_loop: + and r5,lr,r9,lsr#24 + and r7,lr,r9,lsr#16 + ldrb r5,[r10,r5] + and r8,lr,r9,lsr#8 + ldrb r7,[r10,r7] + and r9,lr,r9 + ldrb r8,[r10,r8] + orr r5,r5,r7,lsl#24 + ldrb r9,[r10,r9] + orr r5,r5,r8,lsl#16 + ldr r4,[r6],#4 @ rcon[i++] + orr r5,r5,r9,lsl#8 + eor r9,r5,r4 + eor r0,r0,r9 @ rk[8]=rk[0]^... + eor r1,r1,r0 @ rk[9]=rk[1]^rk[8] + str r0,[r11],#32 + eor r2,r2,r1 @ rk[10]=rk[2]^rk[9] + str r1,[r11,#-28] + eor r3,r3,r2 @ rk[11]=rk[3]^rk[10] + str r2,[r11,#-24] + subs r12,r12,#1 + str r3,[r11,#-20] + subeq r2,r11,#256 + beq .Ldone + + and r5,lr,r3 + and r7,lr,r3,lsr#8 + ldrb r5,[r10,r5] + and r8,lr,r3,lsr#16 + ldrb r7,[r10,r7] + and r9,lr,r3,lsr#24 + ldrb r8,[r10,r8] + orr r5,r5,r7,lsl#8 + ldrb r9,[r10,r9] + orr r5,r5,r8,lsl#16 + ldr r4,[r11,#-48] + orr r5,r5,r9,lsl#24 + + ldr r7,[r11,#-44] + ldr r8,[r11,#-40] + eor r4,r4,r5 @ rk[12]=rk[4]^... + ldr r9,[r11,#-36] + eor r7,r7,r4 @ rk[13]=rk[5]^rk[12] + str r4,[r11,#-16] + eor r8,r8,r7 @ rk[14]=rk[6]^rk[13] + str r7,[r11,#-12] + eor r9,r9,r8 @ rk[15]=rk[7]^rk[14] + str r8,[r11,#-8] + str r9,[r11,#-4] + b .L256_loop + +.Ldone: mov r0,#0 + ldmia sp!,{r4-r12,lr} +.Labrt: tst lr,#1 + moveq pc,lr @ be binary compatible with V4, yet + .word 0xe12fff1e @ interoperable with Thumb ISA:-) +.size private_AES_set_encrypt_key,.-private_AES_set_encrypt_key + +.global private_AES_set_decrypt_key +.type private_AES_set_decrypt_key,%function +.align 5 +private_AES_set_decrypt_key: + str lr,[sp,#-4]! @ push lr +#if 0 + @ kernel does both of these in setkey so optimise this bit out by + @ expecting the key to already have the enc_key work done (see aes_glue.c) + bl _armv4_AES_set_encrypt_key +#else + mov r0,#0 +#endif + teq r0,#0 + ldrne lr,[sp],#4 @ pop lr + bne .Labrt + + stmdb sp!,{r4-r12} + + ldr r12,[r2,#240] @ AES_set_encrypt_key preserves r2, + mov r11,r2 @ which is AES_KEY *key + mov r7,r2 + add r8,r2,r12,lsl#4 + +.Linv: ldr r0,[r7] + ldr r1,[r7,#4] + ldr r2,[r7,#8] + ldr r3,[r7,#12] + ldr r4,[r8] + ldr r5,[r8,#4] + ldr r6,[r8,#8] + ldr r9,[r8,#12] + str r0,[r8],#-16 + str r1,[r8,#16+4] + str r2,[r8,#16+8] + str r3,[r8,#16+12] + str r4,[r7],#16 + str r5,[r7,#-12] + str r6,[r7,#-8] + str r9,[r7,#-4] + teq r7,r8 + bne .Linv + ldr r0,[r11,#16]! @ prefetch tp1 + mov r7,#0x80 + mov r8,#0x1b + orr r7,r7,#0x8000 + orr r8,r8,#0x1b00 + orr r7,r7,r7,lsl#16 + orr r8,r8,r8,lsl#16 + sub r12,r12,#1 + mvn r9,r7 + mov r12,r12,lsl#2 @ (rounds-1)*4 + +.Lmix: and r4,r0,r7 + and r1,r0,r9 + sub r4,r4,r4,lsr#7 + and r4,r4,r8 + eor r1,r4,r1,lsl#1 @ tp2 + + and r4,r1,r7 + and r2,r1,r9 + sub r4,r4,r4,lsr#7 + and r4,r4,r8 + eor r2,r4,r2,lsl#1 @ tp4 + + and r4,r2,r7 + and r3,r2,r9 + sub r4,r4,r4,lsr#7 + and r4,r4,r8 + eor r3,r4,r3,lsl#1 @ tp8 + + eor r4,r1,r2 + eor r5,r0,r3 @ tp9 + eor r4,r4,r3 @ tpe + eor r4,r4,r1,ror#24 + eor r4,r4,r5,ror#24 @ ^= ROTATE(tpb=tp9^tp2,8) + eor r4,r4,r2,ror#16 + eor r4,r4,r5,ror#16 @ ^= ROTATE(tpd=tp9^tp4,16) + eor r4,r4,r5,ror#8 @ ^= ROTATE(tp9,24) + + ldr r0,[r11,#4] @ prefetch tp1 + str r4,[r11],#4 + subs r12,r12,#1 + bne .Lmix + + mov r0,#0 +#if __ARM_ARCH__>=5 + ldmia sp!,{r4-r12,pc} +#else + ldmia sp!,{r4-r12,lr} + tst lr,#1 + moveq pc,lr @ be binary compatible with V4, yet + .word 0xe12fff1e @ interoperable with Thumb ISA:-) +#endif +.size private_AES_set_decrypt_key,.-private_AES_set_decrypt_key + +.type AES_Td,%object +.align 5 +AES_Td: +.word 0x51f4a750, 0x7e416553, 0x1a17a4c3, 0x3a275e96 +.word 0x3bab6bcb, 0x1f9d45f1, 0xacfa58ab, 0x4be30393 +.word 0x2030fa55, 0xad766df6, 0x88cc7691, 0xf5024c25 +.word 0x4fe5d7fc, 0xc52acbd7, 0x26354480, 0xb562a38f +.word 0xdeb15a49, 0x25ba1b67, 0x45ea0e98, 0x5dfec0e1 +.word 0xc32f7502, 0x814cf012, 0x8d4697a3, 0x6bd3f9c6 +.word 0x038f5fe7, 0x15929c95, 0xbf6d7aeb, 0x955259da +.word 0xd4be832d, 0x587421d3, 0x49e06929, 0x8ec9c844 +.word 0x75c2896a, 0xf48e7978, 0x99583e6b, 0x27b971dd +.word 0xbee14fb6, 0xf088ad17, 0xc920ac66, 0x7dce3ab4 +.word 0x63df4a18, 0xe51a3182, 0x97513360, 0x62537f45 +.word 0xb16477e0, 0xbb6bae84, 0xfe81a01c, 0xf9082b94 +.word 0x70486858, 0x8f45fd19, 0x94de6c87, 0x527bf8b7 +.word 0xab73d323, 0x724b02e2, 0xe31f8f57, 0x6655ab2a +.word 0xb2eb2807, 0x2fb5c203, 0x86c57b9a, 0xd33708a5 +.word 0x302887f2, 0x23bfa5b2, 0x02036aba, 0xed16825c +.word 0x8acf1c2b, 0xa779b492, 0xf307f2f0, 0x4e69e2a1 +.word 0x65daf4cd, 0x0605bed5, 0xd134621f, 0xc4a6fe8a +.word 0x342e539d, 0xa2f355a0, 0x058ae132, 0xa4f6eb75 +.word 0x0b83ec39, 0x4060efaa, 0x5e719f06, 0xbd6e1051 +.word 0x3e218af9, 0x96dd063d, 0xdd3e05ae, 0x4de6bd46 +.word 0x91548db5, 0x71c45d05, 0x0406d46f, 0x605015ff +.word 0x1998fb24, 0xd6bde997, 0x894043cc, 0x67d99e77 +.word 0xb0e842bd, 0x07898b88, 0xe7195b38, 0x79c8eedb +.word 0xa17c0a47, 0x7c420fe9, 0xf8841ec9, 0x00000000 +.word 0x09808683, 0x322bed48, 0x1e1170ac, 0x6c5a724e +.word 0xfd0efffb, 0x0f853856, 0x3daed51e, 0x362d3927 +.word 0x0a0fd964, 0x685ca621, 0x9b5b54d1, 0x24362e3a +.word 0x0c0a67b1, 0x9357e70f, 0xb4ee96d2, 0x1b9b919e +.word 0x80c0c54f, 0x61dc20a2, 0x5a774b69, 0x1c121a16 +.word 0xe293ba0a, 0xc0a02ae5, 0x3c22e043, 0x121b171d +.word 0x0e090d0b, 0xf28bc7ad, 0x2db6a8b9, 0x141ea9c8 +.word 0x57f11985, 0xaf75074c, 0xee99ddbb, 0xa37f60fd +.word 0xf701269f, 0x5c72f5bc, 0x44663bc5, 0x5bfb7e34 +.word 0x8b432976, 0xcb23c6dc, 0xb6edfc68, 0xb8e4f163 +.word 0xd731dcca, 0x42638510, 0x13972240, 0x84c61120 +.word 0x854a247d, 0xd2bb3df8, 0xaef93211, 0xc729a16d +.word 0x1d9e2f4b, 0xdcb230f3, 0x0d8652ec, 0x77c1e3d0 +.word 0x2bb3166c, 0xa970b999, 0x119448fa, 0x47e96422 +.word 0xa8fc8cc4, 0xa0f03f1a, 0x567d2cd8, 0x223390ef +.word 0x87494ec7, 0xd938d1c1, 0x8ccaa2fe, 0x98d40b36 +.word 0xa6f581cf, 0xa57ade28, 0xdab78e26, 0x3fadbfa4 +.word 0x2c3a9de4, 0x5078920d, 0x6a5fcc9b, 0x547e4662 +.word 0xf68d13c2, 0x90d8b8e8, 0x2e39f75e, 0x82c3aff5 +.word 0x9f5d80be, 0x69d0937c, 0x6fd52da9, 0xcf2512b3 +.word 0xc8ac993b, 0x10187da7, 0xe89c636e, 0xdb3bbb7b +.word 0xcd267809, 0x6e5918f4, 0xec9ab701, 0x834f9aa8 +.word 0xe6956e65, 0xaaffe67e, 0x21bccf08, 0xef15e8e6 +.word 0xbae79bd9, 0x4a6f36ce, 0xea9f09d4, 0x29b07cd6 +.word 0x31a4b2af, 0x2a3f2331, 0xc6a59430, 0x35a266c0 +.word 0x744ebc37, 0xfc82caa6, 0xe090d0b0, 0x33a7d815 +.word 0xf104984a, 0x41ecdaf7, 0x7fcd500e, 0x1791f62f +.word 0x764dd68d, 0x43efb04d, 0xccaa4d54, 0xe49604df +.word 0x9ed1b5e3, 0x4c6a881b, 0xc12c1fb8, 0x4665517f +.word 0x9d5eea04, 0x018c355d, 0xfa877473, 0xfb0b412e +.word 0xb3671d5a, 0x92dbd252, 0xe9105633, 0x6dd64713 +.word 0x9ad7618c, 0x37a10c7a, 0x59f8148e, 0xeb133c89 +.word 0xcea927ee, 0xb761c935, 0xe11ce5ed, 0x7a47b13c +.word 0x9cd2df59, 0x55f2733f, 0x1814ce79, 0x73c737bf +.word 0x53f7cdea, 0x5ffdaa5b, 0xdf3d6f14, 0x7844db86 +.word 0xcaaff381, 0xb968c43e, 0x3824342c, 0xc2a3405f +.word 0x161dc372, 0xbce2250c, 0x283c498b, 0xff0d9541 +.word 0x39a80171, 0x080cb3de, 0xd8b4e49c, 0x6456c190 +.word 0x7bcb8461, 0xd532b670, 0x486c5c74, 0xd0b85742 +@ Td4[256] +.byte 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38 +.byte 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb +.byte 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87 +.byte 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb +.byte 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d +.byte 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e +.byte 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2 +.byte 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25 +.byte 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16 +.byte 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92 +.byte 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda +.byte 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84 +.byte 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a +.byte 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06 +.byte 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02 +.byte 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b +.byte 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea +.byte 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73 +.byte 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85 +.byte 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e +.byte 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89 +.byte 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b +.byte 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20 +.byte 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4 +.byte 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31 +.byte 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f +.byte 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d +.byte 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef +.byte 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0 +.byte 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61 +.byte 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26 +.byte 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d +.size AES_Td,.-AES_Td + +@ void AES_decrypt(const unsigned char *in, unsigned char *out, +@ const AES_KEY *key) { +.global AES_decrypt +.type AES_decrypt,%function +.align 5 +AES_decrypt: + sub r3,pc,#8 @ AES_decrypt + stmdb sp!,{r1,r4-r12,lr} + mov r12,r0 @ inp + mov r11,r2 + sub r10,r3,#AES_decrypt-AES_Td @ Td +#if __ARM_ARCH__<7 + ldrb r0,[r12,#3] @ load input data in endian-neutral + ldrb r4,[r12,#2] @ manner... + ldrb r5,[r12,#1] + ldrb r6,[r12,#0] + orr r0,r0,r4,lsl#8 + ldrb r1,[r12,#7] + orr r0,r0,r5,lsl#16 + ldrb r4,[r12,#6] + orr r0,r0,r6,lsl#24 + ldrb r5,[r12,#5] + ldrb r6,[r12,#4] + orr r1,r1,r4,lsl#8 + ldrb r2,[r12,#11] + orr r1,r1,r5,lsl#16 + ldrb r4,[r12,#10] + orr r1,r1,r6,lsl#24 + ldrb r5,[r12,#9] + ldrb r6,[r12,#8] + orr r2,r2,r4,lsl#8 + ldrb r3,[r12,#15] + orr r2,r2,r5,lsl#16 + ldrb r4,[r12,#14] + orr r2,r2,r6,lsl#24 + ldrb r5,[r12,#13] + ldrb r6,[r12,#12] + orr r3,r3,r4,lsl#8 + orr r3,r3,r5,lsl#16 + orr r3,r3,r6,lsl#24 +#else + ldr r0,[r12,#0] + ldr r1,[r12,#4] + ldr r2,[r12,#8] + ldr r3,[r12,#12] +#ifdef __ARMEL__ + rev r0,r0 + rev r1,r1 + rev r2,r2 + rev r3,r3 +#endif +#endif + bl _armv4_AES_decrypt + + ldr r12,[sp],#4 @ pop out +#if __ARM_ARCH__>=7 +#ifdef __ARMEL__ + rev r0,r0 + rev r1,r1 + rev r2,r2 + rev r3,r3 +#endif + str r0,[r12,#0] + str r1,[r12,#4] + str r2,[r12,#8] + str r3,[r12,#12] +#else + mov r4,r0,lsr#24 @ write output in endian-neutral + mov r5,r0,lsr#16 @ manner... + mov r6,r0,lsr#8 + strb r4,[r12,#0] + strb r5,[r12,#1] + mov r4,r1,lsr#24 + strb r6,[r12,#2] + mov r5,r1,lsr#16 + strb r0,[r12,#3] + mov r6,r1,lsr#8 + strb r4,[r12,#4] + strb r5,[r12,#5] + mov r4,r2,lsr#24 + strb r6,[r12,#6] + mov r5,r2,lsr#16 + strb r1,[r12,#7] + mov r6,r2,lsr#8 + strb r4,[r12,#8] + strb r5,[r12,#9] + mov r4,r3,lsr#24 + strb r6,[r12,#10] + mov r5,r3,lsr#16 + strb r2,[r12,#11] + mov r6,r3,lsr#8 + strb r4,[r12,#12] + strb r5,[r12,#13] + strb r6,[r12,#14] + strb r3,[r12,#15] +#endif +#if __ARM_ARCH__>=5 + ldmia sp!,{r4-r12,pc} +#else + ldmia sp!,{r4-r12,lr} + tst lr,#1 + moveq pc,lr @ be binary compatible with V4, yet + .word 0xe12fff1e @ interoperable with Thumb ISA:-) +#endif +.size AES_decrypt,.-AES_decrypt + +.type _armv4_AES_decrypt,%function +.align 2 +_armv4_AES_decrypt: + str lr,[sp,#-4]! @ push lr + ldmia r11!,{r4-r7} + eor r0,r0,r4 + ldr r12,[r11,#240-16] + eor r1,r1,r5 + eor r2,r2,r6 + eor r3,r3,r7 + sub r12,r12,#1 + mov lr,#255 + + and r7,lr,r0,lsr#16 + and r8,lr,r0,lsr#8 + and r9,lr,r0 + mov r0,r0,lsr#24 +.Ldec_loop: + ldr r4,[r10,r7,lsl#2] @ Td1[s0>>16] + and r7,lr,r1 @ i0 + ldr r5,[r10,r8,lsl#2] @ Td2[s0>>8] + and r8,lr,r1,lsr#16 + ldr r6,[r10,r9,lsl#2] @ Td3[s0>>0] + and r9,lr,r1,lsr#8 + ldr r0,[r10,r0,lsl#2] @ Td0[s0>>24] + mov r1,r1,lsr#24 + + ldr r7,[r10,r7,lsl#2] @ Td3[s1>>0] + ldr r8,[r10,r8,lsl#2] @ Td1[s1>>16] + ldr r9,[r10,r9,lsl#2] @ Td2[s1>>8] + eor r0,r0,r7,ror#24 + ldr r1,[r10,r1,lsl#2] @ Td0[s1>>24] + and r7,lr,r2,lsr#8 @ i0 + eor r5,r8,r5,ror#8 + and r8,lr,r2 @ i1 + eor r6,r9,r6,ror#8 + and r9,lr,r2,lsr#16 + ldr r7,[r10,r7,lsl#2] @ Td2[s2>>8] + eor r1,r1,r4,ror#8 + ldr r8,[r10,r8,lsl#2] @ Td3[s2>>0] + mov r2,r2,lsr#24 + + ldr r9,[r10,r9,lsl#2] @ Td1[s2>>16] + eor r0,r0,r7,ror#16 + ldr r2,[r10,r2,lsl#2] @ Td0[s2>>24] + and r7,lr,r3,lsr#16 @ i0 + eor r1,r1,r8,ror#24 + and r8,lr,r3,lsr#8 @ i1 + eor r6,r9,r6,ror#8 + and r9,lr,r3 @ i2 + ldr r7,[r10,r7,lsl#2] @ Td1[s3>>16] + eor r2,r2,r5,ror#8 + ldr r8,[r10,r8,lsl#2] @ Td2[s3>>8] + mov r3,r3,lsr#24 + + ldr r9,[r10,r9,lsl#2] @ Td3[s3>>0] + eor r0,r0,r7,ror#8 + ldr r7,[r11],#16 + eor r1,r1,r8,ror#16 + ldr r3,[r10,r3,lsl#2] @ Td0[s3>>24] + eor r2,r2,r9,ror#24 + + ldr r4,[r11,#-12] + eor r0,r0,r7 + ldr r5,[r11,#-8] + eor r3,r3,r6,ror#8 + ldr r6,[r11,#-4] + and r7,lr,r0,lsr#16 + eor r1,r1,r4 + and r8,lr,r0,lsr#8 + eor r2,r2,r5 + and r9,lr,r0 + eor r3,r3,r6 + mov r0,r0,lsr#24 + + subs r12,r12,#1 + bne .Ldec_loop + + add r10,r10,#1024 + + ldr r5,[r10,#0] @ prefetch Td4 + ldr r6,[r10,#32] + ldr r4,[r10,#64] + ldr r5,[r10,#96] + ldr r6,[r10,#128] + ldr r4,[r10,#160] + ldr r5,[r10,#192] + ldr r6,[r10,#224] + + ldrb r0,[r10,r0] @ Td4[s0>>24] + ldrb r4,[r10,r7] @ Td4[s0>>16] + and r7,lr,r1 @ i0 + ldrb r5,[r10,r8] @ Td4[s0>>8] + and r8,lr,r1,lsr#16 + ldrb r6,[r10,r9] @ Td4[s0>>0] + and r9,lr,r1,lsr#8 + + ldrb r7,[r10,r7] @ Td4[s1>>0] + ldrb r1,[r10,r1,lsr#24] @ Td4[s1>>24] + ldrb r8,[r10,r8] @ Td4[s1>>16] + eor r0,r7,r0,lsl#24 + ldrb r9,[r10,r9] @ Td4[s1>>8] + eor r1,r4,r1,lsl#8 + and r7,lr,r2,lsr#8 @ i0 + eor r5,r5,r8,lsl#8 + and r8,lr,r2 @ i1 + ldrb r7,[r10,r7] @ Td4[s2>>8] + eor r6,r6,r9,lsl#8 + ldrb r8,[r10,r8] @ Td4[s2>>0] + and r9,lr,r2,lsr#16 + + ldrb r2,[r10,r2,lsr#24] @ Td4[s2>>24] + eor r0,r0,r7,lsl#8 + ldrb r9,[r10,r9] @ Td4[s2>>16] + eor r1,r8,r1,lsl#16 + and r7,lr,r3,lsr#16 @ i0 + eor r2,r5,r2,lsl#16 + and r8,lr,r3,lsr#8 @ i1 + ldrb r7,[r10,r7] @ Td4[s3>>16] + eor r6,r6,r9,lsl#16 + ldrb r8,[r10,r8] @ Td4[s3>>8] + and r9,lr,r3 @ i2 + + ldrb r9,[r10,r9] @ Td4[s3>>0] + ldrb r3,[r10,r3,lsr#24] @ Td4[s3>>24] + eor r0,r0,r7,lsl#16 + ldr r7,[r11,#0] + eor r1,r1,r8,lsl#8 + ldr r4,[r11,#4] + eor r2,r9,r2,lsl#8 + ldr r5,[r11,#8] + eor r3,r6,r3,lsl#24 + ldr r6,[r11,#12] + + eor r0,r0,r7 + eor r1,r1,r4 + eor r2,r2,r5 + eor r3,r3,r6 + + sub r10,r10,#1024 + ldr pc,[sp],#4 @ pop and return +.size _armv4_AES_decrypt,.-_armv4_AES_decrypt +.asciz "AES for ARMv4, CRYPTOGAMS by " +.align 2 diff --git a/arch/arm/crypto/aes_glue.c b/arch/arm/crypto/aes_glue.c new file mode 100644 index 00000000000..59f7877ead6 --- /dev/null +++ b/arch/arm/crypto/aes_glue.c @@ -0,0 +1,108 @@ +/* + * Glue Code for the asm optimized version of the AES Cipher Algorithm + */ + +#include +#include +#include + +#define AES_MAXNR 14 + +typedef struct { + unsigned int rd_key[4 *(AES_MAXNR + 1)]; + int rounds; +} AES_KEY; + +struct AES_CTX { + AES_KEY enc_key; + AES_KEY dec_key; +}; + +asmlinkage void AES_encrypt(const u8 *in, u8 *out, AES_KEY *ctx); +asmlinkage void AES_decrypt(const u8 *in, u8 *out, AES_KEY *ctx); +asmlinkage int private_AES_set_decrypt_key(const unsigned char *userKey, const int bits, AES_KEY *key); +asmlinkage int private_AES_set_encrypt_key(const unsigned char *userKey, const int bits, AES_KEY *key); + +static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) +{ + struct AES_CTX *ctx = crypto_tfm_ctx(tfm); + AES_encrypt(src, dst, &ctx->enc_key); +} + +static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) +{ + struct AES_CTX *ctx = crypto_tfm_ctx(tfm); + AES_decrypt(src, dst, &ctx->dec_key); +} + +static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct AES_CTX *ctx = crypto_tfm_ctx(tfm); + + switch (key_len) { + case AES_KEYSIZE_128: + key_len = 128; + break; + case AES_KEYSIZE_192: + key_len = 192; + break; + case AES_KEYSIZE_256: + key_len = 256; + break; + default: + tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + + if (private_AES_set_encrypt_key(in_key, key_len, &ctx->enc_key) == -1) { + tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + /* private_AES_set_decrypt_key expects an encryption key as input */ + ctx->dec_key = ctx->enc_key; + if (private_AES_set_decrypt_key(in_key, key_len, &ctx->dec_key) == -1) { + tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + return 0; +} + +static struct crypto_alg aes_alg = { + .cra_name = "aes", + .cra_driver_name = "aes-asm", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_TYPE_CIPHER, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct AES_CTX), + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(aes_alg.cra_list), + .cra_u = { + .cipher = { + .cia_min_keysize = AES_MIN_KEY_SIZE, + .cia_max_keysize = AES_MAX_KEY_SIZE, + .cia_setkey = aes_set_key, + .cia_encrypt = aes_encrypt, + .cia_decrypt = aes_decrypt + } + } +}; + +static int __init aes_init(void) +{ + return crypto_register_alg(&aes_alg); +} + +static void __exit aes_fini(void) +{ + crypto_unregister_alg(&aes_alg); +} + +module_init(aes_init); +module_exit(aes_fini); + +MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm (ASM)"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("aes"); +MODULE_ALIAS("aes-asm"); +MODULE_AUTHOR("David McCullough "); diff --git a/arch/arm/crypto/sha1-armv4-large.S b/arch/arm/crypto/sha1-armv4-large.S new file mode 100644 index 00000000000..7050ab133b9 --- /dev/null +++ b/arch/arm/crypto/sha1-armv4-large.S @@ -0,0 +1,503 @@ +#define __ARM_ARCH__ __LINUX_ARM_ARCH__ +@ ==================================================================== +@ Written by Andy Polyakov for the OpenSSL +@ project. The module is, however, dual licensed under OpenSSL and +@ CRYPTOGAMS licenses depending on where you obtain it. For further +@ details see http://www.openssl.org/~appro/cryptogams/. +@ ==================================================================== + +@ sha1_block procedure for ARMv4. +@ +@ January 2007. + +@ Size/performance trade-off +@ ==================================================================== +@ impl size in bytes comp cycles[*] measured performance +@ ==================================================================== +@ thumb 304 3212 4420 +@ armv4-small 392/+29% 1958/+64% 2250/+96% +@ armv4-compact 740/+89% 1552/+26% 1840/+22% +@ armv4-large 1420/+92% 1307/+19% 1370/+34%[***] +@ full unroll ~5100/+260% ~1260/+4% ~1300/+5% +@ ==================================================================== +@ thumb = same as 'small' but in Thumb instructions[**] and +@ with recurring code in two private functions; +@ small = detached Xload/update, loops are folded; +@ compact = detached Xload/update, 5x unroll; +@ large = interleaved Xload/update, 5x unroll; +@ full unroll = interleaved Xload/update, full unroll, estimated[!]; +@ +@ [*] Manually counted instructions in "grand" loop body. Measured +@ performance is affected by prologue and epilogue overhead, +@ i-cache availability, branch penalties, etc. +@ [**] While each Thumb instruction is twice smaller, they are not as +@ diverse as ARM ones: e.g., there are only two arithmetic +@ instructions with 3 arguments, no [fixed] rotate, addressing +@ modes are limited. As result it takes more instructions to do +@ the same job in Thumb, therefore the code is never twice as +@ small and always slower. +@ [***] which is also ~35% better than compiler generated code. Dual- +@ issue Cortex A8 core was measured to process input block in +@ ~990 cycles. + +@ August 2010. +@ +@ Rescheduling for dual-issue pipeline resulted in 13% improvement on +@ Cortex A8 core and in absolute terms ~870 cycles per input block +@ [or 13.6 cycles per byte]. + +@ February 2011. +@ +@ Profiler-assisted and platform-specific optimization resulted in 10% +@ improvement on Cortex A8 core and 12.2 cycles per byte. + +.text + +.global sha1_block_data_order +.type sha1_block_data_order,%function + +.align 2 +sha1_block_data_order: + stmdb sp!,{r4-r12,lr} + add r2,r1,r2,lsl#6 @ r2 to point at the end of r1 + ldmia r0,{r3,r4,r5,r6,r7} +.Lloop: + ldr r8,.LK_00_19 + mov r14,sp + sub sp,sp,#15*4 + mov r5,r5,ror#30 + mov r6,r6,ror#30 + mov r7,r7,ror#30 @ [6] +.L_00_15: +#if __ARM_ARCH__<7 + ldrb r10,[r1,#2] + ldrb r9,[r1,#3] + ldrb r11,[r1,#1] + add r7,r8,r7,ror#2 @ E+=K_00_19 + ldrb r12,[r1],#4 + orr r9,r9,r10,lsl#8 + eor r10,r5,r6 @ F_xx_xx + orr r9,r9,r11,lsl#16 + add r7,r7,r3,ror#27 @ E+=ROR(A,27) + orr r9,r9,r12,lsl#24 +#else + ldr r9,[r1],#4 @ handles unaligned + add r7,r8,r7,ror#2 @ E+=K_00_19 + eor r10,r5,r6 @ F_xx_xx + add r7,r7,r3,ror#27 @ E+=ROR(A,27) +#ifdef __ARMEL__ + rev r9,r9 @ byte swap +#endif +#endif + and r10,r4,r10,ror#2 + add r7,r7,r9 @ E+=X[i] + eor r10,r10,r6,ror#2 @ F_00_19(B,C,D) + str r9,[r14,#-4]! + add r7,r7,r10 @ E+=F_00_19(B,C,D) +#if __ARM_ARCH__<7 + ldrb r10,[r1,#2] + ldrb r9,[r1,#3] + ldrb r11,[r1,#1] + add r6,r8,r6,ror#2 @ E+=K_00_19 + ldrb r12,[r1],#4 + orr r9,r9,r10,lsl#8 + eor r10,r4,r5 @ F_xx_xx + orr r9,r9,r11,lsl#16 + add r6,r6,r7,ror#27 @ E+=ROR(A,27) + orr r9,r9,r12,lsl#24 +#else + ldr r9,[r1],#4 @ handles unaligned + add r6,r8,r6,ror#2 @ E+=K_00_19 + eor r10,r4,r5 @ F_xx_xx + add r6,r6,r7,ror#27 @ E+=ROR(A,27) +#ifdef __ARMEL__ + rev r9,r9 @ byte swap +#endif +#endif + and r10,r3,r10,ror#2 + add r6,r6,r9 @ E+=X[i] + eor r10,r10,r5,ror#2 @ F_00_19(B,C,D) + str r9,[r14,#-4]! + add r6,r6,r10 @ E+=F_00_19(B,C,D) +#if __ARM_ARCH__<7 + ldrb r10,[r1,#2] + ldrb r9,[r1,#3] + ldrb r11,[r1,#1] + add r5,r8,r5,ror#2 @ E+=K_00_19 + ldrb r12,[r1],#4 + orr r9,r9,r10,lsl#8 + eor r10,r3,r4 @ F_xx_xx + orr r9,r9,r11,lsl#16 + add r5,r5,r6,ror#27 @ E+=ROR(A,27) + orr r9,r9,r12,lsl#24 +#else + ldr r9,[r1],#4 @ handles unaligned + add r5,r8,r5,ror#2 @ E+=K_00_19 + eor r10,r3,r4 @ F_xx_xx + add r5,r5,r6,ror#27 @ E+=ROR(A,27) +#ifdef __ARMEL__ + rev r9,r9 @ byte swap +#endif +#endif + and r10,r7,r10,ror#2 + add r5,r5,r9 @ E+=X[i] + eor r10,r10,r4,ror#2 @ F_00_19(B,C,D) + str r9,[r14,#-4]! + add r5,r5,r10 @ E+=F_00_19(B,C,D) +#if __ARM_ARCH__<7 + ldrb r10,[r1,#2] + ldrb r9,[r1,#3] + ldrb r11,[r1,#1] + add r4,r8,r4,ror#2 @ E+=K_00_19 + ldrb r12,[r1],#4 + orr r9,r9,r10,lsl#8 + eor r10,r7,r3 @ F_xx_xx + orr r9,r9,r11,lsl#16 + add r4,r4,r5,ror#27 @ E+=ROR(A,27) + orr r9,r9,r12,lsl#24 +#else + ldr r9,[r1],#4 @ handles unaligned + add r4,r8,r4,ror#2 @ E+=K_00_19 + eor r10,r7,r3 @ F_xx_xx + add r4,r4,r5,ror#27 @ E+=ROR(A,27) +#ifdef __ARMEL__ + rev r9,r9 @ byte swap +#endif +#endif + and r10,r6,r10,ror#2 + add r4,r4,r9 @ E+=X[i] + eor r10,r10,r3,ror#2 @ F_00_19(B,C,D) + str r9,[r14,#-4]! + add r4,r4,r10 @ E+=F_00_19(B,C,D) +#if __ARM_ARCH__<7 + ldrb r10,[r1,#2] + ldrb r9,[r1,#3] + ldrb r11,[r1,#1] + add r3,r8,r3,ror#2 @ E+=K_00_19 + ldrb r12,[r1],#4 + orr r9,r9,r10,lsl#8 + eor r10,r6,r7 @ F_xx_xx + orr r9,r9,r11,lsl#16 + add r3,r3,r4,ror#27 @ E+=ROR(A,27) + orr r9,r9,r12,lsl#24 +#else + ldr r9,[r1],#4 @ handles unaligned + add r3,r8,r3,ror#2 @ E+=K_00_19 + eor r10,r6,r7 @ F_xx_xx + add r3,r3,r4,ror#27 @ E+=ROR(A,27) +#ifdef __ARMEL__ + rev r9,r9 @ byte swap +#endif +#endif + and r10,r5,r10,ror#2 + add r3,r3,r9 @ E+=X[i] + eor r10,r10,r7,ror#2 @ F_00_19(B,C,D) + str r9,[r14,#-4]! + add r3,r3,r10 @ E+=F_00_19(B,C,D) + teq r14,sp + bne .L_00_15 @ [((11+4)*5+2)*3] +#if __ARM_ARCH__<7 + ldrb r10,[r1,#2] + ldrb r9,[r1,#3] + ldrb r11,[r1,#1] + add r7,r8,r7,ror#2 @ E+=K_00_19 + ldrb r12,[r1],#4 + orr r9,r9,r10,lsl#8 + eor r10,r5,r6 @ F_xx_xx + orr r9,r9,r11,lsl#16 + add r7,r7,r3,ror#27 @ E+=ROR(A,27) + orr r9,r9,r12,lsl#24 +#else + ldr r9,[r1],#4 @ handles unaligned + add r7,r8,r7,ror#2 @ E+=K_00_19 + eor r10,r5,r6 @ F_xx_xx + add r7,r7,r3,ror#27 @ E+=ROR(A,27) +#ifdef __ARMEL__ + rev r9,r9 @ byte swap +#endif +#endif + and r10,r4,r10,ror#2 + add r7,r7,r9 @ E+=X[i] + eor r10,r10,r6,ror#2 @ F_00_19(B,C,D) + str r9,[r14,#-4]! + add r7,r7,r10 @ E+=F_00_19(B,C,D) + ldr r9,[r14,#15*4] + ldr r10,[r14,#13*4] + ldr r11,[r14,#7*4] + add r6,r8,r6,ror#2 @ E+=K_xx_xx + ldr r12,[r14,#2*4] + eor r9,r9,r10 + eor r11,r11,r12 @ 1 cycle stall + eor r10,r4,r5 @ F_xx_xx + mov r9,r9,ror#31 + add r6,r6,r7,ror#27 @ E+=ROR(A,27) + eor r9,r9,r11,ror#31 + str r9,[r14,#-4]! + and r10,r3,r10,ror#2 @ F_xx_xx + @ F_xx_xx + add r6,r6,r9 @ E+=X[i] + eor r10,r10,r5,ror#2 @ F_00_19(B,C,D) + add r6,r6,r10 @ E+=F_00_19(B,C,D) + ldr r9,[r14,#15*4] + ldr r10,[r14,#13*4] + ldr r11,[r14,#7*4] + add r5,r8,r5,ror#2 @ E+=K_xx_xx + ldr r12,[r14,#2*4] + eor r9,r9,r10 + eor r11,r11,r12 @ 1 cycle stall + eor r10,r3,r4 @ F_xx_xx + mov r9,r9,ror#31 + add r5,r5,r6,ror#27 @ E+=ROR(A,27) + eor r9,r9,r11,ror#31 + str r9,[r14,#-4]! + and r10,r7,r10,ror#2 @ F_xx_xx + @ F_xx_xx + add r5,r5,r9 @ E+=X[i] + eor r10,r10,r4,ror#2 @ F_00_19(B,C,D) + add r5,r5,r10 @ E+=F_00_19(B,C,D) + ldr r9,[r14,#15*4] + ldr r10,[r14,#13*4] + ldr r11,[r14,#7*4] + add r4,r8,r4,ror#2 @ E+=K_xx_xx + ldr r12,[r14,#2*4] + eor r9,r9,r10 + eor r11,r11,r12 @ 1 cycle stall + eor r10,r7,r3 @ F_xx_xx + mov r9,r9,ror#31 + add r4,r4,r5,ror#27 @ E+=ROR(A,27) + eor r9,r9,r11,ror#31 + str r9,[r14,#-4]! + and r10,r6,r10,ror#2 @ F_xx_xx + @ F_xx_xx + add r4,r4,r9 @ E+=X[i] + eor r10,r10,r3,ror#2 @ F_00_19(B,C,D) + add r4,r4,r10 @ E+=F_00_19(B,C,D) + ldr r9,[r14,#15*4] + ldr r10,[r14,#13*4] + ldr r11,[r14,#7*4] + add r3,r8,r3,ror#2 @ E+=K_xx_xx + ldr r12,[r14,#2*4] + eor r9,r9,r10 + eor r11,r11,r12 @ 1 cycle stall + eor r10,r6,r7 @ F_xx_xx + mov r9,r9,ror#31 + add r3,r3,r4,ror#27 @ E+=ROR(A,27) + eor r9,r9,r11,ror#31 + str r9,[r14,#-4]! + and r10,r5,r10,ror#2 @ F_xx_xx + @ F_xx_xx + add r3,r3,r9 @ E+=X[i] + eor r10,r10,r7,ror#2 @ F_00_19(B,C,D) + add r3,r3,r10 @ E+=F_00_19(B,C,D) + + ldr r8,.LK_20_39 @ [+15+16*4] + sub sp,sp,#25*4 + cmn sp,#0 @ [+3], clear carry to denote 20_39 +.L_20_39_or_60_79: + ldr r9,[r14,#15*4] + ldr r10,[r14,#13*4] + ldr r11,[r14,#7*4] + add r7,r8,r7,ror#2 @ E+=K_xx_xx + ldr r12,[r14,#2*4] + eor r9,r9,r10 + eor r11,r11,r12 @ 1 cycle stall + eor r10,r5,r6 @ F_xx_xx + mov r9,r9,ror#31 + add r7,r7,r3,ror#27 @ E+=ROR(A,27) + eor r9,r9,r11,ror#31 + str r9,[r14,#-4]! + eor r10,r4,r10,ror#2 @ F_xx_xx + @ F_xx_xx + add r7,r7,r9 @ E+=X[i] + add r7,r7,r10 @ E+=F_20_39(B,C,D) + ldr r9,[r14,#15*4] + ldr r10,[r14,#13*4] + ldr r11,[r14,#7*4] + add r6,r8,r6,ror#2 @ E+=K_xx_xx + ldr r12,[r14,#2*4] + eor r9,r9,r10 + eor r11,r11,r12 @ 1 cycle stall + eor r10,r4,r5 @ F_xx_xx + mov r9,r9,ror#31 + add r6,r6,r7,ror#27 @ E+=ROR(A,27) + eor r9,r9,r11,ror#31 + str r9,[r14,#-4]! + eor r10,r3,r10,ror#2 @ F_xx_xx + @ F_xx_xx + add r6,r6,r9 @ E+=X[i] + add r6,r6,r10 @ E+=F_20_39(B,C,D) + ldr r9,[r14,#15*4] + ldr r10,[r14,#13*4] + ldr r11,[r14,#7*4] + add r5,r8,r5,ror#2 @ E+=K_xx_xx + ldr r12,[r14,#2*4] + eor r9,r9,r10 + eor r11,r11,r12 @ 1 cycle stall + eor r10,r3,r4 @ F_xx_xx + mov r9,r9,ror#31 + add r5,r5,r6,ror#27 @ E+=ROR(A,27) + eor r9,r9,r11,ror#31 + str r9,[r14,#-4]! + eor r10,r7,r10,ror#2 @ F_xx_xx + @ F_xx_xx + add r5,r5,r9 @ E+=X[i] + add r5,r5,r10 @ E+=F_20_39(B,C,D) + ldr r9,[r14,#15*4] + ldr r10,[r14,#13*4] + ldr r11,[r14,#7*4] + add r4,r8,r4,ror#2 @ E+=K_xx_xx + ldr r12,[r14,#2*4] + eor r9,r9,r10 + eor r11,r11,r12 @ 1 cycle stall + eor r10,r7,r3 @ F_xx_xx + mov r9,r9,ror#31 + add r4,r4,r5,ror#27 @ E+=ROR(A,27) + eor r9,r9,r11,ror#31 + str r9,[r14,#-4]! + eor r10,r6,r10,ror#2 @ F_xx_xx + @ F_xx_xx + add r4,r4,r9 @ E+=X[i] + add r4,r4,r10 @ E+=F_20_39(B,C,D) + ldr r9,[r14,#15*4] + ldr r10,[r14,#13*4] + ldr r11,[r14,#7*4] + add r3,r8,r3,ror#2 @ E+=K_xx_xx + ldr r12,[r14,#2*4] + eor r9,r9,r10 + eor r11,r11,r12 @ 1 cycle stall + eor r10,r6,r7 @ F_xx_xx + mov r9,r9,ror#31 + add r3,r3,r4,ror#27 @ E+=ROR(A,27) + eor r9,r9,r11,ror#31 + str r9,[r14,#-4]! + eor r10,r5,r10,ror#2 @ F_xx_xx + @ F_xx_xx + add r3,r3,r9 @ E+=X[i] + add r3,r3,r10 @ E+=F_20_39(B,C,D) + teq r14,sp @ preserve carry + bne .L_20_39_or_60_79 @ [+((12+3)*5+2)*4] + bcs .L_done @ [+((12+3)*5+2)*4], spare 300 bytes + + ldr r8,.LK_40_59 + sub sp,sp,#20*4 @ [+2] +.L_40_59: + ldr r9,[r14,#15*4] + ldr r10,[r14,#13*4] + ldr r11,[r14,#7*4] + add r7,r8,r7,ror#2 @ E+=K_xx_xx + ldr r12,[r14,#2*4] + eor r9,r9,r10 + eor r11,r11,r12 @ 1 cycle stall + eor r10,r5,r6 @ F_xx_xx + mov r9,r9,ror#31 + add r7,r7,r3,ror#27 @ E+=ROR(A,27) + eor r9,r9,r11,ror#31 + str r9,[r14,#-4]! + and r10,r4,r10,ror#2 @ F_xx_xx + and r11,r5,r6 @ F_xx_xx + add r7,r7,r9 @ E+=X[i] + add r7,r7,r10 @ E+=F_40_59(B,C,D) + add r7,r7,r11,ror#2 + ldr r9,[r14,#15*4] + ldr r10,[r14,#13*4] + ldr r11,[r14,#7*4] + add r6,r8,r6,ror#2 @ E+=K_xx_xx + ldr r12,[r14,#2*4] + eor r9,r9,r10 + eor r11,r11,r12 @ 1 cycle stall + eor r10,r4,r5 @ F_xx_xx + mov r9,r9,ror#31 + add r6,r6,r7,ror#27 @ E+=ROR(A,27) + eor r9,r9,r11,ror#31 + str r9,[r14,#-4]! + and r10,r3,r10,ror#2 @ F_xx_xx + and r11,r4,r5 @ F_xx_xx + add r6,r6,r9 @ E+=X[i] + add r6,r6,r10 @ E+=F_40_59(B,C,D) + add r6,r6,r11,ror#2 + ldr r9,[r14,#15*4] + ldr r10,[r14,#13*4] + ldr r11,[r14,#7*4] + add r5,r8,r5,ror#2 @ E+=K_xx_xx + ldr r12,[r14,#2*4] + eor r9,r9,r10 + eor r11,r11,r12 @ 1 cycle stall + eor r10,r3,r4 @ F_xx_xx + mov r9,r9,ror#31 + add r5,r5,r6,ror#27 @ E+=ROR(A,27) + eor r9,r9,r11,ror#31 + str r9,[r14,#-4]! + and r10,r7,r10,ror#2 @ F_xx_xx + and r11,r3,r4 @ F_xx_xx + add r5,r5,r9 @ E+=X[i] + add r5,r5,r10 @ E+=F_40_59(B,C,D) + add r5,r5,r11,ror#2 + ldr r9,[r14,#15*4] + ldr r10,[r14,#13*4] + ldr r11,[r14,#7*4] + add r4,r8,r4,ror#2 @ E+=K_xx_xx + ldr r12,[r14,#2*4] + eor r9,r9,r10 + eor r11,r11,r12 @ 1 cycle stall + eor r10,r7,r3 @ F_xx_xx + mov r9,r9,ror#31 + add r4,r4,r5,ror#27 @ E+=ROR(A,27) + eor r9,r9,r11,ror#31 + str r9,[r14,#-4]! + and r10,r6,r10,ror#2 @ F_xx_xx + and r11,r7,r3 @ F_xx_xx + add r4,r4,r9 @ E+=X[i] + add r4,r4,r10 @ E+=F_40_59(B,C,D) + add r4,r4,r11,ror#2 + ldr r9,[r14,#15*4] + ldr r10,[r14,#13*4] + ldr r11,[r14,#7*4] + add r3,r8,r3,ror#2 @ E+=K_xx_xx + ldr r12,[r14,#2*4] + eor r9,r9,r10 + eor r11,r11,r12 @ 1 cycle stall + eor r10,r6,r7 @ F_xx_xx + mov r9,r9,ror#31 + add r3,r3,r4,ror#27 @ E+=ROR(A,27) + eor r9,r9,r11,ror#31 + str r9,[r14,#-4]! + and r10,r5,r10,ror#2 @ F_xx_xx + and r11,r6,r7 @ F_xx_xx + add r3,r3,r9 @ E+=X[i] + add r3,r3,r10 @ E+=F_40_59(B,C,D) + add r3,r3,r11,ror#2 + teq r14,sp + bne .L_40_59 @ [+((12+5)*5+2)*4] + + ldr r8,.LK_60_79 + sub sp,sp,#20*4 + cmp sp,#0 @ set carry to denote 60_79 + b .L_20_39_or_60_79 @ [+4], spare 300 bytes +.L_done: + add sp,sp,#80*4 @ "deallocate" stack frame + ldmia r0,{r8,r9,r10,r11,r12} + add r3,r8,r3 + add r4,r9,r4 + add r5,r10,r5,ror#2 + add r6,r11,r6,ror#2 + add r7,r12,r7,ror#2 + stmia r0,{r3,r4,r5,r6,r7} + teq r1,r2 + bne .Lloop @ [+18], total 1307 + +#if __ARM_ARCH__>=5 + ldmia sp!,{r4-r12,pc} +#else + ldmia sp!,{r4-r12,lr} + tst lr,#1 + moveq pc,lr @ be binary compatible with V4, yet + .word 0xe12fff1e @ interoperable with Thumb ISA:-) +#endif +.align 2 +.LK_00_19: .word 0x5a827999 +.LK_20_39: .word 0x6ed9eba1 +.LK_40_59: .word 0x8f1bbcdc +.LK_60_79: .word 0xca62c1d6 +.size sha1_block_data_order,.-sha1_block_data_order +.asciz "SHA1 block transform for ARMv4, CRYPTOGAMS by " +.align 2 diff --git a/arch/arm/crypto/sha1_glue.c b/arch/arm/crypto/sha1_glue.c new file mode 100644 index 00000000000..76cd976230b --- /dev/null +++ b/arch/arm/crypto/sha1_glue.c @@ -0,0 +1,179 @@ +/* + * Cryptographic API. + * Glue code for the SHA1 Secure Hash Algorithm assembler implementation + * + * This file is based on sha1_generic.c and sha1_ssse3_glue.c + * + * Copyright (c) Alan Smithee. + * Copyright (c) Andrew McDonald + * Copyright (c) Jean-Francois Dive + * Copyright (c) Mathias Krause + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ + +#include +#include +#include +#include +#include +#include +#include + +struct SHA1_CTX { + uint32_t h0,h1,h2,h3,h4; + u64 count; + u8 data[SHA1_BLOCK_SIZE]; +}; + +asmlinkage void sha1_block_data_order(struct SHA1_CTX *digest, + const unsigned char *data, unsigned int rounds); + + +static int sha1_init(struct shash_desc *desc) +{ + struct SHA1_CTX *sctx = shash_desc_ctx(desc); + memset(sctx, 0, sizeof(*sctx)); + sctx->h0 = SHA1_H0; + sctx->h1 = SHA1_H1; + sctx->h2 = SHA1_H2; + sctx->h3 = SHA1_H3; + sctx->h4 = SHA1_H4; + return 0; +} + + +static int __sha1_update(struct SHA1_CTX *sctx, const u8 *data, + unsigned int len, unsigned int partial) +{ + unsigned int done = 0; + + sctx->count += len; + + if (partial) { + done = SHA1_BLOCK_SIZE - partial; + memcpy(sctx->data + partial, data, done); + sha1_block_data_order(sctx, sctx->data, 1); + } + + if (len - done >= SHA1_BLOCK_SIZE) { + const unsigned int rounds = (len - done) / SHA1_BLOCK_SIZE; + sha1_block_data_order(sctx, data + done, rounds); + done += rounds * SHA1_BLOCK_SIZE; + } + + memcpy(sctx->data, data + done, len - done); + return 0; +} + + +static int sha1_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct SHA1_CTX *sctx = shash_desc_ctx(desc); + unsigned int partial = sctx->count % SHA1_BLOCK_SIZE; + int res; + + /* Handle the fast case right here */ + if (partial + len < SHA1_BLOCK_SIZE) { + sctx->count += len; + memcpy(sctx->data + partial, data, len); + return 0; + } + res = __sha1_update(sctx, data, len, partial); + return res; +} + + +/* Add padding and return the message digest. */ +static int sha1_final(struct shash_desc *desc, u8 *out) +{ + struct SHA1_CTX *sctx = shash_desc_ctx(desc); + unsigned int i, index, padlen; + __be32 *dst = (__be32 *)out; + __be64 bits; + static const u8 padding[SHA1_BLOCK_SIZE] = { 0x80, }; + + bits = cpu_to_be64(sctx->count << 3); + + /* Pad out to 56 mod 64 and append length */ + index = sctx->count % SHA1_BLOCK_SIZE; + padlen = (index < 56) ? (56 - index) : ((SHA1_BLOCK_SIZE+56) - index); + /* We need to fill a whole block for __sha1_update() */ + if (padlen <= 56) { + sctx->count += padlen; + memcpy(sctx->data + index, padding, padlen); + } else { + __sha1_update(sctx, padding, padlen, index); + } + __sha1_update(sctx, (const u8 *)&bits, sizeof(bits), 56); + + /* Store state in digest */ + for (i = 0; i < 5; i++) + dst[i] = cpu_to_be32(((u32 *)sctx)[i]); + + /* Wipe context */ + memset(sctx, 0, sizeof(*sctx)); + return 0; +} + + +static int sha1_export(struct shash_desc *desc, void *out) +{ + struct SHA1_CTX *sctx = shash_desc_ctx(desc); + memcpy(out, sctx, sizeof(*sctx)); + return 0; +} + + +static int sha1_import(struct shash_desc *desc, const void *in) +{ + struct SHA1_CTX *sctx = shash_desc_ctx(desc); + memcpy(sctx, in, sizeof(*sctx)); + return 0; +} + + +static struct shash_alg alg = { + .digestsize = SHA1_DIGEST_SIZE, + .init = sha1_init, + .update = sha1_update, + .final = sha1_final, + .export = sha1_export, + .import = sha1_import, + .descsize = sizeof(struct SHA1_CTX), + .statesize = sizeof(struct SHA1_CTX), + .base = { + .cra_name = "sha1", + .cra_driver_name= "sha1-asm", + .cra_priority = 150, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + + +static int __init sha1_mod_init(void) +{ + return crypto_register_shash(&alg); +} + + +static void __exit sha1_mod_fini(void) +{ + crypto_unregister_shash(&alg); +} + + +module_init(sha1_mod_init); +module_exit(sha1_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm (ARM)"); +MODULE_ALIAS("sha1"); +MODULE_AUTHOR("David McCullough "); diff --git a/crypto/Kconfig b/crypto/Kconfig index 8e84225c096..2615d4cfc61 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -423,6 +423,15 @@ config CRYPTO_SHA1_SSSE3 using Supplemental SSE3 (SSSE3) instructions or Advanced Vector Extensions (AVX), when available. +config CRYPTO_SHA1_ARM + tristate "SHA1 digest algorithm (ARM-asm)" + depends on ARM + select CRYPTO_SHA1 + select CRYPTO_HASH + help + SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented + using optimized ARM assembler. + config CRYPTO_SHA256 tristate "SHA224 and SHA256 digest algorithm" select CRYPTO_HASH @@ -577,6 +586,30 @@ config CRYPTO_AES_NI_INTEL ECB, CBC, LRW, PCBC, XTS. The 64 bit version has additional acceleration for CTR. +config CRYPTO_AES_ARM + tristate "AES cipher algorithms (ARM-asm)" + depends on ARM + select CRYPTO_ALGAPI + select CRYPTO_AES + help + Use optimized AES assembler routines for ARM platforms. + + AES cipher algorithms (FIPS-197). AES uses the Rijndael + algorithm. + + Rijndael appears to be consistently a very good performer in + both hardware and software across a wide range of computing + environments regardless of its use in feedback or non-feedback + modes. Its key setup time is excellent, and its key agility is + good. Rijndael's very low memory requirements make it very well + suited for restricted-space environments, in which it also + demonstrates excellent performance. Rijndael's operations are + among the easiest to defend against power and timing attacks. + + The AES specifies three key sizes: 128, 192 and 256 bits + + See for more information. + config CRYPTO_ANUBIS tristate "Anubis cipher algorithm" select CRYPTO_ALGAPI From 1b0c649aa766d49e746e7bada1a017e070de1b9c Mon Sep 17 00:00:00 2001 From: Dave Martin Date: Thu, 10 Jan 2013 12:20:15 +0100 Subject: [PATCH 050/168] ARM: 7626/1: arm/crypto: Make asm SHA-1 and AES code Thumb-2 compatible This patch fixes aes-armv4.S and sha1-armv4-large.S to work natively in Thumb. This allows ARM/Thumb interworking workarounds to be removed. I also take the opportunity to convert some explicit assembler directives for exported functions to the standard ENTRY()/ENDPROC(). For the code itself: * In sha1_block_data_order, use of TEQ with sp is deprecated in ARMv7 and not supported in Thumb. For the branches back to .L_00_15 and .L_40_59, the TEQ is converted to a CMP, under the assumption that clobbering the C flag here will not cause incorrect behaviour. For the first branch back to .L_20_39_or_60_79 the C flag is important, so sp is moved temporarily into another register so that TEQ can be used for the comparison. * In the AES code, most forms of register-indexed addressing with shifts and rotates are not permitted for loads and stores in Thumb, so the address calculation is done using a separate instruction for the Thumb case. The resulting code is unlikely to be optimally scheduled, but it should not have a large impact given the overall size of the code. I haven't run any benchmarks. Change-Id: I9063f4af8d3259127504d865ecd123f9f0ea5415 Signed-off-by: Dave Martin Tested-by: David McCullough (ARM only) Acked-by: David McCullough Acked-by: Nicolas Pitre Signed-off-by: Russell King Git-commit: d9d72bdfd2ba36693e6c93cd071e5fb74fe8b6a7 Git-repo: https://android.googlesource.com/kernel/common/ Signed-off-by: Samir Mehta --- arch/arm/crypto/aes-armv4.S | 64 ++++++++++-------------------- arch/arm/crypto/sha1-armv4-large.S | 24 +++++------ 2 files changed, 29 insertions(+), 59 deletions(-) diff --git a/arch/arm/crypto/aes-armv4.S b/arch/arm/crypto/aes-armv4.S index e59b1d505d6..19d6cd6f29f 100644 --- a/arch/arm/crypto/aes-armv4.S +++ b/arch/arm/crypto/aes-armv4.S @@ -34,8 +34,9 @@ @ A little glue here to select the correct code below for the ARM CPU @ that is being targetted. +#include + .text -.code 32 .type AES_Te,%object .align 5 @@ -145,10 +146,8 @@ AES_Te: @ void AES_encrypt(const unsigned char *in, unsigned char *out, @ const AES_KEY *key) { -.global AES_encrypt -.type AES_encrypt,%function .align 5 -AES_encrypt: +ENTRY(AES_encrypt) sub r3,pc,#8 @ AES_encrypt stmdb sp!,{r1,r4-r12,lr} mov r12,r0 @ inp @@ -239,15 +238,8 @@ AES_encrypt: strb r6,[r12,#14] strb r3,[r12,#15] #endif -#if __ARM_ARCH__>=5 ldmia sp!,{r4-r12,pc} -#else - ldmia sp!,{r4-r12,lr} - tst lr,#1 - moveq pc,lr @ be binary compatible with V4, yet - .word 0xe12fff1e @ interoperable with Thumb ISA:-) -#endif -.size AES_encrypt,.-AES_encrypt +ENDPROC(AES_encrypt) .type _armv4_AES_encrypt,%function .align 2 @@ -386,10 +378,8 @@ _armv4_AES_encrypt: ldr pc,[sp],#4 @ pop and return .size _armv4_AES_encrypt,.-_armv4_AES_encrypt -.global private_AES_set_encrypt_key -.type private_AES_set_encrypt_key,%function .align 5 -private_AES_set_encrypt_key: +ENTRY(private_AES_set_encrypt_key) _armv4_AES_set_encrypt_key: sub r3,pc,#8 @ AES_set_encrypt_key teq r0,#0 @@ -658,15 +648,11 @@ _armv4_AES_set_encrypt_key: .Ldone: mov r0,#0 ldmia sp!,{r4-r12,lr} -.Labrt: tst lr,#1 - moveq pc,lr @ be binary compatible with V4, yet - .word 0xe12fff1e @ interoperable with Thumb ISA:-) -.size private_AES_set_encrypt_key,.-private_AES_set_encrypt_key +.Labrt: mov pc,lr +ENDPROC(private_AES_set_encrypt_key) -.global private_AES_set_decrypt_key -.type private_AES_set_decrypt_key,%function .align 5 -private_AES_set_decrypt_key: +ENTRY(private_AES_set_decrypt_key) str lr,[sp,#-4]! @ push lr #if 0 @ kernel does both of these in setkey so optimise this bit out by @@ -748,15 +734,8 @@ private_AES_set_decrypt_key: bne .Lmix mov r0,#0 -#if __ARM_ARCH__>=5 ldmia sp!,{r4-r12,pc} -#else - ldmia sp!,{r4-r12,lr} - tst lr,#1 - moveq pc,lr @ be binary compatible with V4, yet - .word 0xe12fff1e @ interoperable with Thumb ISA:-) -#endif -.size private_AES_set_decrypt_key,.-private_AES_set_decrypt_key +ENDPROC(private_AES_set_decrypt_key) .type AES_Td,%object .align 5 @@ -862,10 +841,8 @@ AES_Td: @ void AES_decrypt(const unsigned char *in, unsigned char *out, @ const AES_KEY *key) { -.global AES_decrypt -.type AES_decrypt,%function .align 5 -AES_decrypt: +ENTRY(AES_decrypt) sub r3,pc,#8 @ AES_decrypt stmdb sp!,{r1,r4-r12,lr} mov r12,r0 @ inp @@ -956,15 +933,8 @@ AES_decrypt: strb r6,[r12,#14] strb r3,[r12,#15] #endif -#if __ARM_ARCH__>=5 ldmia sp!,{r4-r12,pc} -#else - ldmia sp!,{r4-r12,lr} - tst lr,#1 - moveq pc,lr @ be binary compatible with V4, yet - .word 0xe12fff1e @ interoperable with Thumb ISA:-) -#endif -.size AES_decrypt,.-AES_decrypt +ENDPROC(AES_decrypt) .type _armv4_AES_decrypt,%function .align 2 @@ -1064,7 +1034,9 @@ _armv4_AES_decrypt: and r9,lr,r1,lsr#8 ldrb r7,[r10,r7] @ Td4[s1>>0] - ldrb r1,[r10,r1,lsr#24] @ Td4[s1>>24] + ARM( ldrb r1,[r10,r1,lsr#24] ) @ Td4[s1>>24] + THUMB( add r1,r10,r1,lsr#24 ) @ Td4[s1>>24] + THUMB( ldrb r1,[r1] ) ldrb r8,[r10,r8] @ Td4[s1>>16] eor r0,r7,r0,lsl#24 ldrb r9,[r10,r9] @ Td4[s1>>8] @@ -1077,7 +1049,9 @@ _armv4_AES_decrypt: ldrb r8,[r10,r8] @ Td4[s2>>0] and r9,lr,r2,lsr#16 - ldrb r2,[r10,r2,lsr#24] @ Td4[s2>>24] + ARM( ldrb r2,[r10,r2,lsr#24] ) @ Td4[s2>>24] + THUMB( add r2,r10,r2,lsr#24 ) @ Td4[s2>>24] + THUMB( ldrb r2,[r2] ) eor r0,r0,r7,lsl#8 ldrb r9,[r10,r9] @ Td4[s2>>16] eor r1,r8,r1,lsl#16 @@ -1090,7 +1064,9 @@ _armv4_AES_decrypt: and r9,lr,r3 @ i2 ldrb r9,[r10,r9] @ Td4[s3>>0] - ldrb r3,[r10,r3,lsr#24] @ Td4[s3>>24] + ARM( ldrb r3,[r10,r3,lsr#24] ) @ Td4[s3>>24] + THUMB( add r3,r10,r3,lsr#24 ) @ Td4[s3>>24] + THUMB( ldrb r3,[r3] ) eor r0,r0,r7,lsl#16 ldr r7,[r11,#0] eor r1,r1,r8,lsl#8 diff --git a/arch/arm/crypto/sha1-armv4-large.S b/arch/arm/crypto/sha1-armv4-large.S index 7050ab133b9..92c6eed7aac 100644 --- a/arch/arm/crypto/sha1-armv4-large.S +++ b/arch/arm/crypto/sha1-armv4-large.S @@ -51,13 +51,12 @@ @ Profiler-assisted and platform-specific optimization resulted in 10% @ improvement on Cortex A8 core and 12.2 cycles per byte. +#include + .text -.global sha1_block_data_order -.type sha1_block_data_order,%function - .align 2 -sha1_block_data_order: +ENTRY(sha1_block_data_order) stmdb sp!,{r4-r12,lr} add r2,r1,r2,lsl#6 @ r2 to point at the end of r1 ldmia r0,{r3,r4,r5,r6,r7} @@ -194,7 +193,7 @@ sha1_block_data_order: eor r10,r10,r7,ror#2 @ F_00_19(B,C,D) str r9,[r14,#-4]! add r3,r3,r10 @ E+=F_00_19(B,C,D) - teq r14,sp + cmp r14,sp bne .L_00_15 @ [((11+4)*5+2)*3] #if __ARM_ARCH__<7 ldrb r10,[r1,#2] @@ -374,7 +373,9 @@ sha1_block_data_order: @ F_xx_xx add r3,r3,r9 @ E+=X[i] add r3,r3,r10 @ E+=F_20_39(B,C,D) - teq r14,sp @ preserve carry + ARM( teq r14,sp ) @ preserve carry + THUMB( mov r11,sp ) + THUMB( teq r14,r11 ) @ preserve carry bne .L_20_39_or_60_79 @ [+((12+3)*5+2)*4] bcs .L_done @ [+((12+3)*5+2)*4], spare 300 bytes @@ -466,7 +467,7 @@ sha1_block_data_order: add r3,r3,r9 @ E+=X[i] add r3,r3,r10 @ E+=F_40_59(B,C,D) add r3,r3,r11,ror#2 - teq r14,sp + cmp r14,sp bne .L_40_59 @ [+((12+5)*5+2)*4] ldr r8,.LK_60_79 @@ -485,19 +486,12 @@ sha1_block_data_order: teq r1,r2 bne .Lloop @ [+18], total 1307 -#if __ARM_ARCH__>=5 ldmia sp!,{r4-r12,pc} -#else - ldmia sp!,{r4-r12,lr} - tst lr,#1 - moveq pc,lr @ be binary compatible with V4, yet - .word 0xe12fff1e @ interoperable with Thumb ISA:-) -#endif .align 2 .LK_00_19: .word 0x5a827999 .LK_20_39: .word 0x6ed9eba1 .LK_40_59: .word 0x8f1bbcdc .LK_60_79: .word 0xca62c1d6 -.size sha1_block_data_order,.-sha1_block_data_order +ENDPROC(sha1_block_data_order) .asciz "SHA1 block transform for ARMv4, CRYPTOGAMS by " .align 2 From eba50a4ae2fdff89ee0156097e004b682516b922 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 15 May 2013 10:46:30 +0100 Subject: [PATCH 051/168] ARM: 7723/1: crypto: sha1-armv4-large.S: fix SP handling Make the SHA1 asm code ABI conformant by making sure all stack accesses occur above the stack pointer. Origin: http://git.openssl.org/gitweb/?p=openssl.git;a=commit;h=1a9d60d2 Change-Id: I08414e9a0df5952b8a25627ddfd2c93e1a79b394 Signed-off-by: Ard Biesheuvel Acked-by: Nicolas Pitre Cc: stable@vger.kernel.org Signed-off-by: Russell King Git-commit: 5393c535d1afa6a6aa7e02faa1f1ca07cb856fe3 Git-repo: https://android.googlesource.com/kernel/common/ Signed-off-by: Samir Mehta --- arch/arm/crypto/sha1-armv4-large.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/crypto/sha1-armv4-large.S b/arch/arm/crypto/sha1-armv4-large.S index 92c6eed7aac..99207c45ec1 100644 --- a/arch/arm/crypto/sha1-armv4-large.S +++ b/arch/arm/crypto/sha1-armv4-large.S @@ -195,6 +195,7 @@ ENTRY(sha1_block_data_order) add r3,r3,r10 @ E+=F_00_19(B,C,D) cmp r14,sp bne .L_00_15 @ [((11+4)*5+2)*3] + sub sp,sp,#25*4 #if __ARM_ARCH__<7 ldrb r10,[r1,#2] ldrb r9,[r1,#3] @@ -290,7 +291,6 @@ ENTRY(sha1_block_data_order) add r3,r3,r10 @ E+=F_00_19(B,C,D) ldr r8,.LK_20_39 @ [+15+16*4] - sub sp,sp,#25*4 cmn sp,#0 @ [+3], clear carry to denote 20_39 .L_20_39_or_60_79: ldr r9,[r14,#15*4] From f03d1a9af379c12c497a87068ad5f655c8b45fbb Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sat, 21 Sep 2013 11:23:50 +0100 Subject: [PATCH 052/168] ARM: 7837/3: fix Thumb-2 bug in AES assembler code Patch 638591c enabled building the AES assembler code in Thumb2 mode. However, this code used arithmetic involving PC rather than adr{l} instructions to generate PC-relative references to the lookup tables, and this needs to take into account the different PC offset when running in Thumb mode. Change-Id: I7e1f5623df043ec9395b837378523168d52a93d3 Signed-off-by: Ard Biesheuvel Acked-by: Nicolas Pitre Cc: stable@vger.kernel.org Signed-off-by: Russell King Git-commit: 0b6350b0cfc57d012026ab3d92ac62bd21ab71ec Git-repo: https://android.googlesource.com/kernel/common/ Signed-off-by: Samir Mehta --- arch/arm/crypto/aes-armv4.S | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/arm/crypto/aes-armv4.S b/arch/arm/crypto/aes-armv4.S index 19d6cd6f29f..3a14ea8fe97 100644 --- a/arch/arm/crypto/aes-armv4.S +++ b/arch/arm/crypto/aes-armv4.S @@ -148,7 +148,7 @@ AES_Te: @ const AES_KEY *key) { .align 5 ENTRY(AES_encrypt) - sub r3,pc,#8 @ AES_encrypt + adr r3,AES_encrypt stmdb sp!,{r1,r4-r12,lr} mov r12,r0 @ inp mov r11,r2 @@ -381,7 +381,7 @@ _armv4_AES_encrypt: .align 5 ENTRY(private_AES_set_encrypt_key) _armv4_AES_set_encrypt_key: - sub r3,pc,#8 @ AES_set_encrypt_key + adr r3,_armv4_AES_set_encrypt_key teq r0,#0 moveq r0,#-1 beq .Labrt @@ -843,7 +843,7 @@ AES_Td: @ const AES_KEY *key) { .align 5 ENTRY(AES_decrypt) - sub r3,pc,#8 @ AES_decrypt + adr r3,AES_decrypt stmdb sp!,{r1,r4-r12,lr} mov r12,r0 @ inp mov r11,r2 From 7870bc8d35ca2b2e0b63434c9a9ed1004dcf78d8 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 15 Sep 2013 17:10:43 +0200 Subject: [PATCH 053/168] ARM: move AES typedefs and function prototypes to separate header Put the struct definitions for AES keys and the asm function prototypes in a separate header and export the asm functions from the module. This allows other drivers to use them directly. Change-Id: I997f8de3bac3e2bd0726cc2781470a7bd1dd4155 Signed-off-by: Ard Biesheuvel Git-commit: dadc77a9414954118fc4f4bdfe3307ec64f22e80 Git-repo: https://android.googlesource.com/kernel/common/ Signed-off-by: Samir Mehta --- arch/arm/crypto/aes_glue.c | 22 ++++++---------------- arch/arm/crypto/aes_glue.h | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 16 deletions(-) create mode 100644 arch/arm/crypto/aes_glue.h diff --git a/arch/arm/crypto/aes_glue.c b/arch/arm/crypto/aes_glue.c index 59f7877ead6..3003fa1f6fb 100644 --- a/arch/arm/crypto/aes_glue.c +++ b/arch/arm/crypto/aes_glue.c @@ -6,22 +6,12 @@ #include #include -#define AES_MAXNR 14 +#include "aes_glue.h" -typedef struct { - unsigned int rd_key[4 *(AES_MAXNR + 1)]; - int rounds; -} AES_KEY; - -struct AES_CTX { - AES_KEY enc_key; - AES_KEY dec_key; -}; - -asmlinkage void AES_encrypt(const u8 *in, u8 *out, AES_KEY *ctx); -asmlinkage void AES_decrypt(const u8 *in, u8 *out, AES_KEY *ctx); -asmlinkage int private_AES_set_decrypt_key(const unsigned char *userKey, const int bits, AES_KEY *key); -asmlinkage int private_AES_set_encrypt_key(const unsigned char *userKey, const int bits, AES_KEY *key); +EXPORT_SYMBOL(AES_encrypt); +EXPORT_SYMBOL(AES_decrypt); +EXPORT_SYMBOL(private_AES_set_encrypt_key); +EXPORT_SYMBOL(private_AES_set_decrypt_key); static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { @@ -81,7 +71,7 @@ static struct crypto_alg aes_alg = { .cipher = { .cia_min_keysize = AES_MIN_KEY_SIZE, .cia_max_keysize = AES_MAX_KEY_SIZE, - .cia_setkey = aes_set_key, + .cia_setkey = aes_set_key, .cia_encrypt = aes_encrypt, .cia_decrypt = aes_decrypt } diff --git a/arch/arm/crypto/aes_glue.h b/arch/arm/crypto/aes_glue.h new file mode 100644 index 00000000000..cca3e51eb60 --- /dev/null +++ b/arch/arm/crypto/aes_glue.h @@ -0,0 +1,19 @@ + +#define AES_MAXNR 14 + +struct AES_KEY { + unsigned int rd_key[4 * (AES_MAXNR + 1)]; + int rounds; +}; + +struct AES_CTX { + struct AES_KEY enc_key; + struct AES_KEY dec_key; +}; + +asmlinkage void AES_encrypt(const u8 *in, u8 *out, struct AES_KEY *ctx); +asmlinkage void AES_decrypt(const u8 *in, u8 *out, struct AES_KEY *ctx); +asmlinkage int private_AES_set_decrypt_key(const unsigned char *userKey, + const int bits, struct AES_KEY *key); +asmlinkage int private_AES_set_encrypt_key(const unsigned char *userKey, + const int bits, struct AES_KEY *key); From f42c928cced938ed114f3e4352ffdcca447340b8 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 16 Sep 2013 18:31:38 +0200 Subject: [PATCH 054/168] ARM: add support for bit sliced AES using NEON instructions Bit sliced AES gives around 45% speedup on Cortex-A15 for encryption and around 25% for decryption. This implementation of the AES algorithm does not rely on any lookup tables so it is believed to be invulnerable to cache timing attacks. This algorithm processes up to 8 blocks in parallel in constant time. This means that it is not usable by chaining modes that are strictly sequential in nature, such as CBC encryption. CBC decryption, however, can benefit from this implementation and runs about 25% faster. The other chaining modes implemented in this module, XTS and CTR, can execute fully in parallel in both directions. The core code has been adopted from the OpenSSL project (in collaboration with the original author, on cc). For ease of maintenance, this version is identical to the upstream OpenSSL code, i.e., all modifications that were required to make it suitable for inclusion into the kernel have been made upstream. The original can be found here: http://git.openssl.org/gitweb/?p=openssl.git;a=commit;h=6f6a6130 Note to integrators: While this implementation is significantly faster than the existing table based ones (generic or ARM asm), especially in CTR mode, the effects on power efficiency are unclear as of yet. This code does fundamentally more work, by calculating values that the table based code obtains by a simple lookup; only by doing all of that work in a SIMD fashion, it manages to perform better. Change-Id: I74b080141f5a505e5749922f41920b80e319b709 Cc: Andy Polyakov Acked-by: Nicolas Pitre Signed-off-by: Ard Biesheuvel Git-commit: 85c6f44dcce110b4b5ab994a2b88baf346299058 Git-repo: https://android.googlesource.com/kernel/common/ Signed-off-by: Samir Mehta --- arch/arm/crypto/Makefile | 14 +- arch/arm/crypto/aesbs-core.S_shipped | 2544 ++++++++++++++++++++++++++ arch/arm/crypto/aesbs-glue.c | 434 +++++ arch/arm/crypto/bsaes-armv7.pl | 2467 +++++++++++++++++++++++++ crypto/Kconfig | 16 + 5 files changed, 5473 insertions(+), 2 deletions(-) create mode 100644 arch/arm/crypto/aesbs-core.S_shipped create mode 100644 arch/arm/crypto/aesbs-glue.c create mode 100644 arch/arm/crypto/bsaes-armv7.pl diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile index a2c83851bc9..81cda39860c 100644 --- a/arch/arm/crypto/Makefile +++ b/arch/arm/crypto/Makefile @@ -3,7 +3,17 @@ # obj-$(CONFIG_CRYPTO_AES_ARM) += aes-arm.o +obj-$(CONFIG_CRYPTO_AES_ARM_BS) += aes-arm-bs.o obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o -aes-arm-y := aes-armv4.o aes_glue.o -sha1-arm-y := sha1-armv4-large.o sha1_glue.o +aes-arm-y := aes-armv4.o aes_glue.o +aes-arm-bs-y := aesbs-core.o aesbs-glue.o +sha1-arm-y := sha1-armv4-large.o sha1_glue.o + +quiet_cmd_perl = PERL $@ + cmd_perl = $(PERL) $(<) > $(@) + +$(src)/aesbs-core.S_shipped: $(src)/bsaes-armv7.pl + $(call cmd,perl) + +.PRECIOUS: $(obj)/aesbs-core.S diff --git a/arch/arm/crypto/aesbs-core.S_shipped b/arch/arm/crypto/aesbs-core.S_shipped new file mode 100644 index 00000000000..64205d45326 --- /dev/null +++ b/arch/arm/crypto/aesbs-core.S_shipped @@ -0,0 +1,2544 @@ + +@ ==================================================================== +@ Written by Andy Polyakov for the OpenSSL +@ project. The module is, however, dual licensed under OpenSSL and +@ CRYPTOGAMS licenses depending on where you obtain it. For further +@ details see http://www.openssl.org/~appro/cryptogams/. +@ +@ Specific modes and adaptation for Linux kernel by Ard Biesheuvel +@ . Permission to use under GPL terms is +@ granted. +@ ==================================================================== + +@ Bit-sliced AES for ARM NEON +@ +@ February 2012. +@ +@ This implementation is direct adaptation of bsaes-x86_64 module for +@ ARM NEON. Except that this module is endian-neutral [in sense that +@ it can be compiled for either endianness] by courtesy of vld1.8's +@ neutrality. Initial version doesn't implement interface to OpenSSL, +@ only low-level primitives and unsupported entry points, just enough +@ to collect performance results, which for Cortex-A8 core are: +@ +@ encrypt 19.5 cycles per byte processed with 128-bit key +@ decrypt 22.1 cycles per byte processed with 128-bit key +@ key conv. 440 cycles per 128-bit key/0.18 of 8x block +@ +@ Snapdragon S4 encrypts byte in 17.6 cycles and decrypts in 19.7, +@ which is [much] worse than anticipated (for further details see +@ http://www.openssl.org/~appro/Snapdragon-S4.html). +@ +@ Cortex-A15 manages in 14.2/16.1 cycles [when integer-only code +@ manages in 20.0 cycles]. +@ +@ When comparing to x86_64 results keep in mind that NEON unit is +@ [mostly] single-issue and thus can't [fully] benefit from +@ instruction-level parallelism. And when comparing to aes-armv4 +@ results keep in mind key schedule conversion overhead (see +@ bsaes-x86_64.pl for further details)... +@ +@ + +@ April-August 2013 +@ +@ Add CBC, CTR and XTS subroutines, adapt for kernel use. +@ +@ + +#ifndef __KERNEL__ +# include "arm_arch.h" + +# define VFP_ABI_PUSH vstmdb sp!,{d8-d15} +# define VFP_ABI_POP vldmia sp!,{d8-d15} +# define VFP_ABI_FRAME 0x40 +#else +# define VFP_ABI_PUSH +# define VFP_ABI_POP +# define VFP_ABI_FRAME 0 +# define BSAES_ASM_EXTENDED_KEY +# define XTS_CHAIN_TWEAK +# define __ARM_ARCH__ __LINUX_ARM_ARCH__ +#endif + +#ifdef __thumb__ +# define adrl adr +#endif + +#if __ARM_ARCH__>=7 +.text +.syntax unified @ ARMv7-capable assembler is expected to handle this +#ifdef __thumb2__ +.thumb +#else +.code 32 +#endif + +.fpu neon + +.type _bsaes_decrypt8,%function +.align 4 +_bsaes_decrypt8: + adr r6,_bsaes_decrypt8 + vldmia r4!, {q9} @ round 0 key + add r6,r6,#.LM0ISR-_bsaes_decrypt8 + + vldmia r6!, {q8} @ .LM0ISR + veor q10, q0, q9 @ xor with round0 key + veor q11, q1, q9 + vtbl.8 d0, {q10}, d16 + vtbl.8 d1, {q10}, d17 + veor q12, q2, q9 + vtbl.8 d2, {q11}, d16 + vtbl.8 d3, {q11}, d17 + veor q13, q3, q9 + vtbl.8 d4, {q12}, d16 + vtbl.8 d5, {q12}, d17 + veor q14, q4, q9 + vtbl.8 d6, {q13}, d16 + vtbl.8 d7, {q13}, d17 + veor q15, q5, q9 + vtbl.8 d8, {q14}, d16 + vtbl.8 d9, {q14}, d17 + veor q10, q6, q9 + vtbl.8 d10, {q15}, d16 + vtbl.8 d11, {q15}, d17 + veor q11, q7, q9 + vtbl.8 d12, {q10}, d16 + vtbl.8 d13, {q10}, d17 + vtbl.8 d14, {q11}, d16 + vtbl.8 d15, {q11}, d17 + vmov.i8 q8,#0x55 @ compose .LBS0 + vmov.i8 q9,#0x33 @ compose .LBS1 + vshr.u64 q10, q6, #1 + vshr.u64 q11, q4, #1 + veor q10, q10, q7 + veor q11, q11, q5 + vand q10, q10, q8 + vand q11, q11, q8 + veor q7, q7, q10 + vshl.u64 q10, q10, #1 + veor q5, q5, q11 + vshl.u64 q11, q11, #1 + veor q6, q6, q10 + veor q4, q4, q11 + vshr.u64 q10, q2, #1 + vshr.u64 q11, q0, #1 + veor q10, q10, q3 + veor q11, q11, q1 + vand q10, q10, q8 + vand q11, q11, q8 + veor q3, q3, q10 + vshl.u64 q10, q10, #1 + veor q1, q1, q11 + vshl.u64 q11, q11, #1 + veor q2, q2, q10 + veor q0, q0, q11 + vmov.i8 q8,#0x0f @ compose .LBS2 + vshr.u64 q10, q5, #2 + vshr.u64 q11, q4, #2 + veor q10, q10, q7 + veor q11, q11, q6 + vand q10, q10, q9 + vand q11, q11, q9 + veor q7, q7, q10 + vshl.u64 q10, q10, #2 + veor q6, q6, q11 + vshl.u64 q11, q11, #2 + veor q5, q5, q10 + veor q4, q4, q11 + vshr.u64 q10, q1, #2 + vshr.u64 q11, q0, #2 + veor q10, q10, q3 + veor q11, q11, q2 + vand q10, q10, q9 + vand q11, q11, q9 + veor q3, q3, q10 + vshl.u64 q10, q10, #2 + veor q2, q2, q11 + vshl.u64 q11, q11, #2 + veor q1, q1, q10 + veor q0, q0, q11 + vshr.u64 q10, q3, #4 + vshr.u64 q11, q2, #4 + veor q10, q10, q7 + veor q11, q11, q6 + vand q10, q10, q8 + vand q11, q11, q8 + veor q7, q7, q10 + vshl.u64 q10, q10, #4 + veor q6, q6, q11 + vshl.u64 q11, q11, #4 + veor q3, q3, q10 + veor q2, q2, q11 + vshr.u64 q10, q1, #4 + vshr.u64 q11, q0, #4 + veor q10, q10, q5 + veor q11, q11, q4 + vand q10, q10, q8 + vand q11, q11, q8 + veor q5, q5, q10 + vshl.u64 q10, q10, #4 + veor q4, q4, q11 + vshl.u64 q11, q11, #4 + veor q1, q1, q10 + veor q0, q0, q11 + sub r5,r5,#1 + b .Ldec_sbox +.align 4 +.Ldec_loop: + vldmia r4!, {q8-q11} + veor q8, q8, q0 + veor q9, q9, q1 + vtbl.8 d0, {q8}, d24 + vtbl.8 d1, {q8}, d25 + vldmia r4!, {q8} + veor q10, q10, q2 + vtbl.8 d2, {q9}, d24 + vtbl.8 d3, {q9}, d25 + vldmia r4!, {q9} + veor q11, q11, q3 + vtbl.8 d4, {q10}, d24 + vtbl.8 d5, {q10}, d25 + vldmia r4!, {q10} + vtbl.8 d6, {q11}, d24 + vtbl.8 d7, {q11}, d25 + vldmia r4!, {q11} + veor q8, q8, q4 + veor q9, q9, q5 + vtbl.8 d8, {q8}, d24 + vtbl.8 d9, {q8}, d25 + veor q10, q10, q6 + vtbl.8 d10, {q9}, d24 + vtbl.8 d11, {q9}, d25 + veor q11, q11, q7 + vtbl.8 d12, {q10}, d24 + vtbl.8 d13, {q10}, d25 + vtbl.8 d14, {q11}, d24 + vtbl.8 d15, {q11}, d25 +.Ldec_sbox: + veor q1, q1, q4 + veor q3, q3, q4 + + veor q4, q4, q7 + veor q1, q1, q6 + veor q2, q2, q7 + veor q6, q6, q4 + + veor q0, q0, q1 + veor q2, q2, q5 + veor q7, q7, q6 + veor q3, q3, q0 + veor q5, q5, q0 + veor q1, q1, q3 + veor q11, q3, q0 + veor q10, q7, q4 + veor q9, q1, q6 + veor q13, q4, q0 + vmov q8, q10 + veor q12, q5, q2 + + vorr q10, q10, q9 + veor q15, q11, q8 + vand q14, q11, q12 + vorr q11, q11, q12 + veor q12, q12, q9 + vand q8, q8, q9 + veor q9, q6, q2 + vand q15, q15, q12 + vand q13, q13, q9 + veor q9, q3, q7 + veor q12, q1, q5 + veor q11, q11, q13 + veor q10, q10, q13 + vand q13, q9, q12 + vorr q9, q9, q12 + veor q11, q11, q15 + veor q8, q8, q13 + veor q10, q10, q14 + veor q9, q9, q15 + veor q8, q8, q14 + vand q12, q4, q6 + veor q9, q9, q14 + vand q13, q0, q2 + vand q14, q7, q1 + vorr q15, q3, q5 + veor q11, q11, q12 + veor q9, q9, q14 + veor q8, q8, q15 + veor q10, q10, q13 + + @ Inv_GF16 0, 1, 2, 3, s0, s1, s2, s3 + + @ new smaller inversion + + vand q14, q11, q9 + vmov q12, q8 + + veor q13, q10, q14 + veor q15, q8, q14 + veor q14, q8, q14 @ q14=q15 + + vbsl q13, q9, q8 + vbsl q15, q11, q10 + veor q11, q11, q10 + + vbsl q12, q13, q14 + vbsl q8, q14, q13 + + vand q14, q12, q15 + veor q9, q9, q8 + + veor q14, q14, q11 + veor q12, q5, q2 + veor q8, q1, q6 + veor q10, q15, q14 + vand q10, q10, q5 + veor q5, q5, q1 + vand q11, q1, q15 + vand q5, q5, q14 + veor q1, q11, q10 + veor q5, q5, q11 + veor q15, q15, q13 + veor q14, q14, q9 + veor q11, q15, q14 + veor q10, q13, q9 + vand q11, q11, q12 + vand q10, q10, q2 + veor q12, q12, q8 + veor q2, q2, q6 + vand q8, q8, q15 + vand q6, q6, q13 + vand q12, q12, q14 + vand q2, q2, q9 + veor q8, q8, q12 + veor q2, q2, q6 + veor q12, q12, q11 + veor q6, q6, q10 + veor q5, q5, q12 + veor q2, q2, q12 + veor q1, q1, q8 + veor q6, q6, q8 + + veor q12, q3, q0 + veor q8, q7, q4 + veor q11, q15, q14 + veor q10, q13, q9 + vand q11, q11, q12 + vand q10, q10, q0 + veor q12, q12, q8 + veor q0, q0, q4 + vand q8, q8, q15 + vand q4, q4, q13 + vand q12, q12, q14 + vand q0, q0, q9 + veor q8, q8, q12 + veor q0, q0, q4 + veor q12, q12, q11 + veor q4, q4, q10 + veor q15, q15, q13 + veor q14, q14, q9 + veor q10, q15, q14 + vand q10, q10, q3 + veor q3, q3, q7 + vand q11, q7, q15 + vand q3, q3, q14 + veor q7, q11, q10 + veor q3, q3, q11 + veor q3, q3, q12 + veor q0, q0, q12 + veor q7, q7, q8 + veor q4, q4, q8 + veor q1, q1, q7 + veor q6, q6, q5 + + veor q4, q4, q1 + veor q2, q2, q7 + veor q5, q5, q7 + veor q4, q4, q2 + veor q7, q7, q0 + veor q4, q4, q5 + veor q3, q3, q6 + veor q6, q6, q1 + veor q3, q3, q4 + + veor q4, q4, q0 + veor q7, q7, q3 + subs r5,r5,#1 + bcc .Ldec_done + @ multiplication by 0x05-0x00-0x04-0x00 + vext.8 q8, q0, q0, #8 + vext.8 q14, q3, q3, #8 + vext.8 q15, q5, q5, #8 + veor q8, q8, q0 + vext.8 q9, q1, q1, #8 + veor q14, q14, q3 + vext.8 q10, q6, q6, #8 + veor q15, q15, q5 + vext.8 q11, q4, q4, #8 + veor q9, q9, q1 + vext.8 q12, q2, q2, #8 + veor q10, q10, q6 + vext.8 q13, q7, q7, #8 + veor q11, q11, q4 + veor q12, q12, q2 + veor q13, q13, q7 + + veor q0, q0, q14 + veor q1, q1, q14 + veor q6, q6, q8 + veor q2, q2, q10 + veor q4, q4, q9 + veor q1, q1, q15 + veor q6, q6, q15 + veor q2, q2, q14 + veor q7, q7, q11 + veor q4, q4, q14 + veor q3, q3, q12 + veor q2, q2, q15 + veor q7, q7, q15 + veor q5, q5, q13 + vext.8 q8, q0, q0, #12 @ x0 <<< 32 + vext.8 q9, q1, q1, #12 + veor q0, q0, q8 @ x0 ^ (x0 <<< 32) + vext.8 q10, q6, q6, #12 + veor q1, q1, q9 + vext.8 q11, q4, q4, #12 + veor q6, q6, q10 + vext.8 q12, q2, q2, #12 + veor q4, q4, q11 + vext.8 q13, q7, q7, #12 + veor q2, q2, q12 + vext.8 q14, q3, q3, #12 + veor q7, q7, q13 + vext.8 q15, q5, q5, #12 + veor q3, q3, q14 + + veor q9, q9, q0 + veor q5, q5, q15 + vext.8 q0, q0, q0, #8 @ (x0 ^ (x0 <<< 32)) <<< 64) + veor q10, q10, q1 + veor q8, q8, q5 + veor q9, q9, q5 + vext.8 q1, q1, q1, #8 + veor q13, q13, q2 + veor q0, q0, q8 + veor q14, q14, q7 + veor q1, q1, q9 + vext.8 q8, q2, q2, #8 + veor q12, q12, q4 + vext.8 q9, q7, q7, #8 + veor q15, q15, q3 + vext.8 q2, q4, q4, #8 + veor q11, q11, q6 + vext.8 q7, q5, q5, #8 + veor q12, q12, q5 + vext.8 q4, q3, q3, #8 + veor q11, q11, q5 + vext.8 q3, q6, q6, #8 + veor q5, q9, q13 + veor q11, q11, q2 + veor q7, q7, q15 + veor q6, q4, q14 + veor q4, q8, q12 + veor q2, q3, q10 + vmov q3, q11 + @ vmov q5, q9 + vldmia r6, {q12} @ .LISR + ite eq @ Thumb2 thing, sanity check in ARM + addeq r6,r6,#0x10 + bne .Ldec_loop + vldmia r6, {q12} @ .LISRM0 + b .Ldec_loop +.align 4 +.Ldec_done: + vmov.i8 q8,#0x55 @ compose .LBS0 + vmov.i8 q9,#0x33 @ compose .LBS1 + vshr.u64 q10, q3, #1 + vshr.u64 q11, q2, #1 + veor q10, q10, q5 + veor q11, q11, q7 + vand q10, q10, q8 + vand q11, q11, q8 + veor q5, q5, q10 + vshl.u64 q10, q10, #1 + veor q7, q7, q11 + vshl.u64 q11, q11, #1 + veor q3, q3, q10 + veor q2, q2, q11 + vshr.u64 q10, q6, #1 + vshr.u64 q11, q0, #1 + veor q10, q10, q4 + veor q11, q11, q1 + vand q10, q10, q8 + vand q11, q11, q8 + veor q4, q4, q10 + vshl.u64 q10, q10, #1 + veor q1, q1, q11 + vshl.u64 q11, q11, #1 + veor q6, q6, q10 + veor q0, q0, q11 + vmov.i8 q8,#0x0f @ compose .LBS2 + vshr.u64 q10, q7, #2 + vshr.u64 q11, q2, #2 + veor q10, q10, q5 + veor q11, q11, q3 + vand q10, q10, q9 + vand q11, q11, q9 + veor q5, q5, q10 + vshl.u64 q10, q10, #2 + veor q3, q3, q11 + vshl.u64 q11, q11, #2 + veor q7, q7, q10 + veor q2, q2, q11 + vshr.u64 q10, q1, #2 + vshr.u64 q11, q0, #2 + veor q10, q10, q4 + veor q11, q11, q6 + vand q10, q10, q9 + vand q11, q11, q9 + veor q4, q4, q10 + vshl.u64 q10, q10, #2 + veor q6, q6, q11 + vshl.u64 q11, q11, #2 + veor q1, q1, q10 + veor q0, q0, q11 + vshr.u64 q10, q4, #4 + vshr.u64 q11, q6, #4 + veor q10, q10, q5 + veor q11, q11, q3 + vand q10, q10, q8 + vand q11, q11, q8 + veor q5, q5, q10 + vshl.u64 q10, q10, #4 + veor q3, q3, q11 + vshl.u64 q11, q11, #4 + veor q4, q4, q10 + veor q6, q6, q11 + vshr.u64 q10, q1, #4 + vshr.u64 q11, q0, #4 + veor q10, q10, q7 + veor q11, q11, q2 + vand q10, q10, q8 + vand q11, q11, q8 + veor q7, q7, q10 + vshl.u64 q10, q10, #4 + veor q2, q2, q11 + vshl.u64 q11, q11, #4 + veor q1, q1, q10 + veor q0, q0, q11 + vldmia r4, {q8} @ last round key + veor q6, q6, q8 + veor q4, q4, q8 + veor q2, q2, q8 + veor q7, q7, q8 + veor q3, q3, q8 + veor q5, q5, q8 + veor q0, q0, q8 + veor q1, q1, q8 + bx lr +.size _bsaes_decrypt8,.-_bsaes_decrypt8 + +.type _bsaes_const,%object +.align 6 +_bsaes_const: +.LM0ISR: @ InvShiftRows constants + .quad 0x0a0e0206070b0f03, 0x0004080c0d010509 +.LISR: + .quad 0x0504070602010003, 0x0f0e0d0c080b0a09 +.LISRM0: + .quad 0x01040b0e0205080f, 0x0306090c00070a0d +.LM0SR: @ ShiftRows constants + .quad 0x0a0e02060f03070b, 0x0004080c05090d01 +.LSR: + .quad 0x0504070600030201, 0x0f0e0d0c0a09080b +.LSRM0: + .quad 0x0304090e00050a0f, 0x01060b0c0207080d +.LM0: + .quad 0x02060a0e03070b0f, 0x0004080c0105090d +.LREVM0SR: + .quad 0x090d01050c000408, 0x03070b0f060a0e02 +.asciz "Bit-sliced AES for NEON, CRYPTOGAMS by " +.align 6 +.size _bsaes_const,.-_bsaes_const + +.type _bsaes_encrypt8,%function +.align 4 +_bsaes_encrypt8: + adr r6,_bsaes_encrypt8 + vldmia r4!, {q9} @ round 0 key + sub r6,r6,#_bsaes_encrypt8-.LM0SR + + vldmia r6!, {q8} @ .LM0SR +_bsaes_encrypt8_alt: + veor q10, q0, q9 @ xor with round0 key + veor q11, q1, q9 + vtbl.8 d0, {q10}, d16 + vtbl.8 d1, {q10}, d17 + veor q12, q2, q9 + vtbl.8 d2, {q11}, d16 + vtbl.8 d3, {q11}, d17 + veor q13, q3, q9 + vtbl.8 d4, {q12}, d16 + vtbl.8 d5, {q12}, d17 + veor q14, q4, q9 + vtbl.8 d6, {q13}, d16 + vtbl.8 d7, {q13}, d17 + veor q15, q5, q9 + vtbl.8 d8, {q14}, d16 + vtbl.8 d9, {q14}, d17 + veor q10, q6, q9 + vtbl.8 d10, {q15}, d16 + vtbl.8 d11, {q15}, d17 + veor q11, q7, q9 + vtbl.8 d12, {q10}, d16 + vtbl.8 d13, {q10}, d17 + vtbl.8 d14, {q11}, d16 + vtbl.8 d15, {q11}, d17 +_bsaes_encrypt8_bitslice: + vmov.i8 q8,#0x55 @ compose .LBS0 + vmov.i8 q9,#0x33 @ compose .LBS1 + vshr.u64 q10, q6, #1 + vshr.u64 q11, q4, #1 + veor q10, q10, q7 + veor q11, q11, q5 + vand q10, q10, q8 + vand q11, q11, q8 + veor q7, q7, q10 + vshl.u64 q10, q10, #1 + veor q5, q5, q11 + vshl.u64 q11, q11, #1 + veor q6, q6, q10 + veor q4, q4, q11 + vshr.u64 q10, q2, #1 + vshr.u64 q11, q0, #1 + veor q10, q10, q3 + veor q11, q11, q1 + vand q10, q10, q8 + vand q11, q11, q8 + veor q3, q3, q10 + vshl.u64 q10, q10, #1 + veor q1, q1, q11 + vshl.u64 q11, q11, #1 + veor q2, q2, q10 + veor q0, q0, q11 + vmov.i8 q8,#0x0f @ compose .LBS2 + vshr.u64 q10, q5, #2 + vshr.u64 q11, q4, #2 + veor q10, q10, q7 + veor q11, q11, q6 + vand q10, q10, q9 + vand q11, q11, q9 + veor q7, q7, q10 + vshl.u64 q10, q10, #2 + veor q6, q6, q11 + vshl.u64 q11, q11, #2 + veor q5, q5, q10 + veor q4, q4, q11 + vshr.u64 q10, q1, #2 + vshr.u64 q11, q0, #2 + veor q10, q10, q3 + veor q11, q11, q2 + vand q10, q10, q9 + vand q11, q11, q9 + veor q3, q3, q10 + vshl.u64 q10, q10, #2 + veor q2, q2, q11 + vshl.u64 q11, q11, #2 + veor q1, q1, q10 + veor q0, q0, q11 + vshr.u64 q10, q3, #4 + vshr.u64 q11, q2, #4 + veor q10, q10, q7 + veor q11, q11, q6 + vand q10, q10, q8 + vand q11, q11, q8 + veor q7, q7, q10 + vshl.u64 q10, q10, #4 + veor q6, q6, q11 + vshl.u64 q11, q11, #4 + veor q3, q3, q10 + veor q2, q2, q11 + vshr.u64 q10, q1, #4 + vshr.u64 q11, q0, #4 + veor q10, q10, q5 + veor q11, q11, q4 + vand q10, q10, q8 + vand q11, q11, q8 + veor q5, q5, q10 + vshl.u64 q10, q10, #4 + veor q4, q4, q11 + vshl.u64 q11, q11, #4 + veor q1, q1, q10 + veor q0, q0, q11 + sub r5,r5,#1 + b .Lenc_sbox +.align 4 +.Lenc_loop: + vldmia r4!, {q8-q11} + veor q8, q8, q0 + veor q9, q9, q1 + vtbl.8 d0, {q8}, d24 + vtbl.8 d1, {q8}, d25 + vldmia r4!, {q8} + veor q10, q10, q2 + vtbl.8 d2, {q9}, d24 + vtbl.8 d3, {q9}, d25 + vldmia r4!, {q9} + veor q11, q11, q3 + vtbl.8 d4, {q10}, d24 + vtbl.8 d5, {q10}, d25 + vldmia r4!, {q10} + vtbl.8 d6, {q11}, d24 + vtbl.8 d7, {q11}, d25 + vldmia r4!, {q11} + veor q8, q8, q4 + veor q9, q9, q5 + vtbl.8 d8, {q8}, d24 + vtbl.8 d9, {q8}, d25 + veor q10, q10, q6 + vtbl.8 d10, {q9}, d24 + vtbl.8 d11, {q9}, d25 + veor q11, q11, q7 + vtbl.8 d12, {q10}, d24 + vtbl.8 d13, {q10}, d25 + vtbl.8 d14, {q11}, d24 + vtbl.8 d15, {q11}, d25 +.Lenc_sbox: + veor q2, q2, q1 + veor q5, q5, q6 + veor q3, q3, q0 + veor q6, q6, q2 + veor q5, q5, q0 + + veor q6, q6, q3 + veor q3, q3, q7 + veor q7, q7, q5 + veor q3, q3, q4 + veor q4, q4, q5 + + veor q2, q2, q7 + veor q3, q3, q1 + veor q1, q1, q5 + veor q11, q7, q4 + veor q10, q1, q2 + veor q9, q5, q3 + veor q13, q2, q4 + vmov q8, q10 + veor q12, q6, q0 + + vorr q10, q10, q9 + veor q15, q11, q8 + vand q14, q11, q12 + vorr q11, q11, q12 + veor q12, q12, q9 + vand q8, q8, q9 + veor q9, q3, q0 + vand q15, q15, q12 + vand q13, q13, q9 + veor q9, q7, q1 + veor q12, q5, q6 + veor q11, q11, q13 + veor q10, q10, q13 + vand q13, q9, q12 + vorr q9, q9, q12 + veor q11, q11, q15 + veor q8, q8, q13 + veor q10, q10, q14 + veor q9, q9, q15 + veor q8, q8, q14 + vand q12, q2, q3 + veor q9, q9, q14 + vand q13, q4, q0 + vand q14, q1, q5 + vorr q15, q7, q6 + veor q11, q11, q12 + veor q9, q9, q14 + veor q8, q8, q15 + veor q10, q10, q13 + + @ Inv_GF16 0, 1, 2, 3, s0, s1, s2, s3 + + @ new smaller inversion + + vand q14, q11, q9 + vmov q12, q8 + + veor q13, q10, q14 + veor q15, q8, q14 + veor q14, q8, q14 @ q14=q15 + + vbsl q13, q9, q8 + vbsl q15, q11, q10 + veor q11, q11, q10 + + vbsl q12, q13, q14 + vbsl q8, q14, q13 + + vand q14, q12, q15 + veor q9, q9, q8 + + veor q14, q14, q11 + veor q12, q6, q0 + veor q8, q5, q3 + veor q10, q15, q14 + vand q10, q10, q6 + veor q6, q6, q5 + vand q11, q5, q15 + vand q6, q6, q14 + veor q5, q11, q10 + veor q6, q6, q11 + veor q15, q15, q13 + veor q14, q14, q9 + veor q11, q15, q14 + veor q10, q13, q9 + vand q11, q11, q12 + vand q10, q10, q0 + veor q12, q12, q8 + veor q0, q0, q3 + vand q8, q8, q15 + vand q3, q3, q13 + vand q12, q12, q14 + vand q0, q0, q9 + veor q8, q8, q12 + veor q0, q0, q3 + veor q12, q12, q11 + veor q3, q3, q10 + veor q6, q6, q12 + veor q0, q0, q12 + veor q5, q5, q8 + veor q3, q3, q8 + + veor q12, q7, q4 + veor q8, q1, q2 + veor q11, q15, q14 + veor q10, q13, q9 + vand q11, q11, q12 + vand q10, q10, q4 + veor q12, q12, q8 + veor q4, q4, q2 + vand q8, q8, q15 + vand q2, q2, q13 + vand q12, q12, q14 + vand q4, q4, q9 + veor q8, q8, q12 + veor q4, q4, q2 + veor q12, q12, q11 + veor q2, q2, q10 + veor q15, q15, q13 + veor q14, q14, q9 + veor q10, q15, q14 + vand q10, q10, q7 + veor q7, q7, q1 + vand q11, q1, q15 + vand q7, q7, q14 + veor q1, q11, q10 + veor q7, q7, q11 + veor q7, q7, q12 + veor q4, q4, q12 + veor q1, q1, q8 + veor q2, q2, q8 + veor q7, q7, q0 + veor q1, q1, q6 + veor q6, q6, q0 + veor q4, q4, q7 + veor q0, q0, q1 + + veor q1, q1, q5 + veor q5, q5, q2 + veor q2, q2, q3 + veor q3, q3, q5 + veor q4, q4, q5 + + veor q6, q6, q3 + subs r5,r5,#1 + bcc .Lenc_done + vext.8 q8, q0, q0, #12 @ x0 <<< 32 + vext.8 q9, q1, q1, #12 + veor q0, q0, q8 @ x0 ^ (x0 <<< 32) + vext.8 q10, q4, q4, #12 + veor q1, q1, q9 + vext.8 q11, q6, q6, #12 + veor q4, q4, q10 + vext.8 q12, q3, q3, #12 + veor q6, q6, q11 + vext.8 q13, q7, q7, #12 + veor q3, q3, q12 + vext.8 q14, q2, q2, #12 + veor q7, q7, q13 + vext.8 q15, q5, q5, #12 + veor q2, q2, q14 + + veor q9, q9, q0 + veor q5, q5, q15 + vext.8 q0, q0, q0, #8 @ (x0 ^ (x0 <<< 32)) <<< 64) + veor q10, q10, q1 + veor q8, q8, q5 + veor q9, q9, q5 + vext.8 q1, q1, q1, #8 + veor q13, q13, q3 + veor q0, q0, q8 + veor q14, q14, q7 + veor q1, q1, q9 + vext.8 q8, q3, q3, #8 + veor q12, q12, q6 + vext.8 q9, q7, q7, #8 + veor q15, q15, q2 + vext.8 q3, q6, q6, #8 + veor q11, q11, q4 + vext.8 q7, q5, q5, #8 + veor q12, q12, q5 + vext.8 q6, q2, q2, #8 + veor q11, q11, q5 + vext.8 q2, q4, q4, #8 + veor q5, q9, q13 + veor q4, q8, q12 + veor q3, q3, q11 + veor q7, q7, q15 + veor q6, q6, q14 + @ vmov q4, q8 + veor q2, q2, q10 + @ vmov q5, q9 + vldmia r6, {q12} @ .LSR + ite eq @ Thumb2 thing, samity check in ARM + addeq r6,r6,#0x10 + bne .Lenc_loop + vldmia r6, {q12} @ .LSRM0 + b .Lenc_loop +.align 4 +.Lenc_done: + vmov.i8 q8,#0x55 @ compose .LBS0 + vmov.i8 q9,#0x33 @ compose .LBS1 + vshr.u64 q10, q2, #1 + vshr.u64 q11, q3, #1 + veor q10, q10, q5 + veor q11, q11, q7 + vand q10, q10, q8 + vand q11, q11, q8 + veor q5, q5, q10 + vshl.u64 q10, q10, #1 + veor q7, q7, q11 + vshl.u64 q11, q11, #1 + veor q2, q2, q10 + veor q3, q3, q11 + vshr.u64 q10, q4, #1 + vshr.u64 q11, q0, #1 + veor q10, q10, q6 + veor q11, q11, q1 + vand q10, q10, q8 + vand q11, q11, q8 + veor q6, q6, q10 + vshl.u64 q10, q10, #1 + veor q1, q1, q11 + vshl.u64 q11, q11, #1 + veor q4, q4, q10 + veor q0, q0, q11 + vmov.i8 q8,#0x0f @ compose .LBS2 + vshr.u64 q10, q7, #2 + vshr.u64 q11, q3, #2 + veor q10, q10, q5 + veor q11, q11, q2 + vand q10, q10, q9 + vand q11, q11, q9 + veor q5, q5, q10 + vshl.u64 q10, q10, #2 + veor q2, q2, q11 + vshl.u64 q11, q11, #2 + veor q7, q7, q10 + veor q3, q3, q11 + vshr.u64 q10, q1, #2 + vshr.u64 q11, q0, #2 + veor q10, q10, q6 + veor q11, q11, q4 + vand q10, q10, q9 + vand q11, q11, q9 + veor q6, q6, q10 + vshl.u64 q10, q10, #2 + veor q4, q4, q11 + vshl.u64 q11, q11, #2 + veor q1, q1, q10 + veor q0, q0, q11 + vshr.u64 q10, q6, #4 + vshr.u64 q11, q4, #4 + veor q10, q10, q5 + veor q11, q11, q2 + vand q10, q10, q8 + vand q11, q11, q8 + veor q5, q5, q10 + vshl.u64 q10, q10, #4 + veor q2, q2, q11 + vshl.u64 q11, q11, #4 + veor q6, q6, q10 + veor q4, q4, q11 + vshr.u64 q10, q1, #4 + vshr.u64 q11, q0, #4 + veor q10, q10, q7 + veor q11, q11, q3 + vand q10, q10, q8 + vand q11, q11, q8 + veor q7, q7, q10 + vshl.u64 q10, q10, #4 + veor q3, q3, q11 + vshl.u64 q11, q11, #4 + veor q1, q1, q10 + veor q0, q0, q11 + vldmia r4, {q8} @ last round key + veor q4, q4, q8 + veor q6, q6, q8 + veor q3, q3, q8 + veor q7, q7, q8 + veor q2, q2, q8 + veor q5, q5, q8 + veor q0, q0, q8 + veor q1, q1, q8 + bx lr +.size _bsaes_encrypt8,.-_bsaes_encrypt8 +.type _bsaes_key_convert,%function +.align 4 +_bsaes_key_convert: + adr r6,_bsaes_key_convert + vld1.8 {q7}, [r4]! @ load round 0 key + sub r6,r6,#_bsaes_key_convert-.LM0 + vld1.8 {q15}, [r4]! @ load round 1 key + + vmov.i8 q8, #0x01 @ bit masks + vmov.i8 q9, #0x02 + vmov.i8 q10, #0x04 + vmov.i8 q11, #0x08 + vmov.i8 q12, #0x10 + vmov.i8 q13, #0x20 + vldmia r6, {q14} @ .LM0 + +#ifdef __ARMEL__ + vrev32.8 q7, q7 + vrev32.8 q15, q15 +#endif + sub r5,r5,#1 + vstmia r12!, {q7} @ save round 0 key + b .Lkey_loop + +.align 4 +.Lkey_loop: + vtbl.8 d14,{q15},d28 + vtbl.8 d15,{q15},d29 + vmov.i8 q6, #0x40 + vmov.i8 q15, #0x80 + + vtst.8 q0, q7, q8 + vtst.8 q1, q7, q9 + vtst.8 q2, q7, q10 + vtst.8 q3, q7, q11 + vtst.8 q4, q7, q12 + vtst.8 q5, q7, q13 + vtst.8 q6, q7, q6 + vtst.8 q7, q7, q15 + vld1.8 {q15}, [r4]! @ load next round key + vmvn q0, q0 @ "pnot" + vmvn q1, q1 + vmvn q5, q5 + vmvn q6, q6 +#ifdef __ARMEL__ + vrev32.8 q15, q15 +#endif + subs r5,r5,#1 + vstmia r12!,{q0-q7} @ write bit-sliced round key + bne .Lkey_loop + + vmov.i8 q7,#0x63 @ compose .L63 + @ don't save last round key + bx lr +.size _bsaes_key_convert,.-_bsaes_key_convert +.extern AES_cbc_encrypt +.extern AES_decrypt + +.global bsaes_cbc_encrypt +.type bsaes_cbc_encrypt,%function +.align 5 +bsaes_cbc_encrypt: +#ifndef __KERNEL__ + cmp r2, #128 +#ifndef __thumb__ + blo AES_cbc_encrypt +#else + bhs 1f + b AES_cbc_encrypt +1: +#endif +#endif + + @ it is up to the caller to make sure we are called with enc == 0 + + mov ip, sp + stmdb sp!, {r4-r10, lr} + VFP_ABI_PUSH + ldr r8, [ip] @ IV is 1st arg on the stack + mov r2, r2, lsr#4 @ len in 16 byte blocks + sub sp, #0x10 @ scratch space to carry over the IV + mov r9, sp @ save sp + + ldr r10, [r3, #240] @ get # of rounds +#ifndef BSAES_ASM_EXTENDED_KEY + @ allocate the key schedule on the stack + sub r12, sp, r10, lsl#7 @ 128 bytes per inner round key + add r12, #96 @ sifze of bit-slices key schedule + + @ populate the key schedule + mov r4, r3 @ pass key + mov r5, r10 @ pass # of rounds + mov sp, r12 @ sp is sp + bl _bsaes_key_convert + vldmia sp, {q6} + vstmia r12, {q15} @ save last round key + veor q7, q7, q6 @ fix up round 0 key + vstmia sp, {q7} +#else + ldr r12, [r3, #244] + eors r12, #1 + beq 0f + + @ populate the key schedule + str r12, [r3, #244] + mov r4, r3 @ pass key + mov r5, r10 @ pass # of rounds + add r12, r3, #248 @ pass key schedule + bl _bsaes_key_convert + add r4, r3, #248 + vldmia r4, {q6} + vstmia r12, {q15} @ save last round key + veor q7, q7, q6 @ fix up round 0 key + vstmia r4, {q7} + +.align 2 +0: +#endif + + vld1.8 {q15}, [r8] @ load IV + b .Lcbc_dec_loop + +.align 4 +.Lcbc_dec_loop: + subs r2, r2, #0x8 + bmi .Lcbc_dec_loop_finish + + vld1.8 {q0-q1}, [r0]! @ load input + vld1.8 {q2-q3}, [r0]! +#ifndef BSAES_ASM_EXTENDED_KEY + mov r4, sp @ pass the key +#else + add r4, r3, #248 +#endif + vld1.8 {q4-q5}, [r0]! + mov r5, r10 + vld1.8 {q6-q7}, [r0] + sub r0, r0, #0x60 + vstmia r9, {q15} @ put aside IV + + bl _bsaes_decrypt8 + + vldmia r9, {q14} @ reload IV + vld1.8 {q8-q9}, [r0]! @ reload input + veor q0, q0, q14 @ ^= IV + vld1.8 {q10-q11}, [r0]! + veor q1, q1, q8 + veor q6, q6, q9 + vld1.8 {q12-q13}, [r0]! + veor q4, q4, q10 + veor q2, q2, q11 + vld1.8 {q14-q15}, [r0]! + veor q7, q7, q12 + vst1.8 {q0-q1}, [r1]! @ write output + veor q3, q3, q13 + vst1.8 {q6}, [r1]! + veor q5, q5, q14 + vst1.8 {q4}, [r1]! + vst1.8 {q2}, [r1]! + vst1.8 {q7}, [r1]! + vst1.8 {q3}, [r1]! + vst1.8 {q5}, [r1]! + + b .Lcbc_dec_loop + +.Lcbc_dec_loop_finish: + adds r2, r2, #8 + beq .Lcbc_dec_done + + vld1.8 {q0}, [r0]! @ load input + cmp r2, #2 + blo .Lcbc_dec_one + vld1.8 {q1}, [r0]! +#ifndef BSAES_ASM_EXTENDED_KEY + mov r4, sp @ pass the key +#else + add r4, r3, #248 +#endif + mov r5, r10 + vstmia r9, {q15} @ put aside IV + beq .Lcbc_dec_two + vld1.8 {q2}, [r0]! + cmp r2, #4 + blo .Lcbc_dec_three + vld1.8 {q3}, [r0]! + beq .Lcbc_dec_four + vld1.8 {q4}, [r0]! + cmp r2, #6 + blo .Lcbc_dec_five + vld1.8 {q5}, [r0]! + beq .Lcbc_dec_six + vld1.8 {q6}, [r0]! + sub r0, r0, #0x70 + + bl _bsaes_decrypt8 + + vldmia r9, {q14} @ reload IV + vld1.8 {q8-q9}, [r0]! @ reload input + veor q0, q0, q14 @ ^= IV + vld1.8 {q10-q11}, [r0]! + veor q1, q1, q8 + veor q6, q6, q9 + vld1.8 {q12-q13}, [r0]! + veor q4, q4, q10 + veor q2, q2, q11 + vld1.8 {q15}, [r0]! + veor q7, q7, q12 + vst1.8 {q0-q1}, [r1]! @ write output + veor q3, q3, q13 + vst1.8 {q6}, [r1]! + vst1.8 {q4}, [r1]! + vst1.8 {q2}, [r1]! + vst1.8 {q7}, [r1]! + vst1.8 {q3}, [r1]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_six: + sub r0, r0, #0x60 + bl _bsaes_decrypt8 + vldmia r9,{q14} @ reload IV + vld1.8 {q8-q9}, [r0]! @ reload input + veor q0, q0, q14 @ ^= IV + vld1.8 {q10-q11}, [r0]! + veor q1, q1, q8 + veor q6, q6, q9 + vld1.8 {q12}, [r0]! + veor q4, q4, q10 + veor q2, q2, q11 + vld1.8 {q15}, [r0]! + veor q7, q7, q12 + vst1.8 {q0-q1}, [r1]! @ write output + vst1.8 {q6}, [r1]! + vst1.8 {q4}, [r1]! + vst1.8 {q2}, [r1]! + vst1.8 {q7}, [r1]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_five: + sub r0, r0, #0x50 + bl _bsaes_decrypt8 + vldmia r9, {q14} @ reload IV + vld1.8 {q8-q9}, [r0]! @ reload input + veor q0, q0, q14 @ ^= IV + vld1.8 {q10-q11}, [r0]! + veor q1, q1, q8 + veor q6, q6, q9 + vld1.8 {q15}, [r0]! + veor q4, q4, q10 + vst1.8 {q0-q1}, [r1]! @ write output + veor q2, q2, q11 + vst1.8 {q6}, [r1]! + vst1.8 {q4}, [r1]! + vst1.8 {q2}, [r1]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_four: + sub r0, r0, #0x40 + bl _bsaes_decrypt8 + vldmia r9, {q14} @ reload IV + vld1.8 {q8-q9}, [r0]! @ reload input + veor q0, q0, q14 @ ^= IV + vld1.8 {q10}, [r0]! + veor q1, q1, q8 + veor q6, q6, q9 + vld1.8 {q15}, [r0]! + veor q4, q4, q10 + vst1.8 {q0-q1}, [r1]! @ write output + vst1.8 {q6}, [r1]! + vst1.8 {q4}, [r1]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_three: + sub r0, r0, #0x30 + bl _bsaes_decrypt8 + vldmia r9, {q14} @ reload IV + vld1.8 {q8-q9}, [r0]! @ reload input + veor q0, q0, q14 @ ^= IV + vld1.8 {q15}, [r0]! + veor q1, q1, q8 + veor q6, q6, q9 + vst1.8 {q0-q1}, [r1]! @ write output + vst1.8 {q6}, [r1]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_two: + sub r0, r0, #0x20 + bl _bsaes_decrypt8 + vldmia r9, {q14} @ reload IV + vld1.8 {q8}, [r0]! @ reload input + veor q0, q0, q14 @ ^= IV + vld1.8 {q15}, [r0]! @ reload input + veor q1, q1, q8 + vst1.8 {q0-q1}, [r1]! @ write output + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_one: + sub r0, r0, #0x10 + mov r10, r1 @ save original out pointer + mov r1, r9 @ use the iv scratch space as out buffer + mov r2, r3 + vmov q4,q15 @ just in case ensure that IV + vmov q5,q0 @ and input are preserved + bl AES_decrypt + vld1.8 {q0}, [r9,:64] @ load result + veor q0, q0, q4 @ ^= IV + vmov q15, q5 @ q5 holds input + vst1.8 {q0}, [r10] @ write output + +.Lcbc_dec_done: +#ifndef BSAES_ASM_EXTENDED_KEY + vmov.i32 q0, #0 + vmov.i32 q1, #0 +.Lcbc_dec_bzero: @ wipe key schedule [if any] + vstmia sp!, {q0-q1} + cmp sp, r9 + bne .Lcbc_dec_bzero +#endif + + mov sp, r9 + add sp, #0x10 @ add sp,r9,#0x10 is no good for thumb + vst1.8 {q15}, [r8] @ return IV + VFP_ABI_POP + ldmia sp!, {r4-r10, pc} +.size bsaes_cbc_encrypt,.-bsaes_cbc_encrypt +.extern AES_encrypt +.global bsaes_ctr32_encrypt_blocks +.type bsaes_ctr32_encrypt_blocks,%function +.align 5 +bsaes_ctr32_encrypt_blocks: + cmp r2, #8 @ use plain AES for + blo .Lctr_enc_short @ small sizes + + mov ip, sp + stmdb sp!, {r4-r10, lr} + VFP_ABI_PUSH + ldr r8, [ip] @ ctr is 1st arg on the stack + sub sp, sp, #0x10 @ scratch space to carry over the ctr + mov r9, sp @ save sp + + ldr r10, [r3, #240] @ get # of rounds +#ifndef BSAES_ASM_EXTENDED_KEY + @ allocate the key schedule on the stack + sub r12, sp, r10, lsl#7 @ 128 bytes per inner round key + add r12, #96 @ size of bit-sliced key schedule + + @ populate the key schedule + mov r4, r3 @ pass key + mov r5, r10 @ pass # of rounds + mov sp, r12 @ sp is sp + bl _bsaes_key_convert + veor q7,q7,q15 @ fix up last round key + vstmia r12, {q7} @ save last round key + + vld1.8 {q0}, [r8] @ load counter + add r8, r6, #.LREVM0SR-.LM0 @ borrow r8 + vldmia sp, {q4} @ load round0 key +#else + ldr r12, [r3, #244] + eors r12, #1 + beq 0f + + @ populate the key schedule + str r12, [r3, #244] + mov r4, r3 @ pass key + mov r5, r10 @ pass # of rounds + add r12, r3, #248 @ pass key schedule + bl _bsaes_key_convert + veor q7,q7,q15 @ fix up last round key + vstmia r12, {q7} @ save last round key + +.align 2 +0: add r12, r3, #248 + vld1.8 {q0}, [r8] @ load counter + adrl r8, .LREVM0SR @ borrow r8 + vldmia r12, {q4} @ load round0 key + sub sp, #0x10 @ place for adjusted round0 key +#endif + + vmov.i32 q8,#1 @ compose 1<<96 + veor q9,q9,q9 + vrev32.8 q0,q0 + vext.8 q8,q9,q8,#4 + vrev32.8 q4,q4 + vadd.u32 q9,q8,q8 @ compose 2<<96 + vstmia sp, {q4} @ save adjusted round0 key + b .Lctr_enc_loop + +.align 4 +.Lctr_enc_loop: + vadd.u32 q10, q8, q9 @ compose 3<<96 + vadd.u32 q1, q0, q8 @ +1 + vadd.u32 q2, q0, q9 @ +2 + vadd.u32 q3, q0, q10 @ +3 + vadd.u32 q4, q1, q10 + vadd.u32 q5, q2, q10 + vadd.u32 q6, q3, q10 + vadd.u32 q7, q4, q10 + vadd.u32 q10, q5, q10 @ next counter + + @ Borrow prologue from _bsaes_encrypt8 to use the opportunity + @ to flip byte order in 32-bit counter + + vldmia sp, {q9} @ load round0 key +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x10 @ pass next round key +#else + add r4, r3, #264 +#endif + vldmia r8, {q8} @ .LREVM0SR + mov r5, r10 @ pass rounds + vstmia r9, {q10} @ save next counter + sub r6, r8, #.LREVM0SR-.LSR @ pass constants + + bl _bsaes_encrypt8_alt + + subs r2, r2, #8 + blo .Lctr_enc_loop_done + + vld1.8 {q8-q9}, [r0]! @ load input + vld1.8 {q10-q11}, [r0]! + veor q0, q8 + veor q1, q9 + vld1.8 {q12-q13}, [r0]! + veor q4, q10 + veor q6, q11 + vld1.8 {q14-q15}, [r0]! + veor q3, q12 + vst1.8 {q0-q1}, [r1]! @ write output + veor q7, q13 + veor q2, q14 + vst1.8 {q4}, [r1]! + veor q5, q15 + vst1.8 {q6}, [r1]! + vmov.i32 q8, #1 @ compose 1<<96 + vst1.8 {q3}, [r1]! + veor q9, q9, q9 + vst1.8 {q7}, [r1]! + vext.8 q8, q9, q8, #4 + vst1.8 {q2}, [r1]! + vadd.u32 q9,q8,q8 @ compose 2<<96 + vst1.8 {q5}, [r1]! + vldmia r9, {q0} @ load counter + + bne .Lctr_enc_loop + b .Lctr_enc_done + +.align 4 +.Lctr_enc_loop_done: + add r2, r2, #8 + vld1.8 {q8}, [r0]! @ load input + veor q0, q8 + vst1.8 {q0}, [r1]! @ write output + cmp r2, #2 + blo .Lctr_enc_done + vld1.8 {q9}, [r0]! + veor q1, q9 + vst1.8 {q1}, [r1]! + beq .Lctr_enc_done + vld1.8 {q10}, [r0]! + veor q4, q10 + vst1.8 {q4}, [r1]! + cmp r2, #4 + blo .Lctr_enc_done + vld1.8 {q11}, [r0]! + veor q6, q11 + vst1.8 {q6}, [r1]! + beq .Lctr_enc_done + vld1.8 {q12}, [r0]! + veor q3, q12 + vst1.8 {q3}, [r1]! + cmp r2, #6 + blo .Lctr_enc_done + vld1.8 {q13}, [r0]! + veor q7, q13 + vst1.8 {q7}, [r1]! + beq .Lctr_enc_done + vld1.8 {q14}, [r0] + veor q2, q14 + vst1.8 {q2}, [r1]! + +.Lctr_enc_done: + vmov.i32 q0, #0 + vmov.i32 q1, #0 +#ifndef BSAES_ASM_EXTENDED_KEY +.Lctr_enc_bzero: @ wipe key schedule [if any] + vstmia sp!, {q0-q1} + cmp sp, r9 + bne .Lctr_enc_bzero +#else + vstmia sp, {q0-q1} +#endif + + mov sp, r9 + add sp, #0x10 @ add sp,r9,#0x10 is no good for thumb + VFP_ABI_POP + ldmia sp!, {r4-r10, pc} @ return + +.align 4 +.Lctr_enc_short: + ldr ip, [sp] @ ctr pointer is passed on stack + stmdb sp!, {r4-r8, lr} + + mov r4, r0 @ copy arguments + mov r5, r1 + mov r6, r2 + mov r7, r3 + ldr r8, [ip, #12] @ load counter LSW + vld1.8 {q1}, [ip] @ load whole counter value +#ifdef __ARMEL__ + rev r8, r8 +#endif + sub sp, sp, #0x10 + vst1.8 {q1}, [sp,:64] @ copy counter value + sub sp, sp, #0x10 + +.Lctr_enc_short_loop: + add r0, sp, #0x10 @ input counter value + mov r1, sp @ output on the stack + mov r2, r7 @ key + + bl AES_encrypt + + vld1.8 {q0}, [r4]! @ load input + vld1.8 {q1}, [sp,:64] @ load encrypted counter + add r8, r8, #1 +#ifdef __ARMEL__ + rev r0, r8 + str r0, [sp, #0x1c] @ next counter value +#else + str r8, [sp, #0x1c] @ next counter value +#endif + veor q0,q0,q1 + vst1.8 {q0}, [r5]! @ store output + subs r6, r6, #1 + bne .Lctr_enc_short_loop + + vmov.i32 q0, #0 + vmov.i32 q1, #0 + vstmia sp!, {q0-q1} + + ldmia sp!, {r4-r8, pc} +.size bsaes_ctr32_encrypt_blocks,.-bsaes_ctr32_encrypt_blocks +.globl bsaes_xts_encrypt +.type bsaes_xts_encrypt,%function +.align 4 +bsaes_xts_encrypt: + mov ip, sp + stmdb sp!, {r4-r10, lr} @ 0x20 + VFP_ABI_PUSH + mov r6, sp @ future r3 + + mov r7, r0 + mov r8, r1 + mov r9, r2 + mov r10, r3 + + sub r0, sp, #0x10 @ 0x10 + bic r0, #0xf @ align at 16 bytes + mov sp, r0 + +#ifdef XTS_CHAIN_TWEAK + ldr r0, [ip] @ pointer to input tweak +#else + @ generate initial tweak + ldr r0, [ip, #4] @ iv[] + mov r1, sp + ldr r2, [ip, #0] @ key2 + bl AES_encrypt + mov r0,sp @ pointer to initial tweak +#endif + + ldr r1, [r10, #240] @ get # of rounds + mov r3, r6 +#ifndef BSAES_ASM_EXTENDED_KEY + @ allocate the key schedule on the stack + sub r12, sp, r1, lsl#7 @ 128 bytes per inner round key + @ add r12, #96 @ size of bit-sliced key schedule + sub r12, #48 @ place for tweak[9] + + @ populate the key schedule + mov r4, r10 @ pass key + mov r5, r1 @ pass # of rounds + mov sp, r12 + add r12, #0x90 @ pass key schedule + bl _bsaes_key_convert + veor q7, q7, q15 @ fix up last round key + vstmia r12, {q7} @ save last round key +#else + ldr r12, [r10, #244] + eors r12, #1 + beq 0f + + str r12, [r10, #244] + mov r4, r10 @ pass key + mov r5, r1 @ pass # of rounds + add r12, r10, #248 @ pass key schedule + bl _bsaes_key_convert + veor q7, q7, q15 @ fix up last round key + vstmia r12, {q7} + +.align 2 +0: sub sp, #0x90 @ place for tweak[9] +#endif + + vld1.8 {q8}, [r0] @ initial tweak + adr r2, .Lxts_magic + + subs r9, #0x80 + blo .Lxts_enc_short + b .Lxts_enc_loop + +.align 4 +.Lxts_enc_loop: + vldmia r2, {q5} @ load XTS magic + vshr.s64 q6, q8, #63 + mov r0, sp + vand q6, q6, q5 + vadd.u64 q9, q8, q8 + vst1.64 {q8}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q9, #63 + veor q9, q9, q6 + vand q7, q7, q5 + vadd.u64 q10, q9, q9 + vst1.64 {q9}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q10, #63 + veor q10, q10, q7 + vand q6, q6, q5 + vld1.8 {q0}, [r7]! + vadd.u64 q11, q10, q10 + vst1.64 {q10}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q11, #63 + veor q11, q11, q6 + vand q7, q7, q5 + vld1.8 {q1}, [r7]! + veor q0, q0, q8 + vadd.u64 q12, q11, q11 + vst1.64 {q11}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q12, #63 + veor q12, q12, q7 + vand q6, q6, q5 + vld1.8 {q2}, [r7]! + veor q1, q1, q9 + vadd.u64 q13, q12, q12 + vst1.64 {q12}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q13, #63 + veor q13, q13, q6 + vand q7, q7, q5 + vld1.8 {q3}, [r7]! + veor q2, q2, q10 + vadd.u64 q14, q13, q13 + vst1.64 {q13}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q14, #63 + veor q14, q14, q7 + vand q6, q6, q5 + vld1.8 {q4}, [r7]! + veor q3, q3, q11 + vadd.u64 q15, q14, q14 + vst1.64 {q14}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q15, #63 + veor q15, q15, q6 + vand q7, q7, q5 + vld1.8 {q5}, [r7]! + veor q4, q4, q12 + vadd.u64 q8, q15, q15 + vst1.64 {q15}, [r0,:128]! + vswp d15,d14 + veor q8, q8, q7 + vst1.64 {q8}, [r0,:128] @ next round tweak + + vld1.8 {q6-q7}, [r7]! + veor q5, q5, q13 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q6, q6, q14 + mov r5, r1 @ pass rounds + veor q7, q7, q15 + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + vld1.64 {q12-q13}, [r0,:128]! + veor q1, q1, q9 + veor q8, q4, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q6, q11 + vld1.64 {q14-q15}, [r0,:128]! + veor q10, q3, q12 + vst1.8 {q8-q9}, [r8]! + veor q11, q7, q13 + veor q12, q2, q14 + vst1.8 {q10-q11}, [r8]! + veor q13, q5, q15 + vst1.8 {q12-q13}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + + subs r9, #0x80 + bpl .Lxts_enc_loop + +.Lxts_enc_short: + adds r9, #0x70 + bmi .Lxts_enc_done + + vldmia r2, {q5} @ load XTS magic + vshr.s64 q7, q8, #63 + mov r0, sp + vand q7, q7, q5 + vadd.u64 q9, q8, q8 + vst1.64 {q8}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q9, #63 + veor q9, q9, q7 + vand q6, q6, q5 + vadd.u64 q10, q9, q9 + vst1.64 {q9}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q10, #63 + veor q10, q10, q6 + vand q7, q7, q5 + vld1.8 {q0}, [r7]! + subs r9, #0x10 + bmi .Lxts_enc_1 + vadd.u64 q11, q10, q10 + vst1.64 {q10}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q11, #63 + veor q11, q11, q7 + vand q6, q6, q5 + vld1.8 {q1}, [r7]! + subs r9, #0x10 + bmi .Lxts_enc_2 + veor q0, q0, q8 + vadd.u64 q12, q11, q11 + vst1.64 {q11}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q12, #63 + veor q12, q12, q6 + vand q7, q7, q5 + vld1.8 {q2}, [r7]! + subs r9, #0x10 + bmi .Lxts_enc_3 + veor q1, q1, q9 + vadd.u64 q13, q12, q12 + vst1.64 {q12}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q13, #63 + veor q13, q13, q7 + vand q6, q6, q5 + vld1.8 {q3}, [r7]! + subs r9, #0x10 + bmi .Lxts_enc_4 + veor q2, q2, q10 + vadd.u64 q14, q13, q13 + vst1.64 {q13}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q14, #63 + veor q14, q14, q6 + vand q7, q7, q5 + vld1.8 {q4}, [r7]! + subs r9, #0x10 + bmi .Lxts_enc_5 + veor q3, q3, q11 + vadd.u64 q15, q14, q14 + vst1.64 {q14}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q15, #63 + veor q15, q15, q7 + vand q6, q6, q5 + vld1.8 {q5}, [r7]! + subs r9, #0x10 + bmi .Lxts_enc_6 + veor q4, q4, q12 + sub r9, #0x10 + vst1.64 {q15}, [r0,:128] @ next round tweak + + vld1.8 {q6}, [r7]! + veor q5, q5, q13 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q6, q6, q14 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + vld1.64 {q12-q13}, [r0,:128]! + veor q1, q1, q9 + veor q8, q4, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q6, q11 + vld1.64 {q14}, [r0,:128]! + veor q10, q3, q12 + vst1.8 {q8-q9}, [r8]! + veor q11, q7, q13 + veor q12, q2, q14 + vst1.8 {q10-q11}, [r8]! + vst1.8 {q12}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_6: + vst1.64 {q14}, [r0,:128] @ next round tweak + + veor q4, q4, q12 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q5, q5, q13 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + vld1.64 {q12-q13}, [r0,:128]! + veor q1, q1, q9 + veor q8, q4, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q6, q11 + veor q10, q3, q12 + vst1.8 {q8-q9}, [r8]! + veor q11, q7, q13 + vst1.8 {q10-q11}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_enc_done + +@ put this in range for both ARM and Thumb mode adr instructions +.align 5 +.Lxts_magic: + .quad 1, 0x87 + +.align 5 +.Lxts_enc_5: + vst1.64 {q13}, [r0,:128] @ next round tweak + + veor q3, q3, q11 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q4, q4, q12 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + vld1.64 {q12}, [r0,:128]! + veor q1, q1, q9 + veor q8, q4, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q6, q11 + veor q10, q3, q12 + vst1.8 {q8-q9}, [r8]! + vst1.8 {q10}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_4: + vst1.64 {q12}, [r0,:128] @ next round tweak + + veor q2, q2, q10 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q3, q3, q11 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + veor q1, q1, q9 + veor q8, q4, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q6, q11 + vst1.8 {q8-q9}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_3: + vst1.64 {q11}, [r0,:128] @ next round tweak + + veor q1, q1, q9 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q2, q2, q10 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10}, [r0,:128]! + veor q0, q0, q8 + veor q1, q1, q9 + veor q8, q4, q10 + vst1.8 {q0-q1}, [r8]! + vst1.8 {q8}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_2: + vst1.64 {q10}, [r0,:128] @ next round tweak + + veor q0, q0, q8 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q1, q1, q9 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + veor q0, q0, q8 + veor q1, q1, q9 + vst1.8 {q0-q1}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_1: + mov r0, sp + veor q0, q8 + mov r1, sp + vst1.8 {q0}, [sp,:128] + mov r2, r10 + mov r4, r3 @ preserve fp + + bl AES_encrypt + + vld1.8 {q0}, [sp,:128] + veor q0, q0, q8 + vst1.8 {q0}, [r8]! + mov r3, r4 + + vmov q8, q9 @ next round tweak + +.Lxts_enc_done: +#ifndef XTS_CHAIN_TWEAK + adds r9, #0x10 + beq .Lxts_enc_ret + sub r6, r8, #0x10 + +.Lxts_enc_steal: + ldrb r0, [r7], #1 + ldrb r1, [r8, #-0x10] + strb r0, [r8, #-0x10] + strb r1, [r8], #1 + + subs r9, #1 + bhi .Lxts_enc_steal + + vld1.8 {q0}, [r6] + mov r0, sp + veor q0, q0, q8 + mov r1, sp + vst1.8 {q0}, [sp,:128] + mov r2, r10 + mov r4, r3 @ preserve fp + + bl AES_encrypt + + vld1.8 {q0}, [sp,:128] + veor q0, q0, q8 + vst1.8 {q0}, [r6] + mov r3, r4 +#endif + +.Lxts_enc_ret: + bic r0, r3, #0xf + vmov.i32 q0, #0 + vmov.i32 q1, #0 +#ifdef XTS_CHAIN_TWEAK + ldr r1, [r3, #0x20+VFP_ABI_FRAME] @ chain tweak +#endif +.Lxts_enc_bzero: @ wipe key schedule [if any] + vstmia sp!, {q0-q1} + cmp sp, r0 + bne .Lxts_enc_bzero + + mov sp, r3 +#ifdef XTS_CHAIN_TWEAK + vst1.8 {q8}, [r1] +#endif + VFP_ABI_POP + ldmia sp!, {r4-r10, pc} @ return + +.size bsaes_xts_encrypt,.-bsaes_xts_encrypt + +.globl bsaes_xts_decrypt +.type bsaes_xts_decrypt,%function +.align 4 +bsaes_xts_decrypt: + mov ip, sp + stmdb sp!, {r4-r10, lr} @ 0x20 + VFP_ABI_PUSH + mov r6, sp @ future r3 + + mov r7, r0 + mov r8, r1 + mov r9, r2 + mov r10, r3 + + sub r0, sp, #0x10 @ 0x10 + bic r0, #0xf @ align at 16 bytes + mov sp, r0 + +#ifdef XTS_CHAIN_TWEAK + ldr r0, [ip] @ pointer to input tweak +#else + @ generate initial tweak + ldr r0, [ip, #4] @ iv[] + mov r1, sp + ldr r2, [ip, #0] @ key2 + bl AES_encrypt + mov r0, sp @ pointer to initial tweak +#endif + + ldr r1, [r10, #240] @ get # of rounds + mov r3, r6 +#ifndef BSAES_ASM_EXTENDED_KEY + @ allocate the key schedule on the stack + sub r12, sp, r1, lsl#7 @ 128 bytes per inner round key + @ add r12, #96 @ size of bit-sliced key schedule + sub r12, #48 @ place for tweak[9] + + @ populate the key schedule + mov r4, r10 @ pass key + mov r5, r1 @ pass # of rounds + mov sp, r12 + add r12, #0x90 @ pass key schedule + bl _bsaes_key_convert + add r4, sp, #0x90 + vldmia r4, {q6} + vstmia r12, {q15} @ save last round key + veor q7, q7, q6 @ fix up round 0 key + vstmia r4, {q7} +#else + ldr r12, [r10, #244] + eors r12, #1 + beq 0f + + str r12, [r10, #244] + mov r4, r10 @ pass key + mov r5, r1 @ pass # of rounds + add r12, r10, #248 @ pass key schedule + bl _bsaes_key_convert + add r4, r10, #248 + vldmia r4, {q6} + vstmia r12, {q15} @ save last round key + veor q7, q7, q6 @ fix up round 0 key + vstmia r4, {q7} + +.align 2 +0: sub sp, #0x90 @ place for tweak[9] +#endif + vld1.8 {q8}, [r0] @ initial tweak + adr r2, .Lxts_magic + + tst r9, #0xf @ if not multiple of 16 + it ne @ Thumb2 thing, sanity check in ARM + subne r9, #0x10 @ subtract another 16 bytes + subs r9, #0x80 + + blo .Lxts_dec_short + b .Lxts_dec_loop + +.align 4 +.Lxts_dec_loop: + vldmia r2, {q5} @ load XTS magic + vshr.s64 q6, q8, #63 + mov r0, sp + vand q6, q6, q5 + vadd.u64 q9, q8, q8 + vst1.64 {q8}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q9, #63 + veor q9, q9, q6 + vand q7, q7, q5 + vadd.u64 q10, q9, q9 + vst1.64 {q9}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q10, #63 + veor q10, q10, q7 + vand q6, q6, q5 + vld1.8 {q0}, [r7]! + vadd.u64 q11, q10, q10 + vst1.64 {q10}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q11, #63 + veor q11, q11, q6 + vand q7, q7, q5 + vld1.8 {q1}, [r7]! + veor q0, q0, q8 + vadd.u64 q12, q11, q11 + vst1.64 {q11}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q12, #63 + veor q12, q12, q7 + vand q6, q6, q5 + vld1.8 {q2}, [r7]! + veor q1, q1, q9 + vadd.u64 q13, q12, q12 + vst1.64 {q12}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q13, #63 + veor q13, q13, q6 + vand q7, q7, q5 + vld1.8 {q3}, [r7]! + veor q2, q2, q10 + vadd.u64 q14, q13, q13 + vst1.64 {q13}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q14, #63 + veor q14, q14, q7 + vand q6, q6, q5 + vld1.8 {q4}, [r7]! + veor q3, q3, q11 + vadd.u64 q15, q14, q14 + vst1.64 {q14}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q15, #63 + veor q15, q15, q6 + vand q7, q7, q5 + vld1.8 {q5}, [r7]! + veor q4, q4, q12 + vadd.u64 q8, q15, q15 + vst1.64 {q15}, [r0,:128]! + vswp d15,d14 + veor q8, q8, q7 + vst1.64 {q8}, [r0,:128] @ next round tweak + + vld1.8 {q6-q7}, [r7]! + veor q5, q5, q13 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q6, q6, q14 + mov r5, r1 @ pass rounds + veor q7, q7, q15 + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + vld1.64 {q12-q13}, [r0,:128]! + veor q1, q1, q9 + veor q8, q6, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q4, q11 + vld1.64 {q14-q15}, [r0,:128]! + veor q10, q2, q12 + vst1.8 {q8-q9}, [r8]! + veor q11, q7, q13 + veor q12, q3, q14 + vst1.8 {q10-q11}, [r8]! + veor q13, q5, q15 + vst1.8 {q12-q13}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + + subs r9, #0x80 + bpl .Lxts_dec_loop + +.Lxts_dec_short: + adds r9, #0x70 + bmi .Lxts_dec_done + + vldmia r2, {q5} @ load XTS magic + vshr.s64 q7, q8, #63 + mov r0, sp + vand q7, q7, q5 + vadd.u64 q9, q8, q8 + vst1.64 {q8}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q9, #63 + veor q9, q9, q7 + vand q6, q6, q5 + vadd.u64 q10, q9, q9 + vst1.64 {q9}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q10, #63 + veor q10, q10, q6 + vand q7, q7, q5 + vld1.8 {q0}, [r7]! + subs r9, #0x10 + bmi .Lxts_dec_1 + vadd.u64 q11, q10, q10 + vst1.64 {q10}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q11, #63 + veor q11, q11, q7 + vand q6, q6, q5 + vld1.8 {q1}, [r7]! + subs r9, #0x10 + bmi .Lxts_dec_2 + veor q0, q0, q8 + vadd.u64 q12, q11, q11 + vst1.64 {q11}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q12, #63 + veor q12, q12, q6 + vand q7, q7, q5 + vld1.8 {q2}, [r7]! + subs r9, #0x10 + bmi .Lxts_dec_3 + veor q1, q1, q9 + vadd.u64 q13, q12, q12 + vst1.64 {q12}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q13, #63 + veor q13, q13, q7 + vand q6, q6, q5 + vld1.8 {q3}, [r7]! + subs r9, #0x10 + bmi .Lxts_dec_4 + veor q2, q2, q10 + vadd.u64 q14, q13, q13 + vst1.64 {q13}, [r0,:128]! + vswp d13,d12 + vshr.s64 q7, q14, #63 + veor q14, q14, q6 + vand q7, q7, q5 + vld1.8 {q4}, [r7]! + subs r9, #0x10 + bmi .Lxts_dec_5 + veor q3, q3, q11 + vadd.u64 q15, q14, q14 + vst1.64 {q14}, [r0,:128]! + vswp d15,d14 + vshr.s64 q6, q15, #63 + veor q15, q15, q7 + vand q6, q6, q5 + vld1.8 {q5}, [r7]! + subs r9, #0x10 + bmi .Lxts_dec_6 + veor q4, q4, q12 + sub r9, #0x10 + vst1.64 {q15}, [r0,:128] @ next round tweak + + vld1.8 {q6}, [r7]! + veor q5, q5, q13 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q6, q6, q14 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + vld1.64 {q12-q13}, [r0,:128]! + veor q1, q1, q9 + veor q8, q6, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q4, q11 + vld1.64 {q14}, [r0,:128]! + veor q10, q2, q12 + vst1.8 {q8-q9}, [r8]! + veor q11, q7, q13 + veor q12, q3, q14 + vst1.8 {q10-q11}, [r8]! + vst1.8 {q12}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_6: + vst1.64 {q14}, [r0,:128] @ next round tweak + + veor q4, q4, q12 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q5, q5, q13 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + vld1.64 {q12-q13}, [r0,:128]! + veor q1, q1, q9 + veor q8, q6, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q4, q11 + veor q10, q2, q12 + vst1.8 {q8-q9}, [r8]! + veor q11, q7, q13 + vst1.8 {q10-q11}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_5: + vst1.64 {q13}, [r0,:128] @ next round tweak + + veor q3, q3, q11 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q4, q4, q12 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + vld1.64 {q12}, [r0,:128]! + veor q1, q1, q9 + veor q8, q6, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q4, q11 + veor q10, q2, q12 + vst1.8 {q8-q9}, [r8]! + vst1.8 {q10}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_4: + vst1.64 {q12}, [r0,:128] @ next round tweak + + veor q2, q2, q10 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q3, q3, q11 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10-q11}, [r0,:128]! + veor q0, q0, q8 + veor q1, q1, q9 + veor q8, q6, q10 + vst1.8 {q0-q1}, [r8]! + veor q9, q4, q11 + vst1.8 {q8-q9}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_3: + vst1.64 {q11}, [r0,:128] @ next round tweak + + veor q1, q1, q9 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q2, q2, q10 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + vld1.64 {q10}, [r0,:128]! + veor q0, q0, q8 + veor q1, q1, q9 + veor q8, q6, q10 + vst1.8 {q0-q1}, [r8]! + vst1.8 {q8}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_2: + vst1.64 {q10}, [r0,:128] @ next round tweak + + veor q0, q0, q8 +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, r10, #248 @ pass key schedule +#endif + veor q1, q1, q9 + mov r5, r1 @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {q8-q9}, [r0,:128]! + veor q0, q0, q8 + veor q1, q1, q9 + vst1.8 {q0-q1}, [r8]! + + vld1.64 {q8}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_1: + mov r0, sp + veor q0, q8 + mov r1, sp + vst1.8 {q0}, [sp,:128] + mov r2, r10 + mov r4, r3 @ preserve fp + mov r5, r2 @ preserve magic + + bl AES_decrypt + + vld1.8 {q0}, [sp,:128] + veor q0, q0, q8 + vst1.8 {q0}, [r8]! + mov r3, r4 + mov r2, r5 + + vmov q8, q9 @ next round tweak + +.Lxts_dec_done: +#ifndef XTS_CHAIN_TWEAK + adds r9, #0x10 + beq .Lxts_dec_ret + + @ calculate one round of extra tweak for the stolen ciphertext + vldmia r2, {q5} + vshr.s64 q6, q8, #63 + vand q6, q6, q5 + vadd.u64 q9, q8, q8 + vswp d13,d12 + veor q9, q9, q6 + + @ perform the final decryption with the last tweak value + vld1.8 {q0}, [r7]! + mov r0, sp + veor q0, q0, q9 + mov r1, sp + vst1.8 {q0}, [sp,:128] + mov r2, r10 + mov r4, r3 @ preserve fp + + bl AES_decrypt + + vld1.8 {q0}, [sp,:128] + veor q0, q0, q9 + vst1.8 {q0}, [r8] + + mov r6, r8 +.Lxts_dec_steal: + ldrb r1, [r8] + ldrb r0, [r7], #1 + strb r1, [r8, #0x10] + strb r0, [r8], #1 + + subs r9, #1 + bhi .Lxts_dec_steal + + vld1.8 {q0}, [r6] + mov r0, sp + veor q0, q8 + mov r1, sp + vst1.8 {q0}, [sp,:128] + mov r2, r10 + + bl AES_decrypt + + vld1.8 {q0}, [sp,:128] + veor q0, q0, q8 + vst1.8 {q0}, [r6] + mov r3, r4 +#endif + +.Lxts_dec_ret: + bic r0, r3, #0xf + vmov.i32 q0, #0 + vmov.i32 q1, #0 +#ifdef XTS_CHAIN_TWEAK + ldr r1, [r3, #0x20+VFP_ABI_FRAME] @ chain tweak +#endif +.Lxts_dec_bzero: @ wipe key schedule [if any] + vstmia sp!, {q0-q1} + cmp sp, r0 + bne .Lxts_dec_bzero + + mov sp, r3 +#ifdef XTS_CHAIN_TWEAK + vst1.8 {q8}, [r1] +#endif + VFP_ABI_POP + ldmia sp!, {r4-r10, pc} @ return + +.size bsaes_xts_decrypt,.-bsaes_xts_decrypt +#endif diff --git a/arch/arm/crypto/aesbs-glue.c b/arch/arm/crypto/aesbs-glue.c new file mode 100644 index 00000000000..4522366da75 --- /dev/null +++ b/arch/arm/crypto/aesbs-glue.c @@ -0,0 +1,434 @@ +/* + * linux/arch/arm/crypto/aesbs-glue.c - glue code for NEON bit sliced AES + * + * Copyright (C) 2013 Linaro Ltd + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include + +#include "aes_glue.h" + +#define BIT_SLICED_KEY_MAXSIZE (128 * (AES_MAXNR - 1) + 2 * AES_BLOCK_SIZE) + +struct BS_KEY { + struct AES_KEY rk; + int converted; + u8 __aligned(8) bs[BIT_SLICED_KEY_MAXSIZE]; +} __aligned(8); + +asmlinkage void bsaes_enc_key_convert(u8 out[], struct AES_KEY const *in); +asmlinkage void bsaes_dec_key_convert(u8 out[], struct AES_KEY const *in); + +asmlinkage void bsaes_cbc_encrypt(u8 const in[], u8 out[], u32 bytes, + struct BS_KEY *key, u8 iv[]); + +asmlinkage void bsaes_ctr32_encrypt_blocks(u8 const in[], u8 out[], u32 blocks, + struct BS_KEY *key, u8 const iv[]); + +asmlinkage void bsaes_xts_encrypt(u8 const in[], u8 out[], u32 bytes, + struct BS_KEY *key, u8 tweak[]); + +asmlinkage void bsaes_xts_decrypt(u8 const in[], u8 out[], u32 bytes, + struct BS_KEY *key, u8 tweak[]); + +struct aesbs_cbc_ctx { + struct AES_KEY enc; + struct BS_KEY dec; +}; + +struct aesbs_ctr_ctx { + struct BS_KEY enc; +}; + +struct aesbs_xts_ctx { + struct BS_KEY enc; + struct BS_KEY dec; + struct AES_KEY twkey; +}; + +static int aesbs_cbc_set_key(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct aesbs_cbc_ctx *ctx = crypto_tfm_ctx(tfm); + int bits = key_len * 8; + + if (private_AES_set_encrypt_key(in_key, bits, &ctx->enc)) { + tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + ctx->dec.rk = ctx->enc; + private_AES_set_decrypt_key(in_key, bits, &ctx->dec.rk); + ctx->dec.converted = 0; + return 0; +} + +static int aesbs_ctr_set_key(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct aesbs_ctr_ctx *ctx = crypto_tfm_ctx(tfm); + int bits = key_len * 8; + + if (private_AES_set_encrypt_key(in_key, bits, &ctx->enc.rk)) { + tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + ctx->enc.converted = 0; + return 0; +} + +static int aesbs_xts_set_key(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct aesbs_xts_ctx *ctx = crypto_tfm_ctx(tfm); + int bits = key_len * 4; + + if (private_AES_set_encrypt_key(in_key, bits, &ctx->enc.rk)) { + tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + ctx->dec.rk = ctx->enc.rk; + private_AES_set_decrypt_key(in_key, bits, &ctx->dec.rk); + private_AES_set_encrypt_key(in_key + key_len / 2, bits, &ctx->twkey); + ctx->enc.converted = ctx->dec.converted = 0; + return 0; +} + +static int aesbs_cbc_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct aesbs_cbc_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + + while (walk.nbytes) { + u32 blocks = walk.nbytes / AES_BLOCK_SIZE; + u8 *src = walk.src.virt.addr; + + if (walk.dst.virt.addr == walk.src.virt.addr) { + u8 *iv = walk.iv; + + do { + crypto_xor(src, iv, AES_BLOCK_SIZE); + AES_encrypt(src, src, &ctx->enc); + iv = src; + src += AES_BLOCK_SIZE; + } while (--blocks); + memcpy(walk.iv, iv, AES_BLOCK_SIZE); + } else { + u8 *dst = walk.dst.virt.addr; + + do { + crypto_xor(walk.iv, src, AES_BLOCK_SIZE); + AES_encrypt(walk.iv, dst, &ctx->enc); + memcpy(walk.iv, dst, AES_BLOCK_SIZE); + src += AES_BLOCK_SIZE; + dst += AES_BLOCK_SIZE; + } while (--blocks); + } + err = blkcipher_walk_done(desc, &walk, 0); + } + return err; +} + +static int aesbs_cbc_decrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct aesbs_cbc_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE); + + while ((walk.nbytes / AES_BLOCK_SIZE) >= 8) { + kernel_neon_begin(); + bsaes_cbc_encrypt(walk.src.virt.addr, walk.dst.virt.addr, + walk.nbytes, &ctx->dec, walk.iv); + kernel_neon_end(); + err = blkcipher_walk_done(desc, &walk, 0); + } + while (walk.nbytes) { + u32 blocks = walk.nbytes / AES_BLOCK_SIZE; + u8 *dst = walk.dst.virt.addr; + u8 *src = walk.src.virt.addr; + u8 bk[2][AES_BLOCK_SIZE]; + u8 *iv = walk.iv; + + do { + if (walk.dst.virt.addr == walk.src.virt.addr) + memcpy(bk[blocks & 1], src, AES_BLOCK_SIZE); + + AES_decrypt(src, dst, &ctx->dec.rk); + crypto_xor(dst, iv, AES_BLOCK_SIZE); + + if (walk.dst.virt.addr == walk.src.virt.addr) + iv = bk[blocks & 1]; + else + iv = src; + + dst += AES_BLOCK_SIZE; + src += AES_BLOCK_SIZE; + } while (--blocks); + err = blkcipher_walk_done(desc, &walk, 0); + } + return err; +} + +static void inc_be128_ctr(__be32 ctr[], u32 addend) +{ + int i; + + for (i = 3; i >= 0; i--, addend = 1) { + u32 n = be32_to_cpu(ctr[i]) + addend; + + ctr[i] = cpu_to_be32(n); + if (n >= addend) + break; + } +} + +static int aesbs_ctr_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct aesbs_ctr_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + u32 blocks; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE); + + while ((blocks = walk.nbytes / AES_BLOCK_SIZE)) { + u32 tail = walk.nbytes % AES_BLOCK_SIZE; + __be32 *ctr = (__be32 *)walk.iv; + u32 headroom = UINT_MAX - be32_to_cpu(ctr[3]); + + /* avoid 32 bit counter overflow in the NEON code */ + if (unlikely(headroom < blocks)) { + blocks = headroom + 1; + tail = walk.nbytes - blocks * AES_BLOCK_SIZE; + } + kernel_neon_begin(); + bsaes_ctr32_encrypt_blocks(walk.src.virt.addr, + walk.dst.virt.addr, blocks, + &ctx->enc, walk.iv); + kernel_neon_end(); + inc_be128_ctr(ctr, blocks); + + nbytes -= blocks * AES_BLOCK_SIZE; + if (nbytes && nbytes == tail && nbytes <= AES_BLOCK_SIZE) + break; + + err = blkcipher_walk_done(desc, &walk, tail); + } + if (walk.nbytes) { + u8 *tdst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE; + u8 *tsrc = walk.src.virt.addr + blocks * AES_BLOCK_SIZE; + u8 ks[AES_BLOCK_SIZE]; + + AES_encrypt(walk.iv, ks, &ctx->enc.rk); + if (tdst != tsrc) + memcpy(tdst, tsrc, nbytes); + crypto_xor(tdst, ks, nbytes); + err = blkcipher_walk_done(desc, &walk, 0); + } + return err; +} + +static int aesbs_xts_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct aesbs_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE); + + /* generate the initial tweak */ + AES_encrypt(walk.iv, walk.iv, &ctx->twkey); + + while (walk.nbytes) { + kernel_neon_begin(); + bsaes_xts_encrypt(walk.src.virt.addr, walk.dst.virt.addr, + walk.nbytes, &ctx->enc, walk.iv); + kernel_neon_end(); + err = blkcipher_walk_done(desc, &walk, 0); + } + return err; +} + +static int aesbs_xts_decrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct aesbs_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE); + + /* generate the initial tweak */ + AES_encrypt(walk.iv, walk.iv, &ctx->twkey); + + while (walk.nbytes) { + kernel_neon_begin(); + bsaes_xts_decrypt(walk.src.virt.addr, walk.dst.virt.addr, + walk.nbytes, &ctx->dec, walk.iv); + kernel_neon_end(); + err = blkcipher_walk_done(desc, &walk, 0); + } + return err; +} + +static struct crypto_alg aesbs_algs[] = { { + .cra_name = "__cbc-aes-neonbs", + .cra_driver_name = "__driver-cbc-aes-neonbs", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aesbs_cbc_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_blkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = aesbs_cbc_set_key, + .encrypt = aesbs_cbc_encrypt, + .decrypt = aesbs_cbc_decrypt, + }, +}, { + .cra_name = "__ctr-aes-neonbs", + .cra_driver_name = "__driver-ctr-aes-neonbs", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct aesbs_ctr_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_blkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = aesbs_ctr_set_key, + .encrypt = aesbs_ctr_encrypt, + .decrypt = aesbs_ctr_encrypt, + }, +}, { + .cra_name = "__xts-aes-neonbs", + .cra_driver_name = "__driver-xts-aes-neonbs", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aesbs_xts_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_blkcipher = { + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = aesbs_xts_set_key, + .encrypt = aesbs_xts_encrypt, + .decrypt = aesbs_xts_decrypt, + }, +}, { + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-aes-neonbs", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = __ablk_encrypt, + .decrypt = ablk_decrypt, + } +}, { + .cra_name = "ctr(aes)", + .cra_driver_name = "ctr-aes-neonbs", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + } +}, { + .cra_name = "xts(aes)", + .cra_driver_name = "xts-aes-neonbs", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_ablkcipher = { + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + } +} }; + +static int __init aesbs_mod_init(void) +{ + if (!cpu_has_neon()) + return -ENODEV; + + return crypto_register_algs(aesbs_algs, ARRAY_SIZE(aesbs_algs)); +} + +static void __exit aesbs_mod_exit(void) +{ + crypto_unregister_algs(aesbs_algs, ARRAY_SIZE(aesbs_algs)); +} + +module_init(aesbs_mod_init); +module_exit(aesbs_mod_exit); + +MODULE_DESCRIPTION("Bit sliced AES in CBC/CTR/XTS modes using NEON"); +MODULE_AUTHOR("Ard Biesheuvel "); +MODULE_LICENSE("GPL"); diff --git a/arch/arm/crypto/bsaes-armv7.pl b/arch/arm/crypto/bsaes-armv7.pl new file mode 100644 index 00000000000..f3d96d93257 --- /dev/null +++ b/arch/arm/crypto/bsaes-armv7.pl @@ -0,0 +1,2467 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# +# Specific modes and adaptation for Linux kernel by Ard Biesheuvel +# . Permission to use under GPL terms is +# granted. +# ==================================================================== + +# Bit-sliced AES for ARM NEON +# +# February 2012. +# +# This implementation is direct adaptation of bsaes-x86_64 module for +# ARM NEON. Except that this module is endian-neutral [in sense that +# it can be compiled for either endianness] by courtesy of vld1.8's +# neutrality. Initial version doesn't implement interface to OpenSSL, +# only low-level primitives and unsupported entry points, just enough +# to collect performance results, which for Cortex-A8 core are: +# +# encrypt 19.5 cycles per byte processed with 128-bit key +# decrypt 22.1 cycles per byte processed with 128-bit key +# key conv. 440 cycles per 128-bit key/0.18 of 8x block +# +# Snapdragon S4 encrypts byte in 17.6 cycles and decrypts in 19.7, +# which is [much] worse than anticipated (for further details see +# http://www.openssl.org/~appro/Snapdragon-S4.html). +# +# Cortex-A15 manages in 14.2/16.1 cycles [when integer-only code +# manages in 20.0 cycles]. +# +# When comparing to x86_64 results keep in mind that NEON unit is +# [mostly] single-issue and thus can't [fully] benefit from +# instruction-level parallelism. And when comparing to aes-armv4 +# results keep in mind key schedule conversion overhead (see +# bsaes-x86_64.pl for further details)... +# +# + +# April-August 2013 +# +# Add CBC, CTR and XTS subroutines, adapt for kernel use. +# +# + +while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {} +open STDOUT,">$output"; + +my ($inp,$out,$len,$key)=("r0","r1","r2","r3"); +my @XMM=map("q$_",(0..15)); + +{ +my ($key,$rounds,$const)=("r4","r5","r6"); + +sub Dlo() { shift=~m|q([1]?[0-9])|?"d".($1*2):""; } +sub Dhi() { shift=~m|q([1]?[0-9])|?"d".($1*2+1):""; } + +sub Sbox { +# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb +# output in lsb > [b0, b1, b4, b6, b3, b7, b2, b5] < msb +my @b=@_[0..7]; +my @t=@_[8..11]; +my @s=@_[12..15]; + &InBasisChange (@b); + &Inv_GF256 (@b[6,5,0,3,7,1,4,2],@t,@s); + &OutBasisChange (@b[7,1,4,2,6,5,0,3]); +} + +sub InBasisChange { +# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb +# output in lsb > [b6, b5, b0, b3, b7, b1, b4, b2] < msb +my @b=@_[0..7]; +$code.=<<___; + veor @b[2], @b[2], @b[1] + veor @b[5], @b[5], @b[6] + veor @b[3], @b[3], @b[0] + veor @b[6], @b[6], @b[2] + veor @b[5], @b[5], @b[0] + + veor @b[6], @b[6], @b[3] + veor @b[3], @b[3], @b[7] + veor @b[7], @b[7], @b[5] + veor @b[3], @b[3], @b[4] + veor @b[4], @b[4], @b[5] + + veor @b[2], @b[2], @b[7] + veor @b[3], @b[3], @b[1] + veor @b[1], @b[1], @b[5] +___ +} + +sub OutBasisChange { +# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb +# output in lsb > [b6, b1, b2, b4, b7, b0, b3, b5] < msb +my @b=@_[0..7]; +$code.=<<___; + veor @b[0], @b[0], @b[6] + veor @b[1], @b[1], @b[4] + veor @b[4], @b[4], @b[6] + veor @b[2], @b[2], @b[0] + veor @b[6], @b[6], @b[1] + + veor @b[1], @b[1], @b[5] + veor @b[5], @b[5], @b[3] + veor @b[3], @b[3], @b[7] + veor @b[7], @b[7], @b[5] + veor @b[2], @b[2], @b[5] + + veor @b[4], @b[4], @b[7] +___ +} + +sub InvSbox { +# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb +# output in lsb > [b0, b1, b6, b4, b2, b7, b3, b5] < msb +my @b=@_[0..7]; +my @t=@_[8..11]; +my @s=@_[12..15]; + &InvInBasisChange (@b); + &Inv_GF256 (@b[5,1,2,6,3,7,0,4],@t,@s); + &InvOutBasisChange (@b[3,7,0,4,5,1,2,6]); +} + +sub InvInBasisChange { # OutBasisChange in reverse (with twist) +my @b=@_[5,1,2,6,3,7,0,4]; +$code.=<<___ + veor @b[1], @b[1], @b[7] + veor @b[4], @b[4], @b[7] + + veor @b[7], @b[7], @b[5] + veor @b[1], @b[1], @b[3] + veor @b[2], @b[2], @b[5] + veor @b[3], @b[3], @b[7] + + veor @b[6], @b[6], @b[1] + veor @b[2], @b[2], @b[0] + veor @b[5], @b[5], @b[3] + veor @b[4], @b[4], @b[6] + veor @b[0], @b[0], @b[6] + veor @b[1], @b[1], @b[4] +___ +} + +sub InvOutBasisChange { # InBasisChange in reverse +my @b=@_[2,5,7,3,6,1,0,4]; +$code.=<<___; + veor @b[1], @b[1], @b[5] + veor @b[2], @b[2], @b[7] + + veor @b[3], @b[3], @b[1] + veor @b[4], @b[4], @b[5] + veor @b[7], @b[7], @b[5] + veor @b[3], @b[3], @b[4] + veor @b[5], @b[5], @b[0] + veor @b[3], @b[3], @b[7] + veor @b[6], @b[6], @b[2] + veor @b[2], @b[2], @b[1] + veor @b[6], @b[6], @b[3] + + veor @b[3], @b[3], @b[0] + veor @b[5], @b[5], @b[6] +___ +} + +sub Mul_GF4 { +#;************************************************************* +#;* Mul_GF4: Input x0-x1,y0-y1 Output x0-x1 Temp t0 (8) * +#;************************************************************* +my ($x0,$x1,$y0,$y1,$t0,$t1)=@_; +$code.=<<___; + veor $t0, $y0, $y1 + vand $t0, $t0, $x0 + veor $x0, $x0, $x1 + vand $t1, $x1, $y0 + vand $x0, $x0, $y1 + veor $x1, $t1, $t0 + veor $x0, $x0, $t1 +___ +} + +sub Mul_GF4_N { # not used, see next subroutine +# multiply and scale by N +my ($x0,$x1,$y0,$y1,$t0)=@_; +$code.=<<___; + veor $t0, $y0, $y1 + vand $t0, $t0, $x0 + veor $x0, $x0, $x1 + vand $x1, $x1, $y0 + vand $x0, $x0, $y1 + veor $x1, $x1, $x0 + veor $x0, $x0, $t0 +___ +} + +sub Mul_GF4_N_GF4 { +# interleaved Mul_GF4_N and Mul_GF4 +my ($x0,$x1,$y0,$y1,$t0, + $x2,$x3,$y2,$y3,$t1)=@_; +$code.=<<___; + veor $t0, $y0, $y1 + veor $t1, $y2, $y3 + vand $t0, $t0, $x0 + vand $t1, $t1, $x2 + veor $x0, $x0, $x1 + veor $x2, $x2, $x3 + vand $x1, $x1, $y0 + vand $x3, $x3, $y2 + vand $x0, $x0, $y1 + vand $x2, $x2, $y3 + veor $x1, $x1, $x0 + veor $x2, $x2, $x3 + veor $x0, $x0, $t0 + veor $x3, $x3, $t1 +___ +} +sub Mul_GF16_2 { +my @x=@_[0..7]; +my @y=@_[8..11]; +my @t=@_[12..15]; +$code.=<<___; + veor @t[0], @x[0], @x[2] + veor @t[1], @x[1], @x[3] +___ + &Mul_GF4 (@x[0], @x[1], @y[0], @y[1], @t[2..3]); +$code.=<<___; + veor @y[0], @y[0], @y[2] + veor @y[1], @y[1], @y[3] +___ + Mul_GF4_N_GF4 (@t[0], @t[1], @y[0], @y[1], @t[3], + @x[2], @x[3], @y[2], @y[3], @t[2]); +$code.=<<___; + veor @x[0], @x[0], @t[0] + veor @x[2], @x[2], @t[0] + veor @x[1], @x[1], @t[1] + veor @x[3], @x[3], @t[1] + + veor @t[0], @x[4], @x[6] + veor @t[1], @x[5], @x[7] +___ + &Mul_GF4_N_GF4 (@t[0], @t[1], @y[0], @y[1], @t[3], + @x[6], @x[7], @y[2], @y[3], @t[2]); +$code.=<<___; + veor @y[0], @y[0], @y[2] + veor @y[1], @y[1], @y[3] +___ + &Mul_GF4 (@x[4], @x[5], @y[0], @y[1], @t[2..3]); +$code.=<<___; + veor @x[4], @x[4], @t[0] + veor @x[6], @x[6], @t[0] + veor @x[5], @x[5], @t[1] + veor @x[7], @x[7], @t[1] +___ +} +sub Inv_GF256 { +#;******************************************************************** +#;* Inv_GF256: Input x0-x7 Output x0-x7 Temp t0-t3,s0-s3 (144) * +#;******************************************************************** +my @x=@_[0..7]; +my @t=@_[8..11]; +my @s=@_[12..15]; +# direct optimizations from hardware +$code.=<<___; + veor @t[3], @x[4], @x[6] + veor @t[2], @x[5], @x[7] + veor @t[1], @x[1], @x[3] + veor @s[1], @x[7], @x[6] + vmov @t[0], @t[2] + veor @s[0], @x[0], @x[2] + + vorr @t[2], @t[2], @t[1] + veor @s[3], @t[3], @t[0] + vand @s[2], @t[3], @s[0] + vorr @t[3], @t[3], @s[0] + veor @s[0], @s[0], @t[1] + vand @t[0], @t[0], @t[1] + veor @t[1], @x[3], @x[2] + vand @s[3], @s[3], @s[0] + vand @s[1], @s[1], @t[1] + veor @t[1], @x[4], @x[5] + veor @s[0], @x[1], @x[0] + veor @t[3], @t[3], @s[1] + veor @t[2], @t[2], @s[1] + vand @s[1], @t[1], @s[0] + vorr @t[1], @t[1], @s[0] + veor @t[3], @t[3], @s[3] + veor @t[0], @t[0], @s[1] + veor @t[2], @t[2], @s[2] + veor @t[1], @t[1], @s[3] + veor @t[0], @t[0], @s[2] + vand @s[0], @x[7], @x[3] + veor @t[1], @t[1], @s[2] + vand @s[1], @x[6], @x[2] + vand @s[2], @x[5], @x[1] + vorr @s[3], @x[4], @x[0] + veor @t[3], @t[3], @s[0] + veor @t[1], @t[1], @s[2] + veor @t[0], @t[0], @s[3] + veor @t[2], @t[2], @s[1] + + @ Inv_GF16 \t0, \t1, \t2, \t3, \s0, \s1, \s2, \s3 + + @ new smaller inversion + + vand @s[2], @t[3], @t[1] + vmov @s[0], @t[0] + + veor @s[1], @t[2], @s[2] + veor @s[3], @t[0], @s[2] + veor @s[2], @t[0], @s[2] @ @s[2]=@s[3] + + vbsl @s[1], @t[1], @t[0] + vbsl @s[3], @t[3], @t[2] + veor @t[3], @t[3], @t[2] + + vbsl @s[0], @s[1], @s[2] + vbsl @t[0], @s[2], @s[1] + + vand @s[2], @s[0], @s[3] + veor @t[1], @t[1], @t[0] + + veor @s[2], @s[2], @t[3] +___ +# output in s3, s2, s1, t1 + +# Mul_GF16_2 \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \t2, \t3, \t0, \t1, \s0, \s1, \s2, \s3 + +# Mul_GF16_2 \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \s3, \s2, \s1, \t1, \s0, \t0, \t2, \t3 + &Mul_GF16_2(@x,@s[3,2,1],@t[1],@s[0],@t[0,2,3]); + +### output msb > [x3,x2,x1,x0,x7,x6,x5,x4] < lsb +} + +# AES linear components + +sub ShiftRows { +my @x=@_[0..7]; +my @t=@_[8..11]; +my $mask=pop; +$code.=<<___; + vldmia $key!, {@t[0]-@t[3]} + veor @t[0], @t[0], @x[0] + veor @t[1], @t[1], @x[1] + vtbl.8 `&Dlo(@x[0])`, {@t[0]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[0])`, {@t[0]}, `&Dhi($mask)` + vldmia $key!, {@t[0]} + veor @t[2], @t[2], @x[2] + vtbl.8 `&Dlo(@x[1])`, {@t[1]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[1])`, {@t[1]}, `&Dhi($mask)` + vldmia $key!, {@t[1]} + veor @t[3], @t[3], @x[3] + vtbl.8 `&Dlo(@x[2])`, {@t[2]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[2])`, {@t[2]}, `&Dhi($mask)` + vldmia $key!, {@t[2]} + vtbl.8 `&Dlo(@x[3])`, {@t[3]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[3])`, {@t[3]}, `&Dhi($mask)` + vldmia $key!, {@t[3]} + veor @t[0], @t[0], @x[4] + veor @t[1], @t[1], @x[5] + vtbl.8 `&Dlo(@x[4])`, {@t[0]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[4])`, {@t[0]}, `&Dhi($mask)` + veor @t[2], @t[2], @x[6] + vtbl.8 `&Dlo(@x[5])`, {@t[1]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[5])`, {@t[1]}, `&Dhi($mask)` + veor @t[3], @t[3], @x[7] + vtbl.8 `&Dlo(@x[6])`, {@t[2]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[6])`, {@t[2]}, `&Dhi($mask)` + vtbl.8 `&Dlo(@x[7])`, {@t[3]}, `&Dlo($mask)` + vtbl.8 `&Dhi(@x[7])`, {@t[3]}, `&Dhi($mask)` +___ +} + +sub MixColumns { +# modified to emit output in order suitable for feeding back to aesenc[last] +my @x=@_[0..7]; +my @t=@_[8..15]; +my $inv=@_[16]; # optional +$code.=<<___; + vext.8 @t[0], @x[0], @x[0], #12 @ x0 <<< 32 + vext.8 @t[1], @x[1], @x[1], #12 + veor @x[0], @x[0], @t[0] @ x0 ^ (x0 <<< 32) + vext.8 @t[2], @x[2], @x[2], #12 + veor @x[1], @x[1], @t[1] + vext.8 @t[3], @x[3], @x[3], #12 + veor @x[2], @x[2], @t[2] + vext.8 @t[4], @x[4], @x[4], #12 + veor @x[3], @x[3], @t[3] + vext.8 @t[5], @x[5], @x[5], #12 + veor @x[4], @x[4], @t[4] + vext.8 @t[6], @x[6], @x[6], #12 + veor @x[5], @x[5], @t[5] + vext.8 @t[7], @x[7], @x[7], #12 + veor @x[6], @x[6], @t[6] + + veor @t[1], @t[1], @x[0] + veor @x[7], @x[7], @t[7] + vext.8 @x[0], @x[0], @x[0], #8 @ (x0 ^ (x0 <<< 32)) <<< 64) + veor @t[2], @t[2], @x[1] + veor @t[0], @t[0], @x[7] + veor @t[1], @t[1], @x[7] + vext.8 @x[1], @x[1], @x[1], #8 + veor @t[5], @t[5], @x[4] + veor @x[0], @x[0], @t[0] + veor @t[6], @t[6], @x[5] + veor @x[1], @x[1], @t[1] + vext.8 @t[0], @x[4], @x[4], #8 + veor @t[4], @t[4], @x[3] + vext.8 @t[1], @x[5], @x[5], #8 + veor @t[7], @t[7], @x[6] + vext.8 @x[4], @x[3], @x[3], #8 + veor @t[3], @t[3], @x[2] + vext.8 @x[5], @x[7], @x[7], #8 + veor @t[4], @t[4], @x[7] + vext.8 @x[3], @x[6], @x[6], #8 + veor @t[3], @t[3], @x[7] + vext.8 @x[6], @x[2], @x[2], #8 + veor @x[7], @t[1], @t[5] +___ +$code.=<<___ if (!$inv); + veor @x[2], @t[0], @t[4] + veor @x[4], @x[4], @t[3] + veor @x[5], @x[5], @t[7] + veor @x[3], @x[3], @t[6] + @ vmov @x[2], @t[0] + veor @x[6], @x[6], @t[2] + @ vmov @x[7], @t[1] +___ +$code.=<<___ if ($inv); + veor @t[3], @t[3], @x[4] + veor @x[5], @x[5], @t[7] + veor @x[2], @x[3], @t[6] + veor @x[3], @t[0], @t[4] + veor @x[4], @x[6], @t[2] + vmov @x[6], @t[3] + @ vmov @x[7], @t[1] +___ +} + +sub InvMixColumns_orig { +my @x=@_[0..7]; +my @t=@_[8..15]; + +$code.=<<___; + @ multiplication by 0x0e + vext.8 @t[7], @x[7], @x[7], #12 + vmov @t[2], @x[2] + veor @x[2], @x[2], @x[5] @ 2 5 + veor @x[7], @x[7], @x[5] @ 7 5 + vext.8 @t[0], @x[0], @x[0], #12 + vmov @t[5], @x[5] + veor @x[5], @x[5], @x[0] @ 5 0 [1] + veor @x[0], @x[0], @x[1] @ 0 1 + vext.8 @t[1], @x[1], @x[1], #12 + veor @x[1], @x[1], @x[2] @ 1 25 + veor @x[0], @x[0], @x[6] @ 01 6 [2] + vext.8 @t[3], @x[3], @x[3], #12 + veor @x[1], @x[1], @x[3] @ 125 3 [4] + veor @x[2], @x[2], @x[0] @ 25 016 [3] + veor @x[3], @x[3], @x[7] @ 3 75 + veor @x[7], @x[7], @x[6] @ 75 6 [0] + vext.8 @t[6], @x[6], @x[6], #12 + vmov @t[4], @x[4] + veor @x[6], @x[6], @x[4] @ 6 4 + veor @x[4], @x[4], @x[3] @ 4 375 [6] + veor @x[3], @x[3], @x[7] @ 375 756=36 + veor @x[6], @x[6], @t[5] @ 64 5 [7] + veor @x[3], @x[3], @t[2] @ 36 2 + vext.8 @t[5], @t[5], @t[5], #12 + veor @x[3], @x[3], @t[4] @ 362 4 [5] +___ + my @y = @x[7,5,0,2,1,3,4,6]; +$code.=<<___; + @ multiplication by 0x0b + veor @y[1], @y[1], @y[0] + veor @y[0], @y[0], @t[0] + vext.8 @t[2], @t[2], @t[2], #12 + veor @y[1], @y[1], @t[1] + veor @y[0], @y[0], @t[5] + vext.8 @t[4], @t[4], @t[4], #12 + veor @y[1], @y[1], @t[6] + veor @y[0], @y[0], @t[7] + veor @t[7], @t[7], @t[6] @ clobber t[7] + + veor @y[3], @y[3], @t[0] + veor @y[1], @y[1], @y[0] + vext.8 @t[0], @t[0], @t[0], #12 + veor @y[2], @y[2], @t[1] + veor @y[4], @y[4], @t[1] + vext.8 @t[1], @t[1], @t[1], #12 + veor @y[2], @y[2], @t[2] + veor @y[3], @y[3], @t[2] + veor @y[5], @y[5], @t[2] + veor @y[2], @y[2], @t[7] + vext.8 @t[2], @t[2], @t[2], #12 + veor @y[3], @y[3], @t[3] + veor @y[6], @y[6], @t[3] + veor @y[4], @y[4], @t[3] + veor @y[7], @y[7], @t[4] + vext.8 @t[3], @t[3], @t[3], #12 + veor @y[5], @y[5], @t[4] + veor @y[7], @y[7], @t[7] + veor @t[7], @t[7], @t[5] @ clobber t[7] even more + veor @y[3], @y[3], @t[5] + veor @y[4], @y[4], @t[4] + + veor @y[5], @y[5], @t[7] + vext.8 @t[4], @t[4], @t[4], #12 + veor @y[6], @y[6], @t[7] + veor @y[4], @y[4], @t[7] + + veor @t[7], @t[7], @t[5] + vext.8 @t[5], @t[5], @t[5], #12 + + @ multiplication by 0x0d + veor @y[4], @y[4], @y[7] + veor @t[7], @t[7], @t[6] @ restore t[7] + veor @y[7], @y[7], @t[4] + vext.8 @t[6], @t[6], @t[6], #12 + veor @y[2], @y[2], @t[0] + veor @y[7], @y[7], @t[5] + vext.8 @t[7], @t[7], @t[7], #12 + veor @y[2], @y[2], @t[2] + + veor @y[3], @y[3], @y[1] + veor @y[1], @y[1], @t[1] + veor @y[0], @y[0], @t[0] + veor @y[3], @y[3], @t[0] + veor @y[1], @y[1], @t[5] + veor @y[0], @y[0], @t[5] + vext.8 @t[0], @t[0], @t[0], #12 + veor @y[1], @y[1], @t[7] + veor @y[0], @y[0], @t[6] + veor @y[3], @y[3], @y[1] + veor @y[4], @y[4], @t[1] + vext.8 @t[1], @t[1], @t[1], #12 + + veor @y[7], @y[7], @t[7] + veor @y[4], @y[4], @t[2] + veor @y[5], @y[5], @t[2] + veor @y[2], @y[2], @t[6] + veor @t[6], @t[6], @t[3] @ clobber t[6] + vext.8 @t[2], @t[2], @t[2], #12 + veor @y[4], @y[4], @y[7] + veor @y[3], @y[3], @t[6] + + veor @y[6], @y[6], @t[6] + veor @y[5], @y[5], @t[5] + vext.8 @t[5], @t[5], @t[5], #12 + veor @y[6], @y[6], @t[4] + vext.8 @t[4], @t[4], @t[4], #12 + veor @y[5], @y[5], @t[6] + veor @y[6], @y[6], @t[7] + vext.8 @t[7], @t[7], @t[7], #12 + veor @t[6], @t[6], @t[3] @ restore t[6] + vext.8 @t[3], @t[3], @t[3], #12 + + @ multiplication by 0x09 + veor @y[4], @y[4], @y[1] + veor @t[1], @t[1], @y[1] @ t[1]=y[1] + veor @t[0], @t[0], @t[5] @ clobber t[0] + vext.8 @t[6], @t[6], @t[6], #12 + veor @t[1], @t[1], @t[5] + veor @y[3], @y[3], @t[0] + veor @t[0], @t[0], @y[0] @ t[0]=y[0] + veor @t[1], @t[1], @t[6] + veor @t[6], @t[6], @t[7] @ clobber t[6] + veor @y[4], @y[4], @t[1] + veor @y[7], @y[7], @t[4] + veor @y[6], @y[6], @t[3] + veor @y[5], @y[5], @t[2] + veor @t[4], @t[4], @y[4] @ t[4]=y[4] + veor @t[3], @t[3], @y[3] @ t[3]=y[3] + veor @t[5], @t[5], @y[5] @ t[5]=y[5] + veor @t[2], @t[2], @y[2] @ t[2]=y[2] + veor @t[3], @t[3], @t[7] + veor @XMM[5], @t[5], @t[6] + veor @XMM[6], @t[6], @y[6] @ t[6]=y[6] + veor @XMM[2], @t[2], @t[6] + veor @XMM[7], @t[7], @y[7] @ t[7]=y[7] + + vmov @XMM[0], @t[0] + vmov @XMM[1], @t[1] + @ vmov @XMM[2], @t[2] + vmov @XMM[3], @t[3] + vmov @XMM[4], @t[4] + @ vmov @XMM[5], @t[5] + @ vmov @XMM[6], @t[6] + @ vmov @XMM[7], @t[7] +___ +} + +sub InvMixColumns { +my @x=@_[0..7]; +my @t=@_[8..15]; + +# Thanks to Jussi Kivilinna for providing pointer to +# +# | 0e 0b 0d 09 | | 02 03 01 01 | | 05 00 04 00 | +# | 09 0e 0b 0d | = | 01 02 03 01 | x | 00 05 00 04 | +# | 0d 09 0e 0b | | 01 01 02 03 | | 04 00 05 00 | +# | 0b 0d 09 0e | | 03 01 01 02 | | 00 04 00 05 | + +$code.=<<___; + @ multiplication by 0x05-0x00-0x04-0x00 + vext.8 @t[0], @x[0], @x[0], #8 + vext.8 @t[6], @x[6], @x[6], #8 + vext.8 @t[7], @x[7], @x[7], #8 + veor @t[0], @t[0], @x[0] + vext.8 @t[1], @x[1], @x[1], #8 + veor @t[6], @t[6], @x[6] + vext.8 @t[2], @x[2], @x[2], #8 + veor @t[7], @t[7], @x[7] + vext.8 @t[3], @x[3], @x[3], #8 + veor @t[1], @t[1], @x[1] + vext.8 @t[4], @x[4], @x[4], #8 + veor @t[2], @t[2], @x[2] + vext.8 @t[5], @x[5], @x[5], #8 + veor @t[3], @t[3], @x[3] + veor @t[4], @t[4], @x[4] + veor @t[5], @t[5], @x[5] + + veor @x[0], @x[0], @t[6] + veor @x[1], @x[1], @t[6] + veor @x[2], @x[2], @t[0] + veor @x[4], @x[4], @t[2] + veor @x[3], @x[3], @t[1] + veor @x[1], @x[1], @t[7] + veor @x[2], @x[2], @t[7] + veor @x[4], @x[4], @t[6] + veor @x[5], @x[5], @t[3] + veor @x[3], @x[3], @t[6] + veor @x[6], @x[6], @t[4] + veor @x[4], @x[4], @t[7] + veor @x[5], @x[5], @t[7] + veor @x[7], @x[7], @t[5] +___ + &MixColumns (@x,@t,1); # flipped 2<->3 and 4<->6 +} + +sub swapmove { +my ($a,$b,$n,$mask,$t)=@_; +$code.=<<___; + vshr.u64 $t, $b, #$n + veor $t, $t, $a + vand $t, $t, $mask + veor $a, $a, $t + vshl.u64 $t, $t, #$n + veor $b, $b, $t +___ +} +sub swapmove2x { +my ($a0,$b0,$a1,$b1,$n,$mask,$t0,$t1)=@_; +$code.=<<___; + vshr.u64 $t0, $b0, #$n + vshr.u64 $t1, $b1, #$n + veor $t0, $t0, $a0 + veor $t1, $t1, $a1 + vand $t0, $t0, $mask + vand $t1, $t1, $mask + veor $a0, $a0, $t0 + vshl.u64 $t0, $t0, #$n + veor $a1, $a1, $t1 + vshl.u64 $t1, $t1, #$n + veor $b0, $b0, $t0 + veor $b1, $b1, $t1 +___ +} + +sub bitslice { +my @x=reverse(@_[0..7]); +my ($t0,$t1,$t2,$t3)=@_[8..11]; +$code.=<<___; + vmov.i8 $t0,#0x55 @ compose .LBS0 + vmov.i8 $t1,#0x33 @ compose .LBS1 +___ + &swapmove2x(@x[0,1,2,3],1,$t0,$t2,$t3); + &swapmove2x(@x[4,5,6,7],1,$t0,$t2,$t3); +$code.=<<___; + vmov.i8 $t0,#0x0f @ compose .LBS2 +___ + &swapmove2x(@x[0,2,1,3],2,$t1,$t2,$t3); + &swapmove2x(@x[4,6,5,7],2,$t1,$t2,$t3); + + &swapmove2x(@x[0,4,1,5],4,$t0,$t2,$t3); + &swapmove2x(@x[2,6,3,7],4,$t0,$t2,$t3); +} + +$code.=<<___; +#ifndef __KERNEL__ +# include "arm_arch.h" + +# define VFP_ABI_PUSH vstmdb sp!,{d8-d15} +# define VFP_ABI_POP vldmia sp!,{d8-d15} +# define VFP_ABI_FRAME 0x40 +#else +# define VFP_ABI_PUSH +# define VFP_ABI_POP +# define VFP_ABI_FRAME 0 +# define BSAES_ASM_EXTENDED_KEY +# define XTS_CHAIN_TWEAK +# define __ARM_ARCH__ __LINUX_ARM_ARCH__ +#endif + +#ifdef __thumb__ +# define adrl adr +#endif + +#if __ARM_ARCH__>=7 +.text +.syntax unified @ ARMv7-capable assembler is expected to handle this +#ifdef __thumb2__ +.thumb +#else +.code 32 +#endif + +.fpu neon + +.type _bsaes_decrypt8,%function +.align 4 +_bsaes_decrypt8: + adr $const,_bsaes_decrypt8 + vldmia $key!, {@XMM[9]} @ round 0 key + add $const,$const,#.LM0ISR-_bsaes_decrypt8 + + vldmia $const!, {@XMM[8]} @ .LM0ISR + veor @XMM[10], @XMM[0], @XMM[9] @ xor with round0 key + veor @XMM[11], @XMM[1], @XMM[9] + vtbl.8 `&Dlo(@XMM[0])`, {@XMM[10]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[0])`, {@XMM[10]}, `&Dhi(@XMM[8])` + veor @XMM[12], @XMM[2], @XMM[9] + vtbl.8 `&Dlo(@XMM[1])`, {@XMM[11]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[1])`, {@XMM[11]}, `&Dhi(@XMM[8])` + veor @XMM[13], @XMM[3], @XMM[9] + vtbl.8 `&Dlo(@XMM[2])`, {@XMM[12]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[2])`, {@XMM[12]}, `&Dhi(@XMM[8])` + veor @XMM[14], @XMM[4], @XMM[9] + vtbl.8 `&Dlo(@XMM[3])`, {@XMM[13]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[3])`, {@XMM[13]}, `&Dhi(@XMM[8])` + veor @XMM[15], @XMM[5], @XMM[9] + vtbl.8 `&Dlo(@XMM[4])`, {@XMM[14]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[4])`, {@XMM[14]}, `&Dhi(@XMM[8])` + veor @XMM[10], @XMM[6], @XMM[9] + vtbl.8 `&Dlo(@XMM[5])`, {@XMM[15]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[5])`, {@XMM[15]}, `&Dhi(@XMM[8])` + veor @XMM[11], @XMM[7], @XMM[9] + vtbl.8 `&Dlo(@XMM[6])`, {@XMM[10]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[6])`, {@XMM[10]}, `&Dhi(@XMM[8])` + vtbl.8 `&Dlo(@XMM[7])`, {@XMM[11]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[7])`, {@XMM[11]}, `&Dhi(@XMM[8])` +___ + &bitslice (@XMM[0..7, 8..11]); +$code.=<<___; + sub $rounds,$rounds,#1 + b .Ldec_sbox +.align 4 +.Ldec_loop: +___ + &ShiftRows (@XMM[0..7, 8..12]); +$code.=".Ldec_sbox:\n"; + &InvSbox (@XMM[0..7, 8..15]); +$code.=<<___; + subs $rounds,$rounds,#1 + bcc .Ldec_done +___ + &InvMixColumns (@XMM[0,1,6,4,2,7,3,5, 8..15]); +$code.=<<___; + vldmia $const, {@XMM[12]} @ .LISR + ite eq @ Thumb2 thing, sanity check in ARM + addeq $const,$const,#0x10 + bne .Ldec_loop + vldmia $const, {@XMM[12]} @ .LISRM0 + b .Ldec_loop +.align 4 +.Ldec_done: +___ + &bitslice (@XMM[0,1,6,4,2,7,3,5, 8..11]); +$code.=<<___; + vldmia $key, {@XMM[8]} @ last round key + veor @XMM[6], @XMM[6], @XMM[8] + veor @XMM[4], @XMM[4], @XMM[8] + veor @XMM[2], @XMM[2], @XMM[8] + veor @XMM[7], @XMM[7], @XMM[8] + veor @XMM[3], @XMM[3], @XMM[8] + veor @XMM[5], @XMM[5], @XMM[8] + veor @XMM[0], @XMM[0], @XMM[8] + veor @XMM[1], @XMM[1], @XMM[8] + bx lr +.size _bsaes_decrypt8,.-_bsaes_decrypt8 + +.type _bsaes_const,%object +.align 6 +_bsaes_const: +.LM0ISR: @ InvShiftRows constants + .quad 0x0a0e0206070b0f03, 0x0004080c0d010509 +.LISR: + .quad 0x0504070602010003, 0x0f0e0d0c080b0a09 +.LISRM0: + .quad 0x01040b0e0205080f, 0x0306090c00070a0d +.LM0SR: @ ShiftRows constants + .quad 0x0a0e02060f03070b, 0x0004080c05090d01 +.LSR: + .quad 0x0504070600030201, 0x0f0e0d0c0a09080b +.LSRM0: + .quad 0x0304090e00050a0f, 0x01060b0c0207080d +.LM0: + .quad 0x02060a0e03070b0f, 0x0004080c0105090d +.LREVM0SR: + .quad 0x090d01050c000408, 0x03070b0f060a0e02 +.asciz "Bit-sliced AES for NEON, CRYPTOGAMS by " +.align 6 +.size _bsaes_const,.-_bsaes_const + +.type _bsaes_encrypt8,%function +.align 4 +_bsaes_encrypt8: + adr $const,_bsaes_encrypt8 + vldmia $key!, {@XMM[9]} @ round 0 key + sub $const,$const,#_bsaes_encrypt8-.LM0SR + + vldmia $const!, {@XMM[8]} @ .LM0SR +_bsaes_encrypt8_alt: + veor @XMM[10], @XMM[0], @XMM[9] @ xor with round0 key + veor @XMM[11], @XMM[1], @XMM[9] + vtbl.8 `&Dlo(@XMM[0])`, {@XMM[10]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[0])`, {@XMM[10]}, `&Dhi(@XMM[8])` + veor @XMM[12], @XMM[2], @XMM[9] + vtbl.8 `&Dlo(@XMM[1])`, {@XMM[11]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[1])`, {@XMM[11]}, `&Dhi(@XMM[8])` + veor @XMM[13], @XMM[3], @XMM[9] + vtbl.8 `&Dlo(@XMM[2])`, {@XMM[12]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[2])`, {@XMM[12]}, `&Dhi(@XMM[8])` + veor @XMM[14], @XMM[4], @XMM[9] + vtbl.8 `&Dlo(@XMM[3])`, {@XMM[13]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[3])`, {@XMM[13]}, `&Dhi(@XMM[8])` + veor @XMM[15], @XMM[5], @XMM[9] + vtbl.8 `&Dlo(@XMM[4])`, {@XMM[14]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[4])`, {@XMM[14]}, `&Dhi(@XMM[8])` + veor @XMM[10], @XMM[6], @XMM[9] + vtbl.8 `&Dlo(@XMM[5])`, {@XMM[15]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[5])`, {@XMM[15]}, `&Dhi(@XMM[8])` + veor @XMM[11], @XMM[7], @XMM[9] + vtbl.8 `&Dlo(@XMM[6])`, {@XMM[10]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[6])`, {@XMM[10]}, `&Dhi(@XMM[8])` + vtbl.8 `&Dlo(@XMM[7])`, {@XMM[11]}, `&Dlo(@XMM[8])` + vtbl.8 `&Dhi(@XMM[7])`, {@XMM[11]}, `&Dhi(@XMM[8])` +_bsaes_encrypt8_bitslice: +___ + &bitslice (@XMM[0..7, 8..11]); +$code.=<<___; + sub $rounds,$rounds,#1 + b .Lenc_sbox +.align 4 +.Lenc_loop: +___ + &ShiftRows (@XMM[0..7, 8..12]); +$code.=".Lenc_sbox:\n"; + &Sbox (@XMM[0..7, 8..15]); +$code.=<<___; + subs $rounds,$rounds,#1 + bcc .Lenc_done +___ + &MixColumns (@XMM[0,1,4,6,3,7,2,5, 8..15]); +$code.=<<___; + vldmia $const, {@XMM[12]} @ .LSR + ite eq @ Thumb2 thing, samity check in ARM + addeq $const,$const,#0x10 + bne .Lenc_loop + vldmia $const, {@XMM[12]} @ .LSRM0 + b .Lenc_loop +.align 4 +.Lenc_done: +___ + # output in lsb > [t0, t1, t4, t6, t3, t7, t2, t5] < msb + &bitslice (@XMM[0,1,4,6,3,7,2,5, 8..11]); +$code.=<<___; + vldmia $key, {@XMM[8]} @ last round key + veor @XMM[4], @XMM[4], @XMM[8] + veor @XMM[6], @XMM[6], @XMM[8] + veor @XMM[3], @XMM[3], @XMM[8] + veor @XMM[7], @XMM[7], @XMM[8] + veor @XMM[2], @XMM[2], @XMM[8] + veor @XMM[5], @XMM[5], @XMM[8] + veor @XMM[0], @XMM[0], @XMM[8] + veor @XMM[1], @XMM[1], @XMM[8] + bx lr +.size _bsaes_encrypt8,.-_bsaes_encrypt8 +___ +} +{ +my ($out,$inp,$rounds,$const)=("r12","r4","r5","r6"); + +sub bitslice_key { +my @x=reverse(@_[0..7]); +my ($bs0,$bs1,$bs2,$t2,$t3)=@_[8..12]; + + &swapmove (@x[0,1],1,$bs0,$t2,$t3); +$code.=<<___; + @ &swapmove(@x[2,3],1,$t0,$t2,$t3); + vmov @x[2], @x[0] + vmov @x[3], @x[1] +___ + #&swapmove2x(@x[4,5,6,7],1,$t0,$t2,$t3); + + &swapmove2x (@x[0,2,1,3],2,$bs1,$t2,$t3); +$code.=<<___; + @ &swapmove2x(@x[4,6,5,7],2,$t1,$t2,$t3); + vmov @x[4], @x[0] + vmov @x[6], @x[2] + vmov @x[5], @x[1] + vmov @x[7], @x[3] +___ + &swapmove2x (@x[0,4,1,5],4,$bs2,$t2,$t3); + &swapmove2x (@x[2,6,3,7],4,$bs2,$t2,$t3); +} + +$code.=<<___; +.type _bsaes_key_convert,%function +.align 4 +_bsaes_key_convert: + adr $const,_bsaes_key_convert + vld1.8 {@XMM[7]}, [$inp]! @ load round 0 key + sub $const,$const,#_bsaes_key_convert-.LM0 + vld1.8 {@XMM[15]}, [$inp]! @ load round 1 key + + vmov.i8 @XMM[8], #0x01 @ bit masks + vmov.i8 @XMM[9], #0x02 + vmov.i8 @XMM[10], #0x04 + vmov.i8 @XMM[11], #0x08 + vmov.i8 @XMM[12], #0x10 + vmov.i8 @XMM[13], #0x20 + vldmia $const, {@XMM[14]} @ .LM0 + +#ifdef __ARMEL__ + vrev32.8 @XMM[7], @XMM[7] + vrev32.8 @XMM[15], @XMM[15] +#endif + sub $rounds,$rounds,#1 + vstmia $out!, {@XMM[7]} @ save round 0 key + b .Lkey_loop + +.align 4 +.Lkey_loop: + vtbl.8 `&Dlo(@XMM[7])`,{@XMM[15]},`&Dlo(@XMM[14])` + vtbl.8 `&Dhi(@XMM[7])`,{@XMM[15]},`&Dhi(@XMM[14])` + vmov.i8 @XMM[6], #0x40 + vmov.i8 @XMM[15], #0x80 + + vtst.8 @XMM[0], @XMM[7], @XMM[8] + vtst.8 @XMM[1], @XMM[7], @XMM[9] + vtst.8 @XMM[2], @XMM[7], @XMM[10] + vtst.8 @XMM[3], @XMM[7], @XMM[11] + vtst.8 @XMM[4], @XMM[7], @XMM[12] + vtst.8 @XMM[5], @XMM[7], @XMM[13] + vtst.8 @XMM[6], @XMM[7], @XMM[6] + vtst.8 @XMM[7], @XMM[7], @XMM[15] + vld1.8 {@XMM[15]}, [$inp]! @ load next round key + vmvn @XMM[0], @XMM[0] @ "pnot" + vmvn @XMM[1], @XMM[1] + vmvn @XMM[5], @XMM[5] + vmvn @XMM[6], @XMM[6] +#ifdef __ARMEL__ + vrev32.8 @XMM[15], @XMM[15] +#endif + subs $rounds,$rounds,#1 + vstmia $out!,{@XMM[0]-@XMM[7]} @ write bit-sliced round key + bne .Lkey_loop + + vmov.i8 @XMM[7],#0x63 @ compose .L63 + @ don't save last round key + bx lr +.size _bsaes_key_convert,.-_bsaes_key_convert +___ +} + +if (0) { # following four functions are unsupported interface + # used for benchmarking... +$code.=<<___; +.globl bsaes_enc_key_convert +.type bsaes_enc_key_convert,%function +.align 4 +bsaes_enc_key_convert: + stmdb sp!,{r4-r6,lr} + vstmdb sp!,{d8-d15} @ ABI specification says so + + ldr r5,[$inp,#240] @ pass rounds + mov r4,$inp @ pass key + mov r12,$out @ pass key schedule + bl _bsaes_key_convert + veor @XMM[7],@XMM[7],@XMM[15] @ fix up last round key + vstmia r12, {@XMM[7]} @ save last round key + + vldmia sp!,{d8-d15} + ldmia sp!,{r4-r6,pc} +.size bsaes_enc_key_convert,.-bsaes_enc_key_convert + +.globl bsaes_encrypt_128 +.type bsaes_encrypt_128,%function +.align 4 +bsaes_encrypt_128: + stmdb sp!,{r4-r6,lr} + vstmdb sp!,{d8-d15} @ ABI specification says so +.Lenc128_loop: + vld1.8 {@XMM[0]-@XMM[1]}, [$inp]! @ load input + vld1.8 {@XMM[2]-@XMM[3]}, [$inp]! + mov r4,$key @ pass the key + vld1.8 {@XMM[4]-@XMM[5]}, [$inp]! + mov r5,#10 @ pass rounds + vld1.8 {@XMM[6]-@XMM[7]}, [$inp]! + + bl _bsaes_encrypt8 + + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + vst1.8 {@XMM[4]}, [$out]! + vst1.8 {@XMM[6]}, [$out]! + vst1.8 {@XMM[3]}, [$out]! + vst1.8 {@XMM[7]}, [$out]! + vst1.8 {@XMM[2]}, [$out]! + subs $len,$len,#0x80 + vst1.8 {@XMM[5]}, [$out]! + bhi .Lenc128_loop + + vldmia sp!,{d8-d15} + ldmia sp!,{r4-r6,pc} +.size bsaes_encrypt_128,.-bsaes_encrypt_128 + +.globl bsaes_dec_key_convert +.type bsaes_dec_key_convert,%function +.align 4 +bsaes_dec_key_convert: + stmdb sp!,{r4-r6,lr} + vstmdb sp!,{d8-d15} @ ABI specification says so + + ldr r5,[$inp,#240] @ pass rounds + mov r4,$inp @ pass key + mov r12,$out @ pass key schedule + bl _bsaes_key_convert + vldmia $out, {@XMM[6]} + vstmia r12, {@XMM[15]} @ save last round key + veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key + vstmia $out, {@XMM[7]} + + vldmia sp!,{d8-d15} + ldmia sp!,{r4-r6,pc} +.size bsaes_dec_key_convert,.-bsaes_dec_key_convert + +.globl bsaes_decrypt_128 +.type bsaes_decrypt_128,%function +.align 4 +bsaes_decrypt_128: + stmdb sp!,{r4-r6,lr} + vstmdb sp!,{d8-d15} @ ABI specification says so +.Ldec128_loop: + vld1.8 {@XMM[0]-@XMM[1]}, [$inp]! @ load input + vld1.8 {@XMM[2]-@XMM[3]}, [$inp]! + mov r4,$key @ pass the key + vld1.8 {@XMM[4]-@XMM[5]}, [$inp]! + mov r5,#10 @ pass rounds + vld1.8 {@XMM[6]-@XMM[7]}, [$inp]! + + bl _bsaes_decrypt8 + + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + vst1.8 {@XMM[6]}, [$out]! + vst1.8 {@XMM[4]}, [$out]! + vst1.8 {@XMM[2]}, [$out]! + vst1.8 {@XMM[7]}, [$out]! + vst1.8 {@XMM[3]}, [$out]! + subs $len,$len,#0x80 + vst1.8 {@XMM[5]}, [$out]! + bhi .Ldec128_loop + + vldmia sp!,{d8-d15} + ldmia sp!,{r4-r6,pc} +.size bsaes_decrypt_128,.-bsaes_decrypt_128 +___ +} +{ +my ($inp,$out,$len,$key, $ivp,$fp,$rounds)=map("r$_",(0..3,8..10)); +my ($keysched)=("sp"); + +$code.=<<___; +.extern AES_cbc_encrypt +.extern AES_decrypt + +.global bsaes_cbc_encrypt +.type bsaes_cbc_encrypt,%function +.align 5 +bsaes_cbc_encrypt: +#ifndef __KERNEL__ + cmp $len, #128 +#ifndef __thumb__ + blo AES_cbc_encrypt +#else + bhs 1f + b AES_cbc_encrypt +1: +#endif +#endif + + @ it is up to the caller to make sure we are called with enc == 0 + + mov ip, sp + stmdb sp!, {r4-r10, lr} + VFP_ABI_PUSH + ldr $ivp, [ip] @ IV is 1st arg on the stack + mov $len, $len, lsr#4 @ len in 16 byte blocks + sub sp, #0x10 @ scratch space to carry over the IV + mov $fp, sp @ save sp + + ldr $rounds, [$key, #240] @ get # of rounds +#ifndef BSAES_ASM_EXTENDED_KEY + @ allocate the key schedule on the stack + sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key + add r12, #`128-32` @ sifze of bit-slices key schedule + + @ populate the key schedule + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + mov sp, r12 @ sp is $keysched + bl _bsaes_key_convert + vldmia $keysched, {@XMM[6]} + vstmia r12, {@XMM[15]} @ save last round key + veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key + vstmia $keysched, {@XMM[7]} +#else + ldr r12, [$key, #244] + eors r12, #1 + beq 0f + + @ populate the key schedule + str r12, [$key, #244] + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + add r12, $key, #248 @ pass key schedule + bl _bsaes_key_convert + add r4, $key, #248 + vldmia r4, {@XMM[6]} + vstmia r12, {@XMM[15]} @ save last round key + veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key + vstmia r4, {@XMM[7]} + +.align 2 +0: +#endif + + vld1.8 {@XMM[15]}, [$ivp] @ load IV + b .Lcbc_dec_loop + +.align 4 +.Lcbc_dec_loop: + subs $len, $len, #0x8 + bmi .Lcbc_dec_loop_finish + + vld1.8 {@XMM[0]-@XMM[1]}, [$inp]! @ load input + vld1.8 {@XMM[2]-@XMM[3]}, [$inp]! +#ifndef BSAES_ASM_EXTENDED_KEY + mov r4, $keysched @ pass the key +#else + add r4, $key, #248 +#endif + vld1.8 {@XMM[4]-@XMM[5]}, [$inp]! + mov r5, $rounds + vld1.8 {@XMM[6]-@XMM[7]}, [$inp] + sub $inp, $inp, #0x60 + vstmia $fp, {@XMM[15]} @ put aside IV + + bl _bsaes_decrypt8 + + vldmia $fp, {@XMM[14]} @ reload IV + vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input + veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV + vld1.8 {@XMM[10]-@XMM[11]}, [$inp]! + veor @XMM[1], @XMM[1], @XMM[8] + veor @XMM[6], @XMM[6], @XMM[9] + vld1.8 {@XMM[12]-@XMM[13]}, [$inp]! + veor @XMM[4], @XMM[4], @XMM[10] + veor @XMM[2], @XMM[2], @XMM[11] + vld1.8 {@XMM[14]-@XMM[15]}, [$inp]! + veor @XMM[7], @XMM[7], @XMM[12] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + veor @XMM[3], @XMM[3], @XMM[13] + vst1.8 {@XMM[6]}, [$out]! + veor @XMM[5], @XMM[5], @XMM[14] + vst1.8 {@XMM[4]}, [$out]! + vst1.8 {@XMM[2]}, [$out]! + vst1.8 {@XMM[7]}, [$out]! + vst1.8 {@XMM[3]}, [$out]! + vst1.8 {@XMM[5]}, [$out]! + + b .Lcbc_dec_loop + +.Lcbc_dec_loop_finish: + adds $len, $len, #8 + beq .Lcbc_dec_done + + vld1.8 {@XMM[0]}, [$inp]! @ load input + cmp $len, #2 + blo .Lcbc_dec_one + vld1.8 {@XMM[1]}, [$inp]! +#ifndef BSAES_ASM_EXTENDED_KEY + mov r4, $keysched @ pass the key +#else + add r4, $key, #248 +#endif + mov r5, $rounds + vstmia $fp, {@XMM[15]} @ put aside IV + beq .Lcbc_dec_two + vld1.8 {@XMM[2]}, [$inp]! + cmp $len, #4 + blo .Lcbc_dec_three + vld1.8 {@XMM[3]}, [$inp]! + beq .Lcbc_dec_four + vld1.8 {@XMM[4]}, [$inp]! + cmp $len, #6 + blo .Lcbc_dec_five + vld1.8 {@XMM[5]}, [$inp]! + beq .Lcbc_dec_six + vld1.8 {@XMM[6]}, [$inp]! + sub $inp, $inp, #0x70 + + bl _bsaes_decrypt8 + + vldmia $fp, {@XMM[14]} @ reload IV + vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input + veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV + vld1.8 {@XMM[10]-@XMM[11]}, [$inp]! + veor @XMM[1], @XMM[1], @XMM[8] + veor @XMM[6], @XMM[6], @XMM[9] + vld1.8 {@XMM[12]-@XMM[13]}, [$inp]! + veor @XMM[4], @XMM[4], @XMM[10] + veor @XMM[2], @XMM[2], @XMM[11] + vld1.8 {@XMM[15]}, [$inp]! + veor @XMM[7], @XMM[7], @XMM[12] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + veor @XMM[3], @XMM[3], @XMM[13] + vst1.8 {@XMM[6]}, [$out]! + vst1.8 {@XMM[4]}, [$out]! + vst1.8 {@XMM[2]}, [$out]! + vst1.8 {@XMM[7]}, [$out]! + vst1.8 {@XMM[3]}, [$out]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_six: + sub $inp, $inp, #0x60 + bl _bsaes_decrypt8 + vldmia $fp,{@XMM[14]} @ reload IV + vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input + veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV + vld1.8 {@XMM[10]-@XMM[11]}, [$inp]! + veor @XMM[1], @XMM[1], @XMM[8] + veor @XMM[6], @XMM[6], @XMM[9] + vld1.8 {@XMM[12]}, [$inp]! + veor @XMM[4], @XMM[4], @XMM[10] + veor @XMM[2], @XMM[2], @XMM[11] + vld1.8 {@XMM[15]}, [$inp]! + veor @XMM[7], @XMM[7], @XMM[12] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + vst1.8 {@XMM[6]}, [$out]! + vst1.8 {@XMM[4]}, [$out]! + vst1.8 {@XMM[2]}, [$out]! + vst1.8 {@XMM[7]}, [$out]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_five: + sub $inp, $inp, #0x50 + bl _bsaes_decrypt8 + vldmia $fp, {@XMM[14]} @ reload IV + vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input + veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV + vld1.8 {@XMM[10]-@XMM[11]}, [$inp]! + veor @XMM[1], @XMM[1], @XMM[8] + veor @XMM[6], @XMM[6], @XMM[9] + vld1.8 {@XMM[15]}, [$inp]! + veor @XMM[4], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + veor @XMM[2], @XMM[2], @XMM[11] + vst1.8 {@XMM[6]}, [$out]! + vst1.8 {@XMM[4]}, [$out]! + vst1.8 {@XMM[2]}, [$out]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_four: + sub $inp, $inp, #0x40 + bl _bsaes_decrypt8 + vldmia $fp, {@XMM[14]} @ reload IV + vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input + veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV + vld1.8 {@XMM[10]}, [$inp]! + veor @XMM[1], @XMM[1], @XMM[8] + veor @XMM[6], @XMM[6], @XMM[9] + vld1.8 {@XMM[15]}, [$inp]! + veor @XMM[4], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + vst1.8 {@XMM[6]}, [$out]! + vst1.8 {@XMM[4]}, [$out]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_three: + sub $inp, $inp, #0x30 + bl _bsaes_decrypt8 + vldmia $fp, {@XMM[14]} @ reload IV + vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input + veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV + vld1.8 {@XMM[15]}, [$inp]! + veor @XMM[1], @XMM[1], @XMM[8] + veor @XMM[6], @XMM[6], @XMM[9] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + vst1.8 {@XMM[6]}, [$out]! + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_two: + sub $inp, $inp, #0x20 + bl _bsaes_decrypt8 + vldmia $fp, {@XMM[14]} @ reload IV + vld1.8 {@XMM[8]}, [$inp]! @ reload input + veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV + vld1.8 {@XMM[15]}, [$inp]! @ reload input + veor @XMM[1], @XMM[1], @XMM[8] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + b .Lcbc_dec_done +.align 4 +.Lcbc_dec_one: + sub $inp, $inp, #0x10 + mov $rounds, $out @ save original out pointer + mov $out, $fp @ use the iv scratch space as out buffer + mov r2, $key + vmov @XMM[4],@XMM[15] @ just in case ensure that IV + vmov @XMM[5],@XMM[0] @ and input are preserved + bl AES_decrypt + vld1.8 {@XMM[0]}, [$fp,:64] @ load result + veor @XMM[0], @XMM[0], @XMM[4] @ ^= IV + vmov @XMM[15], @XMM[5] @ @XMM[5] holds input + vst1.8 {@XMM[0]}, [$rounds] @ write output + +.Lcbc_dec_done: +#ifndef BSAES_ASM_EXTENDED_KEY + vmov.i32 q0, #0 + vmov.i32 q1, #0 +.Lcbc_dec_bzero: @ wipe key schedule [if any] + vstmia $keysched!, {q0-q1} + cmp $keysched, $fp + bne .Lcbc_dec_bzero +#endif + + mov sp, $fp + add sp, #0x10 @ add sp,$fp,#0x10 is no good for thumb + vst1.8 {@XMM[15]}, [$ivp] @ return IV + VFP_ABI_POP + ldmia sp!, {r4-r10, pc} +.size bsaes_cbc_encrypt,.-bsaes_cbc_encrypt +___ +} +{ +my ($inp,$out,$len,$key, $ctr,$fp,$rounds)=(map("r$_",(0..3,8..10))); +my $const = "r6"; # shared with _bsaes_encrypt8_alt +my $keysched = "sp"; + +$code.=<<___; +.extern AES_encrypt +.global bsaes_ctr32_encrypt_blocks +.type bsaes_ctr32_encrypt_blocks,%function +.align 5 +bsaes_ctr32_encrypt_blocks: + cmp $len, #8 @ use plain AES for + blo .Lctr_enc_short @ small sizes + + mov ip, sp + stmdb sp!, {r4-r10, lr} + VFP_ABI_PUSH + ldr $ctr, [ip] @ ctr is 1st arg on the stack + sub sp, sp, #0x10 @ scratch space to carry over the ctr + mov $fp, sp @ save sp + + ldr $rounds, [$key, #240] @ get # of rounds +#ifndef BSAES_ASM_EXTENDED_KEY + @ allocate the key schedule on the stack + sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key + add r12, #`128-32` @ size of bit-sliced key schedule + + @ populate the key schedule + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + mov sp, r12 @ sp is $keysched + bl _bsaes_key_convert + veor @XMM[7],@XMM[7],@XMM[15] @ fix up last round key + vstmia r12, {@XMM[7]} @ save last round key + + vld1.8 {@XMM[0]}, [$ctr] @ load counter + add $ctr, $const, #.LREVM0SR-.LM0 @ borrow $ctr + vldmia $keysched, {@XMM[4]} @ load round0 key +#else + ldr r12, [$key, #244] + eors r12, #1 + beq 0f + + @ populate the key schedule + str r12, [$key, #244] + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + add r12, $key, #248 @ pass key schedule + bl _bsaes_key_convert + veor @XMM[7],@XMM[7],@XMM[15] @ fix up last round key + vstmia r12, {@XMM[7]} @ save last round key + +.align 2 +0: add r12, $key, #248 + vld1.8 {@XMM[0]}, [$ctr] @ load counter + adrl $ctr, .LREVM0SR @ borrow $ctr + vldmia r12, {@XMM[4]} @ load round0 key + sub sp, #0x10 @ place for adjusted round0 key +#endif + + vmov.i32 @XMM[8],#1 @ compose 1<<96 + veor @XMM[9],@XMM[9],@XMM[9] + vrev32.8 @XMM[0],@XMM[0] + vext.8 @XMM[8],@XMM[9],@XMM[8],#4 + vrev32.8 @XMM[4],@XMM[4] + vadd.u32 @XMM[9],@XMM[8],@XMM[8] @ compose 2<<96 + vstmia $keysched, {@XMM[4]} @ save adjusted round0 key + b .Lctr_enc_loop + +.align 4 +.Lctr_enc_loop: + vadd.u32 @XMM[10], @XMM[8], @XMM[9] @ compose 3<<96 + vadd.u32 @XMM[1], @XMM[0], @XMM[8] @ +1 + vadd.u32 @XMM[2], @XMM[0], @XMM[9] @ +2 + vadd.u32 @XMM[3], @XMM[0], @XMM[10] @ +3 + vadd.u32 @XMM[4], @XMM[1], @XMM[10] + vadd.u32 @XMM[5], @XMM[2], @XMM[10] + vadd.u32 @XMM[6], @XMM[3], @XMM[10] + vadd.u32 @XMM[7], @XMM[4], @XMM[10] + vadd.u32 @XMM[10], @XMM[5], @XMM[10] @ next counter + + @ Borrow prologue from _bsaes_encrypt8 to use the opportunity + @ to flip byte order in 32-bit counter + + vldmia $keysched, {@XMM[9]} @ load round0 key +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, $keysched, #0x10 @ pass next round key +#else + add r4, $key, #`248+16` +#endif + vldmia $ctr, {@XMM[8]} @ .LREVM0SR + mov r5, $rounds @ pass rounds + vstmia $fp, {@XMM[10]} @ save next counter + sub $const, $ctr, #.LREVM0SR-.LSR @ pass constants + + bl _bsaes_encrypt8_alt + + subs $len, $len, #8 + blo .Lctr_enc_loop_done + + vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ load input + vld1.8 {@XMM[10]-@XMM[11]}, [$inp]! + veor @XMM[0], @XMM[8] + veor @XMM[1], @XMM[9] + vld1.8 {@XMM[12]-@XMM[13]}, [$inp]! + veor @XMM[4], @XMM[10] + veor @XMM[6], @XMM[11] + vld1.8 {@XMM[14]-@XMM[15]}, [$inp]! + veor @XMM[3], @XMM[12] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output + veor @XMM[7], @XMM[13] + veor @XMM[2], @XMM[14] + vst1.8 {@XMM[4]}, [$out]! + veor @XMM[5], @XMM[15] + vst1.8 {@XMM[6]}, [$out]! + vmov.i32 @XMM[8], #1 @ compose 1<<96 + vst1.8 {@XMM[3]}, [$out]! + veor @XMM[9], @XMM[9], @XMM[9] + vst1.8 {@XMM[7]}, [$out]! + vext.8 @XMM[8], @XMM[9], @XMM[8], #4 + vst1.8 {@XMM[2]}, [$out]! + vadd.u32 @XMM[9],@XMM[8],@XMM[8] @ compose 2<<96 + vst1.8 {@XMM[5]}, [$out]! + vldmia $fp, {@XMM[0]} @ load counter + + bne .Lctr_enc_loop + b .Lctr_enc_done + +.align 4 +.Lctr_enc_loop_done: + add $len, $len, #8 + vld1.8 {@XMM[8]}, [$inp]! @ load input + veor @XMM[0], @XMM[8] + vst1.8 {@XMM[0]}, [$out]! @ write output + cmp $len, #2 + blo .Lctr_enc_done + vld1.8 {@XMM[9]}, [$inp]! + veor @XMM[1], @XMM[9] + vst1.8 {@XMM[1]}, [$out]! + beq .Lctr_enc_done + vld1.8 {@XMM[10]}, [$inp]! + veor @XMM[4], @XMM[10] + vst1.8 {@XMM[4]}, [$out]! + cmp $len, #4 + blo .Lctr_enc_done + vld1.8 {@XMM[11]}, [$inp]! + veor @XMM[6], @XMM[11] + vst1.8 {@XMM[6]}, [$out]! + beq .Lctr_enc_done + vld1.8 {@XMM[12]}, [$inp]! + veor @XMM[3], @XMM[12] + vst1.8 {@XMM[3]}, [$out]! + cmp $len, #6 + blo .Lctr_enc_done + vld1.8 {@XMM[13]}, [$inp]! + veor @XMM[7], @XMM[13] + vst1.8 {@XMM[7]}, [$out]! + beq .Lctr_enc_done + vld1.8 {@XMM[14]}, [$inp] + veor @XMM[2], @XMM[14] + vst1.8 {@XMM[2]}, [$out]! + +.Lctr_enc_done: + vmov.i32 q0, #0 + vmov.i32 q1, #0 +#ifndef BSAES_ASM_EXTENDED_KEY +.Lctr_enc_bzero: @ wipe key schedule [if any] + vstmia $keysched!, {q0-q1} + cmp $keysched, $fp + bne .Lctr_enc_bzero +#else + vstmia $keysched, {q0-q1} +#endif + + mov sp, $fp + add sp, #0x10 @ add sp,$fp,#0x10 is no good for thumb + VFP_ABI_POP + ldmia sp!, {r4-r10, pc} @ return + +.align 4 +.Lctr_enc_short: + ldr ip, [sp] @ ctr pointer is passed on stack + stmdb sp!, {r4-r8, lr} + + mov r4, $inp @ copy arguments + mov r5, $out + mov r6, $len + mov r7, $key + ldr r8, [ip, #12] @ load counter LSW + vld1.8 {@XMM[1]}, [ip] @ load whole counter value +#ifdef __ARMEL__ + rev r8, r8 +#endif + sub sp, sp, #0x10 + vst1.8 {@XMM[1]}, [sp,:64] @ copy counter value + sub sp, sp, #0x10 + +.Lctr_enc_short_loop: + add r0, sp, #0x10 @ input counter value + mov r1, sp @ output on the stack + mov r2, r7 @ key + + bl AES_encrypt + + vld1.8 {@XMM[0]}, [r4]! @ load input + vld1.8 {@XMM[1]}, [sp,:64] @ load encrypted counter + add r8, r8, #1 +#ifdef __ARMEL__ + rev r0, r8 + str r0, [sp, #0x1c] @ next counter value +#else + str r8, [sp, #0x1c] @ next counter value +#endif + veor @XMM[0],@XMM[0],@XMM[1] + vst1.8 {@XMM[0]}, [r5]! @ store output + subs r6, r6, #1 + bne .Lctr_enc_short_loop + + vmov.i32 q0, #0 + vmov.i32 q1, #0 + vstmia sp!, {q0-q1} + + ldmia sp!, {r4-r8, pc} +.size bsaes_ctr32_encrypt_blocks,.-bsaes_ctr32_encrypt_blocks +___ +} +{ +###################################################################### +# void bsaes_xts_[en|de]crypt(const char *inp,char *out,size_t len, +# const AES_KEY *key1, const AES_KEY *key2, +# const unsigned char iv[16]); +# +my ($inp,$out,$len,$key,$rounds,$magic,$fp)=(map("r$_",(7..10,1..3))); +my $const="r6"; # returned by _bsaes_key_convert +my $twmask=@XMM[5]; +my @T=@XMM[6..7]; + +$code.=<<___; +.globl bsaes_xts_encrypt +.type bsaes_xts_encrypt,%function +.align 4 +bsaes_xts_encrypt: + mov ip, sp + stmdb sp!, {r4-r10, lr} @ 0x20 + VFP_ABI_PUSH + mov r6, sp @ future $fp + + mov $inp, r0 + mov $out, r1 + mov $len, r2 + mov $key, r3 + + sub r0, sp, #0x10 @ 0x10 + bic r0, #0xf @ align at 16 bytes + mov sp, r0 + +#ifdef XTS_CHAIN_TWEAK + ldr r0, [ip] @ pointer to input tweak +#else + @ generate initial tweak + ldr r0, [ip, #4] @ iv[] + mov r1, sp + ldr r2, [ip, #0] @ key2 + bl AES_encrypt + mov r0,sp @ pointer to initial tweak +#endif + + ldr $rounds, [$key, #240] @ get # of rounds + mov $fp, r6 +#ifndef BSAES_ASM_EXTENDED_KEY + @ allocate the key schedule on the stack + sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key + @ add r12, #`128-32` @ size of bit-sliced key schedule + sub r12, #`32+16` @ place for tweak[9] + + @ populate the key schedule + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + mov sp, r12 + add r12, #0x90 @ pass key schedule + bl _bsaes_key_convert + veor @XMM[7], @XMM[7], @XMM[15] @ fix up last round key + vstmia r12, {@XMM[7]} @ save last round key +#else + ldr r12, [$key, #244] + eors r12, #1 + beq 0f + + str r12, [$key, #244] + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + add r12, $key, #248 @ pass key schedule + bl _bsaes_key_convert + veor @XMM[7], @XMM[7], @XMM[15] @ fix up last round key + vstmia r12, {@XMM[7]} + +.align 2 +0: sub sp, #0x90 @ place for tweak[9] +#endif + + vld1.8 {@XMM[8]}, [r0] @ initial tweak + adr $magic, .Lxts_magic + + subs $len, #0x80 + blo .Lxts_enc_short + b .Lxts_enc_loop + +.align 4 +.Lxts_enc_loop: + vldmia $magic, {$twmask} @ load XTS magic + vshr.s64 @T[0], @XMM[8], #63 + mov r0, sp + vand @T[0], @T[0], $twmask +___ +for($i=9;$i<16;$i++) { +$code.=<<___; + vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1] + vst1.64 {@XMM[$i-1]}, [r0,:128]! + vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` + vshr.s64 @T[1], @XMM[$i], #63 + veor @XMM[$i], @XMM[$i], @T[0] + vand @T[1], @T[1], $twmask +___ + @T=reverse(@T); + +$code.=<<___ if ($i>=10); + vld1.8 {@XMM[$i-10]}, [$inp]! +___ +$code.=<<___ if ($i>=11); + veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3] +___ +} +$code.=<<___; + vadd.u64 @XMM[8], @XMM[15], @XMM[15] + vst1.64 {@XMM[15]}, [r0,:128]! + vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` + veor @XMM[8], @XMM[8], @T[0] + vst1.64 {@XMM[8]}, [r0,:128] @ next round tweak + + vld1.8 {@XMM[6]-@XMM[7]}, [$inp]! + veor @XMM[5], @XMM[5], @XMM[13] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[6], @XMM[6], @XMM[14] + mov r5, $rounds @ pass rounds + veor @XMM[7], @XMM[7], @XMM[15] + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[6], @XMM[11] + vld1.64 {@XMM[14]-@XMM[15]}, [r0,:128]! + veor @XMM[10], @XMM[3], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + veor @XMM[11], @XMM[7], @XMM[13] + veor @XMM[12], @XMM[2], @XMM[14] + vst1.8 {@XMM[10]-@XMM[11]}, [$out]! + veor @XMM[13], @XMM[5], @XMM[15] + vst1.8 {@XMM[12]-@XMM[13]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + + subs $len, #0x80 + bpl .Lxts_enc_loop + +.Lxts_enc_short: + adds $len, #0x70 + bmi .Lxts_enc_done + + vldmia $magic, {$twmask} @ load XTS magic + vshr.s64 @T[0], @XMM[8], #63 + mov r0, sp + vand @T[0], @T[0], $twmask +___ +for($i=9;$i<16;$i++) { +$code.=<<___; + vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1] + vst1.64 {@XMM[$i-1]}, [r0,:128]! + vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` + vshr.s64 @T[1], @XMM[$i], #63 + veor @XMM[$i], @XMM[$i], @T[0] + vand @T[1], @T[1], $twmask +___ + @T=reverse(@T); + +$code.=<<___ if ($i>=10); + vld1.8 {@XMM[$i-10]}, [$inp]! + subs $len, #0x10 + bmi .Lxts_enc_`$i-9` +___ +$code.=<<___ if ($i>=11); + veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3] +___ +} +$code.=<<___; + sub $len, #0x10 + vst1.64 {@XMM[15]}, [r0,:128] @ next round tweak + + vld1.8 {@XMM[6]}, [$inp]! + veor @XMM[5], @XMM[5], @XMM[13] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[6], @XMM[6], @XMM[14] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[6], @XMM[11] + vld1.64 {@XMM[14]}, [r0,:128]! + veor @XMM[10], @XMM[3], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + veor @XMM[11], @XMM[7], @XMM[13] + veor @XMM[12], @XMM[2], @XMM[14] + vst1.8 {@XMM[10]-@XMM[11]}, [$out]! + vst1.8 {@XMM[12]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_6: + vst1.64 {@XMM[14]}, [r0,:128] @ next round tweak + + veor @XMM[4], @XMM[4], @XMM[12] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[5], @XMM[5], @XMM[13] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[6], @XMM[11] + veor @XMM[10], @XMM[3], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + veor @XMM[11], @XMM[7], @XMM[13] + vst1.8 {@XMM[10]-@XMM[11]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_enc_done + +@ put this in range for both ARM and Thumb mode adr instructions +.align 5 +.Lxts_magic: + .quad 1, 0x87 + +.align 5 +.Lxts_enc_5: + vst1.64 {@XMM[13]}, [r0,:128] @ next round tweak + + veor @XMM[3], @XMM[3], @XMM[11] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[4], @XMM[4], @XMM[12] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[6], @XMM[11] + veor @XMM[10], @XMM[3], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + vst1.8 {@XMM[10]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_4: + vst1.64 {@XMM[12]}, [r0,:128] @ next round tweak + + veor @XMM[2], @XMM[2], @XMM[10] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[3], @XMM[3], @XMM[11] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[6], @XMM[11] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_3: + vst1.64 {@XMM[11]}, [r0,:128] @ next round tweak + + veor @XMM[1], @XMM[1], @XMM[9] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[2], @XMM[2], @XMM[10] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]! + vld1.64 {@XMM[10]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[4], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + vst1.8 {@XMM[8]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_2: + vst1.64 {@XMM[10]}, [r0,:128] @ next round tweak + + veor @XMM[0], @XMM[0], @XMM[8] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[1], @XMM[1], @XMM[9] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_encrypt8 + + vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + veor @XMM[1], @XMM[1], @XMM[ 9] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_enc_done +.align 4 +.Lxts_enc_1: + mov r0, sp + veor @XMM[0], @XMM[8] + mov r1, sp + vst1.8 {@XMM[0]}, [sp,:128] + mov r2, $key + mov r4, $fp @ preserve fp + + bl AES_encrypt + + vld1.8 {@XMM[0]}, [sp,:128] + veor @XMM[0], @XMM[0], @XMM[8] + vst1.8 {@XMM[0]}, [$out]! + mov $fp, r4 + + vmov @XMM[8], @XMM[9] @ next round tweak + +.Lxts_enc_done: +#ifndef XTS_CHAIN_TWEAK + adds $len, #0x10 + beq .Lxts_enc_ret + sub r6, $out, #0x10 + +.Lxts_enc_steal: + ldrb r0, [$inp], #1 + ldrb r1, [$out, #-0x10] + strb r0, [$out, #-0x10] + strb r1, [$out], #1 + + subs $len, #1 + bhi .Lxts_enc_steal + + vld1.8 {@XMM[0]}, [r6] + mov r0, sp + veor @XMM[0], @XMM[0], @XMM[8] + mov r1, sp + vst1.8 {@XMM[0]}, [sp,:128] + mov r2, $key + mov r4, $fp @ preserve fp + + bl AES_encrypt + + vld1.8 {@XMM[0]}, [sp,:128] + veor @XMM[0], @XMM[0], @XMM[8] + vst1.8 {@XMM[0]}, [r6] + mov $fp, r4 +#endif + +.Lxts_enc_ret: + bic r0, $fp, #0xf + vmov.i32 q0, #0 + vmov.i32 q1, #0 +#ifdef XTS_CHAIN_TWEAK + ldr r1, [$fp, #0x20+VFP_ABI_FRAME] @ chain tweak +#endif +.Lxts_enc_bzero: @ wipe key schedule [if any] + vstmia sp!, {q0-q1} + cmp sp, r0 + bne .Lxts_enc_bzero + + mov sp, $fp +#ifdef XTS_CHAIN_TWEAK + vst1.8 {@XMM[8]}, [r1] +#endif + VFP_ABI_POP + ldmia sp!, {r4-r10, pc} @ return + +.size bsaes_xts_encrypt,.-bsaes_xts_encrypt + +.globl bsaes_xts_decrypt +.type bsaes_xts_decrypt,%function +.align 4 +bsaes_xts_decrypt: + mov ip, sp + stmdb sp!, {r4-r10, lr} @ 0x20 + VFP_ABI_PUSH + mov r6, sp @ future $fp + + mov $inp, r0 + mov $out, r1 + mov $len, r2 + mov $key, r3 + + sub r0, sp, #0x10 @ 0x10 + bic r0, #0xf @ align at 16 bytes + mov sp, r0 + +#ifdef XTS_CHAIN_TWEAK + ldr r0, [ip] @ pointer to input tweak +#else + @ generate initial tweak + ldr r0, [ip, #4] @ iv[] + mov r1, sp + ldr r2, [ip, #0] @ key2 + bl AES_encrypt + mov r0, sp @ pointer to initial tweak +#endif + + ldr $rounds, [$key, #240] @ get # of rounds + mov $fp, r6 +#ifndef BSAES_ASM_EXTENDED_KEY + @ allocate the key schedule on the stack + sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key + @ add r12, #`128-32` @ size of bit-sliced key schedule + sub r12, #`32+16` @ place for tweak[9] + + @ populate the key schedule + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + mov sp, r12 + add r12, #0x90 @ pass key schedule + bl _bsaes_key_convert + add r4, sp, #0x90 + vldmia r4, {@XMM[6]} + vstmia r12, {@XMM[15]} @ save last round key + veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key + vstmia r4, {@XMM[7]} +#else + ldr r12, [$key, #244] + eors r12, #1 + beq 0f + + str r12, [$key, #244] + mov r4, $key @ pass key + mov r5, $rounds @ pass # of rounds + add r12, $key, #248 @ pass key schedule + bl _bsaes_key_convert + add r4, $key, #248 + vldmia r4, {@XMM[6]} + vstmia r12, {@XMM[15]} @ save last round key + veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key + vstmia r4, {@XMM[7]} + +.align 2 +0: sub sp, #0x90 @ place for tweak[9] +#endif + vld1.8 {@XMM[8]}, [r0] @ initial tweak + adr $magic, .Lxts_magic + + tst $len, #0xf @ if not multiple of 16 + it ne @ Thumb2 thing, sanity check in ARM + subne $len, #0x10 @ subtract another 16 bytes + subs $len, #0x80 + + blo .Lxts_dec_short + b .Lxts_dec_loop + +.align 4 +.Lxts_dec_loop: + vldmia $magic, {$twmask} @ load XTS magic + vshr.s64 @T[0], @XMM[8], #63 + mov r0, sp + vand @T[0], @T[0], $twmask +___ +for($i=9;$i<16;$i++) { +$code.=<<___; + vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1] + vst1.64 {@XMM[$i-1]}, [r0,:128]! + vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` + vshr.s64 @T[1], @XMM[$i], #63 + veor @XMM[$i], @XMM[$i], @T[0] + vand @T[1], @T[1], $twmask +___ + @T=reverse(@T); + +$code.=<<___ if ($i>=10); + vld1.8 {@XMM[$i-10]}, [$inp]! +___ +$code.=<<___ if ($i>=11); + veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3] +___ +} +$code.=<<___; + vadd.u64 @XMM[8], @XMM[15], @XMM[15] + vst1.64 {@XMM[15]}, [r0,:128]! + vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` + veor @XMM[8], @XMM[8], @T[0] + vst1.64 {@XMM[8]}, [r0,:128] @ next round tweak + + vld1.8 {@XMM[6]-@XMM[7]}, [$inp]! + veor @XMM[5], @XMM[5], @XMM[13] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[6], @XMM[6], @XMM[14] + mov r5, $rounds @ pass rounds + veor @XMM[7], @XMM[7], @XMM[15] + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[6], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[4], @XMM[11] + vld1.64 {@XMM[14]-@XMM[15]}, [r0,:128]! + veor @XMM[10], @XMM[2], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + veor @XMM[11], @XMM[7], @XMM[13] + veor @XMM[12], @XMM[3], @XMM[14] + vst1.8 {@XMM[10]-@XMM[11]}, [$out]! + veor @XMM[13], @XMM[5], @XMM[15] + vst1.8 {@XMM[12]-@XMM[13]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + + subs $len, #0x80 + bpl .Lxts_dec_loop + +.Lxts_dec_short: + adds $len, #0x70 + bmi .Lxts_dec_done + + vldmia $magic, {$twmask} @ load XTS magic + vshr.s64 @T[0], @XMM[8], #63 + mov r0, sp + vand @T[0], @T[0], $twmask +___ +for($i=9;$i<16;$i++) { +$code.=<<___; + vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1] + vst1.64 {@XMM[$i-1]}, [r0,:128]! + vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` + vshr.s64 @T[1], @XMM[$i], #63 + veor @XMM[$i], @XMM[$i], @T[0] + vand @T[1], @T[1], $twmask +___ + @T=reverse(@T); + +$code.=<<___ if ($i>=10); + vld1.8 {@XMM[$i-10]}, [$inp]! + subs $len, #0x10 + bmi .Lxts_dec_`$i-9` +___ +$code.=<<___ if ($i>=11); + veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3] +___ +} +$code.=<<___; + sub $len, #0x10 + vst1.64 {@XMM[15]}, [r0,:128] @ next round tweak + + vld1.8 {@XMM[6]}, [$inp]! + veor @XMM[5], @XMM[5], @XMM[13] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[6], @XMM[6], @XMM[14] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[6], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[4], @XMM[11] + vld1.64 {@XMM[14]}, [r0,:128]! + veor @XMM[10], @XMM[2], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + veor @XMM[11], @XMM[7], @XMM[13] + veor @XMM[12], @XMM[3], @XMM[14] + vst1.8 {@XMM[10]-@XMM[11]}, [$out]! + vst1.8 {@XMM[12]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_6: + vst1.64 {@XMM[14]}, [r0,:128] @ next round tweak + + veor @XMM[4], @XMM[4], @XMM[12] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[5], @XMM[5], @XMM[13] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[6], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[4], @XMM[11] + veor @XMM[10], @XMM[2], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + veor @XMM[11], @XMM[7], @XMM[13] + vst1.8 {@XMM[10]-@XMM[11]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_5: + vst1.64 {@XMM[13]}, [r0,:128] @ next round tweak + + veor @XMM[3], @XMM[3], @XMM[11] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[4], @XMM[4], @XMM[12] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + vld1.64 {@XMM[12]}, [r0,:128]! + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[6], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[4], @XMM[11] + veor @XMM[10], @XMM[2], @XMM[12] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + vst1.8 {@XMM[10]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_4: + vst1.64 {@XMM[12]}, [r0,:128] @ next round tweak + + veor @XMM[2], @XMM[2], @XMM[10] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[3], @XMM[3], @XMM[11] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! + vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[6], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + veor @XMM[9], @XMM[4], @XMM[11] + vst1.8 {@XMM[8]-@XMM[9]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_3: + vst1.64 {@XMM[11]}, [r0,:128] @ next round tweak + + veor @XMM[1], @XMM[1], @XMM[9] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[2], @XMM[2], @XMM[10] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]! + vld1.64 {@XMM[10]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + veor @XMM[1], @XMM[1], @XMM[ 9] + veor @XMM[8], @XMM[6], @XMM[10] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + vst1.8 {@XMM[8]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_2: + vst1.64 {@XMM[10]}, [r0,:128] @ next round tweak + + veor @XMM[0], @XMM[0], @XMM[8] +#ifndef BSAES_ASM_EXTENDED_KEY + add r4, sp, #0x90 @ pass key schedule +#else + add r4, $key, #248 @ pass key schedule +#endif + veor @XMM[1], @XMM[1], @XMM[9] + mov r5, $rounds @ pass rounds + mov r0, sp + + bl _bsaes_decrypt8 + + vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]! + veor @XMM[0], @XMM[0], @XMM[ 8] + veor @XMM[1], @XMM[1], @XMM[ 9] + vst1.8 {@XMM[0]-@XMM[1]}, [$out]! + + vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak + b .Lxts_dec_done +.align 4 +.Lxts_dec_1: + mov r0, sp + veor @XMM[0], @XMM[8] + mov r1, sp + vst1.8 {@XMM[0]}, [sp,:128] + mov r2, $key + mov r4, $fp @ preserve fp + mov r5, $magic @ preserve magic + + bl AES_decrypt + + vld1.8 {@XMM[0]}, [sp,:128] + veor @XMM[0], @XMM[0], @XMM[8] + vst1.8 {@XMM[0]}, [$out]! + mov $fp, r4 + mov $magic, r5 + + vmov @XMM[8], @XMM[9] @ next round tweak + +.Lxts_dec_done: +#ifndef XTS_CHAIN_TWEAK + adds $len, #0x10 + beq .Lxts_dec_ret + + @ calculate one round of extra tweak for the stolen ciphertext + vldmia $magic, {$twmask} + vshr.s64 @XMM[6], @XMM[8], #63 + vand @XMM[6], @XMM[6], $twmask + vadd.u64 @XMM[9], @XMM[8], @XMM[8] + vswp `&Dhi("@XMM[6]")`,`&Dlo("@XMM[6]")` + veor @XMM[9], @XMM[9], @XMM[6] + + @ perform the final decryption with the last tweak value + vld1.8 {@XMM[0]}, [$inp]! + mov r0, sp + veor @XMM[0], @XMM[0], @XMM[9] + mov r1, sp + vst1.8 {@XMM[0]}, [sp,:128] + mov r2, $key + mov r4, $fp @ preserve fp + + bl AES_decrypt + + vld1.8 {@XMM[0]}, [sp,:128] + veor @XMM[0], @XMM[0], @XMM[9] + vst1.8 {@XMM[0]}, [$out] + + mov r6, $out +.Lxts_dec_steal: + ldrb r1, [$out] + ldrb r0, [$inp], #1 + strb r1, [$out, #0x10] + strb r0, [$out], #1 + + subs $len, #1 + bhi .Lxts_dec_steal + + vld1.8 {@XMM[0]}, [r6] + mov r0, sp + veor @XMM[0], @XMM[8] + mov r1, sp + vst1.8 {@XMM[0]}, [sp,:128] + mov r2, $key + + bl AES_decrypt + + vld1.8 {@XMM[0]}, [sp,:128] + veor @XMM[0], @XMM[0], @XMM[8] + vst1.8 {@XMM[0]}, [r6] + mov $fp, r4 +#endif + +.Lxts_dec_ret: + bic r0, $fp, #0xf + vmov.i32 q0, #0 + vmov.i32 q1, #0 +#ifdef XTS_CHAIN_TWEAK + ldr r1, [$fp, #0x20+VFP_ABI_FRAME] @ chain tweak +#endif +.Lxts_dec_bzero: @ wipe key schedule [if any] + vstmia sp!, {q0-q1} + cmp sp, r0 + bne .Lxts_dec_bzero + + mov sp, $fp +#ifdef XTS_CHAIN_TWEAK + vst1.8 {@XMM[8]}, [r1] +#endif + VFP_ABI_POP + ldmia sp!, {r4-r10, pc} @ return + +.size bsaes_xts_decrypt,.-bsaes_xts_decrypt +___ +} +$code.=<<___; +#endif +___ + +$code =~ s/\`([^\`]*)\`/eval($1)/gem; + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/@/ and !/^$/); + print; +} +close SELF; + +print $code; + +close STDOUT; diff --git a/crypto/Kconfig b/crypto/Kconfig index 2615d4cfc61..f7f9204f7ea 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -610,6 +610,22 @@ config CRYPTO_AES_ARM See for more information. +config CRYPTO_AES_ARM_BS + tristate "Bit sliced AES using NEON instructions" + depends on ARM && KERNEL_MODE_NEON + select CRYPTO_ALGAPI + select CRYPTO_AES_ARM + select CRYPTO_ABLK_HELPER + help + Use a faster and more secure NEON based implementation of AES in CBC, + CTR and XTS modes + + Bit sliced AES gives around 45% speedup on Cortex-A15 for CTR mode + and for XTS mode encryption, CBC and XTS mode decryption speedup is + around 25%. (CBC encryption speed is not affected by this driver.) + This implementation does not rely on any lookup tables so it is + believed to be invulnerable to cache timing attacks. + config CRYPTO_ANUBIS tristate "Anubis cipher algorithm" select CRYPTO_ALGAPI From f87f0101b39df577c9239727bcc37f4410c34a71 Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 7 Oct 2013 15:43:53 +0100 Subject: [PATCH 055/168] ARM: add .gitignore entry for aesbs-core.S This avoids this file being incorrectly added to git. Change-Id: I2f32e7b585628949509a2c26048671fd2a4d52af Signed-off-by: Russell King Git-commit: 4e5e4ea6ac51e35922cd8bea663d8604052054a7 Git-repo: https://android.googlesource.com/kernel/common/ Signed-off-by: Samir Mehta --- arch/arm/crypto/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 arch/arm/crypto/.gitignore diff --git a/arch/arm/crypto/.gitignore b/arch/arm/crypto/.gitignore new file mode 100644 index 00000000000..6231d36b363 --- /dev/null +++ b/arch/arm/crypto/.gitignore @@ -0,0 +1 @@ +aesbs-core.S From 74e810314ffce16eeedbe9bed54936cae18bf97d Mon Sep 17 00:00:00 2001 From: Russell King Date: Thu, 2 Jan 2014 17:14:45 +0000 Subject: [PATCH 056/168] CRYPTO: Fix more AES build errors Building a multi-arch kernel results in: arch/arm/crypto/built-in.o: In function `aesbs_xts_decrypt': sha1_glue.c:(.text+0x15c8): undefined reference to `bsaes_xts_decrypt' arch/arm/crypto/built-in.o: In function `aesbs_xts_encrypt': sha1_glue.c:(.text+0x1664): undefined reference to `bsaes_xts_encrypt' arch/arm/crypto/built-in.o: In function `aesbs_ctr_encrypt': sha1_glue.c:(.text+0x184c): undefined reference to `bsaes_ctr32_encrypt_blocks' arch/arm/crypto/built-in.o: In function `aesbs_cbc_decrypt': sha1_glue.c:(.text+0x19b4): undefined reference to `bsaes_cbc_encrypt' This code is already runtime-conditional on NEON being supported, so there's no point compiling it out depending on the minimum build architecture. Change-Id: I3b4b44b83a422c3154d6612c906733201a79711d Acked-by: Ard Biesheuvel Signed-off-by: Russell King Git-commit: ee3315a1f57962fa788e05568c27d05488f862f6 Git-repo: https://android.googlesource.com/kernel/common/ Signed-off-by: Samir Mehta --- arch/arm/crypto/aesbs-core.S_shipped | 2 +- arch/arm/crypto/bsaes-armv7.pl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm/crypto/aesbs-core.S_shipped b/arch/arm/crypto/aesbs-core.S_shipped index 64205d45326..71e5fc7cfb1 100644 --- a/arch/arm/crypto/aesbs-core.S_shipped +++ b/arch/arm/crypto/aesbs-core.S_shipped @@ -58,7 +58,7 @@ # define VFP_ABI_FRAME 0 # define BSAES_ASM_EXTENDED_KEY # define XTS_CHAIN_TWEAK -# define __ARM_ARCH__ __LINUX_ARM_ARCH__ +# define __ARM_ARCH__ 7 #endif #ifdef __thumb__ diff --git a/arch/arm/crypto/bsaes-armv7.pl b/arch/arm/crypto/bsaes-armv7.pl index f3d96d93257..be068db960e 100644 --- a/arch/arm/crypto/bsaes-armv7.pl +++ b/arch/arm/crypto/bsaes-armv7.pl @@ -701,7 +701,7 @@ $code.=<<___; # define VFP_ABI_FRAME 0 # define BSAES_ASM_EXTENDED_KEY # define XTS_CHAIN_TWEAK -# define __ARM_ARCH__ __LINUX_ARM_ARCH__ +# define __ARM_ARCH__ 7 #endif #ifdef __thumb__ From 74445fed05154344d24f19302c7853d392ba609a Mon Sep 17 00:00:00 2001 From: Mikulas Patocka Date: Fri, 25 Jul 2014 19:42:30 -0400 Subject: [PATCH 057/168] crypto: arm-aes - fix encryption of unaligned data Fix the same alignment bug as in arm64 - we need to pass residue unprocessed bytes as the last argument to blkcipher_walk_done. Change-Id: Ie901bd0ee362b67cd2d5cb5efdbfc8f24a9d70ee Signed-off-by: Mikulas Patocka Cc: stable@vger.kernel.org # 3.13+ Acked-by: Ard Biesheuvel Signed-off-by: Herbert Xu Git-commit: 9cf21d90f31d425b7d3b1204667bbd6afe49817f Git-repo: https://android.googlesource.com/kernel/common/ Signed-off-by: Samir Mehta --- arch/arm/crypto/aesbs-glue.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/arm/crypto/aesbs-glue.c b/arch/arm/crypto/aesbs-glue.c index 4522366da75..15468fbbdea 100644 --- a/arch/arm/crypto/aesbs-glue.c +++ b/arch/arm/crypto/aesbs-glue.c @@ -137,7 +137,7 @@ static int aesbs_cbc_encrypt(struct blkcipher_desc *desc, dst += AES_BLOCK_SIZE; } while (--blocks); } - err = blkcipher_walk_done(desc, &walk, 0); + err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE); } return err; } @@ -158,7 +158,7 @@ static int aesbs_cbc_decrypt(struct blkcipher_desc *desc, bsaes_cbc_encrypt(walk.src.virt.addr, walk.dst.virt.addr, walk.nbytes, &ctx->dec, walk.iv); kernel_neon_end(); - err = blkcipher_walk_done(desc, &walk, 0); + err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE); } while (walk.nbytes) { u32 blocks = walk.nbytes / AES_BLOCK_SIZE; @@ -182,7 +182,7 @@ static int aesbs_cbc_decrypt(struct blkcipher_desc *desc, dst += AES_BLOCK_SIZE; src += AES_BLOCK_SIZE; } while (--blocks); - err = blkcipher_walk_done(desc, &walk, 0); + err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE); } return err; } @@ -268,7 +268,7 @@ static int aesbs_xts_encrypt(struct blkcipher_desc *desc, bsaes_xts_encrypt(walk.src.virt.addr, walk.dst.virt.addr, walk.nbytes, &ctx->enc, walk.iv); kernel_neon_end(); - err = blkcipher_walk_done(desc, &walk, 0); + err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE); } return err; } @@ -292,7 +292,7 @@ static int aesbs_xts_decrypt(struct blkcipher_desc *desc, bsaes_xts_decrypt(walk.src.virt.addr, walk.dst.virt.addr, walk.nbytes, &ctx->dec, walk.iv); kernel_neon_end(); - err = blkcipher_walk_done(desc, &walk, 0); + err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE); } return err; } From 6e954c22d83cb0f8d35d600ff48961653176ca68 Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Tue, 29 Jul 2014 17:14:09 +0100 Subject: [PATCH 058/168] ARM: 8118/1: crypto: sha1/make use of common SHA-1 structures Common SHA-1 structures are defined in for code sharing. This patch changes SHA-1/ARM glue code to use these structures. Change-Id: I0feb0de726dc9e70c1c022664a264bb59f32c700 Acked-by: Ard Biesheuvel Signed-off-by: Jussi Kivilinna Signed-off-by: Russell King Git-commit: ffaf87841d9d2cf171b0427298d7aae0876e6368 Git-repo: https://android.googlesource.com/kernel/common/ Signed-off-by: Samir Mehta --- arch/arm/crypto/sha1_glue.c | 50 ++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/arch/arm/crypto/sha1_glue.c b/arch/arm/crypto/sha1_glue.c index 76cd976230b..c494e579ffc 100644 --- a/arch/arm/crypto/sha1_glue.c +++ b/arch/arm/crypto/sha1_glue.c @@ -24,31 +24,25 @@ #include #include -struct SHA1_CTX { - uint32_t h0,h1,h2,h3,h4; - u64 count; - u8 data[SHA1_BLOCK_SIZE]; -}; -asmlinkage void sha1_block_data_order(struct SHA1_CTX *digest, +asmlinkage void sha1_block_data_order(u32 *digest, const unsigned char *data, unsigned int rounds); static int sha1_init(struct shash_desc *desc) { - struct SHA1_CTX *sctx = shash_desc_ctx(desc); - memset(sctx, 0, sizeof(*sctx)); - sctx->h0 = SHA1_H0; - sctx->h1 = SHA1_H1; - sctx->h2 = SHA1_H2; - sctx->h3 = SHA1_H3; - sctx->h4 = SHA1_H4; + struct sha1_state *sctx = shash_desc_ctx(desc); + + *sctx = (struct sha1_state){ + .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, + }; + return 0; } -static int __sha1_update(struct SHA1_CTX *sctx, const u8 *data, - unsigned int len, unsigned int partial) +static int __sha1_update(struct sha1_state *sctx, const u8 *data, + unsigned int len, unsigned int partial) { unsigned int done = 0; @@ -56,17 +50,17 @@ static int __sha1_update(struct SHA1_CTX *sctx, const u8 *data, if (partial) { done = SHA1_BLOCK_SIZE - partial; - memcpy(sctx->data + partial, data, done); - sha1_block_data_order(sctx, sctx->data, 1); + memcpy(sctx->buffer + partial, data, done); + sha1_block_data_order(sctx->state, sctx->buffer, 1); } if (len - done >= SHA1_BLOCK_SIZE) { const unsigned int rounds = (len - done) / SHA1_BLOCK_SIZE; - sha1_block_data_order(sctx, data + done, rounds); + sha1_block_data_order(sctx->state, data + done, rounds); done += rounds * SHA1_BLOCK_SIZE; } - memcpy(sctx->data, data + done, len - done); + memcpy(sctx->buffer, data + done, len - done); return 0; } @@ -74,14 +68,14 @@ static int __sha1_update(struct SHA1_CTX *sctx, const u8 *data, static int sha1_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct SHA1_CTX *sctx = shash_desc_ctx(desc); + struct sha1_state *sctx = shash_desc_ctx(desc); unsigned int partial = sctx->count % SHA1_BLOCK_SIZE; int res; /* Handle the fast case right here */ if (partial + len < SHA1_BLOCK_SIZE) { sctx->count += len; - memcpy(sctx->data + partial, data, len); + memcpy(sctx->buffer + partial, data, len); return 0; } res = __sha1_update(sctx, data, len, partial); @@ -92,7 +86,7 @@ static int sha1_update(struct shash_desc *desc, const u8 *data, /* Add padding and return the message digest. */ static int sha1_final(struct shash_desc *desc, u8 *out) { - struct SHA1_CTX *sctx = shash_desc_ctx(desc); + struct sha1_state *sctx = shash_desc_ctx(desc); unsigned int i, index, padlen; __be32 *dst = (__be32 *)out; __be64 bits; @@ -106,7 +100,7 @@ static int sha1_final(struct shash_desc *desc, u8 *out) /* We need to fill a whole block for __sha1_update() */ if (padlen <= 56) { sctx->count += padlen; - memcpy(sctx->data + index, padding, padlen); + memcpy(sctx->buffer + index, padding, padlen); } else { __sha1_update(sctx, padding, padlen, index); } @@ -114,7 +108,7 @@ static int sha1_final(struct shash_desc *desc, u8 *out) /* Store state in digest */ for (i = 0; i < 5; i++) - dst[i] = cpu_to_be32(((u32 *)sctx)[i]); + dst[i] = cpu_to_be32(sctx->state[i]); /* Wipe context */ memset(sctx, 0, sizeof(*sctx)); @@ -124,7 +118,7 @@ static int sha1_final(struct shash_desc *desc, u8 *out) static int sha1_export(struct shash_desc *desc, void *out) { - struct SHA1_CTX *sctx = shash_desc_ctx(desc); + struct sha1_state *sctx = shash_desc_ctx(desc); memcpy(out, sctx, sizeof(*sctx)); return 0; } @@ -132,7 +126,7 @@ static int sha1_export(struct shash_desc *desc, void *out) static int sha1_import(struct shash_desc *desc, const void *in) { - struct SHA1_CTX *sctx = shash_desc_ctx(desc); + struct sha1_state *sctx = shash_desc_ctx(desc); memcpy(sctx, in, sizeof(*sctx)); return 0; } @@ -145,8 +139,8 @@ static struct shash_alg alg = { .final = sha1_final, .export = sha1_export, .import = sha1_import, - .descsize = sizeof(struct SHA1_CTX), - .statesize = sizeof(struct SHA1_CTX), + .descsize = sizeof(struct sha1_state), + .statesize = sizeof(struct sha1_state), .base = { .cra_name = "sha1", .cra_driver_name= "sha1-asm", From 016e998dcfa461b4f55bf4e2047e3458bac75e0d Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Tue, 29 Jul 2014 17:14:14 +0100 Subject: [PATCH 059/168] ARM: 8119/1: crypto: sha1: add ARM NEON implementation This patch adds ARM NEON assembly implementation of SHA-1 algorithm. tcrypt benchmark results on Cortex-A8, sha1-arm-asm vs sha1-neon-asm: block-size bytes/update old-vs-new 16 16 1.04x 64 16 1.02x 64 64 1.05x 256 16 1.03x 256 64 1.04x 256 256 1.30x 1024 16 1.03x 1024 256 1.36x 1024 1024 1.52x 2048 16 1.03x 2048 256 1.39x 2048 1024 1.55x 2048 2048 1.59x 4096 16 1.03x 4096 256 1.40x 4096 1024 1.57x 4096 4096 1.62x 8192 16 1.03x 8192 256 1.40x 8192 1024 1.58x 8192 4096 1.63x 8192 8192 1.63x Change-Id: I43f856e247de8ee60dd723626195dad825d1be66 Acked-by: Ard Biesheuvel Tested-by: Ard Biesheuvel Signed-off-by: Jussi Kivilinna Signed-off-by: Russell King Git-commit: d075817176091cd6476711692c4b2b707b60c908 Git-repo: https://android.googlesource.com/kernel/common/ Signed-off-by: Samir Mehta --- arch/arm/crypto/Makefile | 2 + arch/arm/crypto/sha1-armv7-neon.S | 634 +++++++++++++++++++++++++++++ arch/arm/crypto/sha1_glue.c | 8 +- arch/arm/crypto/sha1_neon_glue.c | 197 +++++++++ arch/arm/include/asm/crypto/sha1.h | 10 + crypto/Kconfig | 11 + 6 files changed, 859 insertions(+), 3 deletions(-) create mode 100644 arch/arm/crypto/sha1-armv7-neon.S create mode 100644 arch/arm/crypto/sha1_neon_glue.c create mode 100644 arch/arm/include/asm/crypto/sha1.h diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile index 81cda39860c..374956d2f89 100644 --- a/arch/arm/crypto/Makefile +++ b/arch/arm/crypto/Makefile @@ -5,10 +5,12 @@ obj-$(CONFIG_CRYPTO_AES_ARM) += aes-arm.o obj-$(CONFIG_CRYPTO_AES_ARM_BS) += aes-arm-bs.o obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o +obj-$(CONFIG_CRYPTO_SHA1_ARM_NEON) += sha1-arm-neon.o aes-arm-y := aes-armv4.o aes_glue.o aes-arm-bs-y := aesbs-core.o aesbs-glue.o sha1-arm-y := sha1-armv4-large.o sha1_glue.o +sha1-arm-neon-y := sha1-armv7-neon.o sha1_neon_glue.o quiet_cmd_perl = PERL $@ cmd_perl = $(PERL) $(<) > $(@) diff --git a/arch/arm/crypto/sha1-armv7-neon.S b/arch/arm/crypto/sha1-armv7-neon.S new file mode 100644 index 00000000000..50013c0e286 --- /dev/null +++ b/arch/arm/crypto/sha1-armv7-neon.S @@ -0,0 +1,634 @@ +/* sha1-armv7-neon.S - ARM/NEON accelerated SHA-1 transform function + * + * Copyright © 2013-2014 Jussi Kivilinna + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + */ + +#include + + +.syntax unified +.code 32 +.fpu neon + +.text + + +/* Context structure */ + +#define state_h0 0 +#define state_h1 4 +#define state_h2 8 +#define state_h3 12 +#define state_h4 16 + + +/* Constants */ + +#define K1 0x5A827999 +#define K2 0x6ED9EBA1 +#define K3 0x8F1BBCDC +#define K4 0xCA62C1D6 +.align 4 +.LK_VEC: +.LK1: .long K1, K1, K1, K1 +.LK2: .long K2, K2, K2, K2 +.LK3: .long K3, K3, K3, K3 +.LK4: .long K4, K4, K4, K4 + + +/* Register macros */ + +#define RSTATE r0 +#define RDATA r1 +#define RNBLKS r2 +#define ROLDSTACK r3 +#define RWK lr + +#define _a r4 +#define _b r5 +#define _c r6 +#define _d r7 +#define _e r8 + +#define RT0 r9 +#define RT1 r10 +#define RT2 r11 +#define RT3 r12 + +#define W0 q0 +#define W1 q1 +#define W2 q2 +#define W3 q3 +#define W4 q4 +#define W5 q5 +#define W6 q6 +#define W7 q7 + +#define tmp0 q8 +#define tmp1 q9 +#define tmp2 q10 +#define tmp3 q11 + +#define qK1 q12 +#define qK2 q13 +#define qK3 q14 +#define qK4 q15 + + +/* Round function macros. */ + +#define WK_offs(i) (((i) & 15) * 4) + +#define _R_F1(a,b,c,d,e,i,pre1,pre2,pre3,i16,\ + W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + ldr RT3, [sp, WK_offs(i)]; \ + pre1(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \ + bic RT0, d, b; \ + add e, e, a, ror #(32 - 5); \ + and RT1, c, b; \ + pre2(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \ + add RT0, RT0, RT3; \ + add e, e, RT1; \ + ror b, #(32 - 30); \ + pre3(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \ + add e, e, RT0; + +#define _R_F2(a,b,c,d,e,i,pre1,pre2,pre3,i16,\ + W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + ldr RT3, [sp, WK_offs(i)]; \ + pre1(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \ + eor RT0, d, b; \ + add e, e, a, ror #(32 - 5); \ + eor RT0, RT0, c; \ + pre2(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \ + add e, e, RT3; \ + ror b, #(32 - 30); \ + pre3(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \ + add e, e, RT0; \ + +#define _R_F3(a,b,c,d,e,i,pre1,pre2,pre3,i16,\ + W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + ldr RT3, [sp, WK_offs(i)]; \ + pre1(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \ + eor RT0, b, c; \ + and RT1, b, c; \ + add e, e, a, ror #(32 - 5); \ + pre2(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \ + and RT0, RT0, d; \ + add RT1, RT1, RT3; \ + add e, e, RT0; \ + ror b, #(32 - 30); \ + pre3(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \ + add e, e, RT1; + +#define _R_F4(a,b,c,d,e,i,pre1,pre2,pre3,i16,\ + W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + _R_F2(a,b,c,d,e,i,pre1,pre2,pre3,i16,\ + W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) + +#define _R(a,b,c,d,e,f,i,pre1,pre2,pre3,i16,\ + W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + _R_##f(a,b,c,d,e,i,pre1,pre2,pre3,i16,\ + W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) + +#define R(a,b,c,d,e,f,i) \ + _R_##f(a,b,c,d,e,i,dummy,dummy,dummy,i16,\ + W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) + +#define dummy(...) + + +/* Input expansion macros. */ + +/********* Precalc macros for rounds 0-15 *************************************/ + +#define W_PRECALC_00_15() \ + add RWK, sp, #(WK_offs(0)); \ + \ + vld1.32 {tmp0, tmp1}, [RDATA]!; \ + vrev32.8 W0, tmp0; /* big => little */ \ + vld1.32 {tmp2, tmp3}, [RDATA]!; \ + vadd.u32 tmp0, W0, curK; \ + vrev32.8 W7, tmp1; /* big => little */ \ + vrev32.8 W6, tmp2; /* big => little */ \ + vadd.u32 tmp1, W7, curK; \ + vrev32.8 W5, tmp3; /* big => little */ \ + vadd.u32 tmp2, W6, curK; \ + vst1.32 {tmp0, tmp1}, [RWK]!; \ + vadd.u32 tmp3, W5, curK; \ + vst1.32 {tmp2, tmp3}, [RWK]; \ + +#define WPRECALC_00_15_0(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vld1.32 {tmp0, tmp1}, [RDATA]!; \ + +#define WPRECALC_00_15_1(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + add RWK, sp, #(WK_offs(0)); \ + +#define WPRECALC_00_15_2(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vrev32.8 W0, tmp0; /* big => little */ \ + +#define WPRECALC_00_15_3(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vld1.32 {tmp2, tmp3}, [RDATA]!; \ + +#define WPRECALC_00_15_4(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vadd.u32 tmp0, W0, curK; \ + +#define WPRECALC_00_15_5(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vrev32.8 W7, tmp1; /* big => little */ \ + +#define WPRECALC_00_15_6(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vrev32.8 W6, tmp2; /* big => little */ \ + +#define WPRECALC_00_15_7(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vadd.u32 tmp1, W7, curK; \ + +#define WPRECALC_00_15_8(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vrev32.8 W5, tmp3; /* big => little */ \ + +#define WPRECALC_00_15_9(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vadd.u32 tmp2, W6, curK; \ + +#define WPRECALC_00_15_10(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vst1.32 {tmp0, tmp1}, [RWK]!; \ + +#define WPRECALC_00_15_11(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vadd.u32 tmp3, W5, curK; \ + +#define WPRECALC_00_15_12(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vst1.32 {tmp2, tmp3}, [RWK]; \ + + +/********* Precalc macros for rounds 16-31 ************************************/ + +#define WPRECALC_16_31_0(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + veor tmp0, tmp0; \ + vext.8 W, W_m16, W_m12, #8; \ + +#define WPRECALC_16_31_1(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + add RWK, sp, #(WK_offs(i)); \ + vext.8 tmp0, W_m04, tmp0, #4; \ + +#define WPRECALC_16_31_2(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + veor tmp0, tmp0, W_m16; \ + veor.32 W, W, W_m08; \ + +#define WPRECALC_16_31_3(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + veor tmp1, tmp1; \ + veor W, W, tmp0; \ + +#define WPRECALC_16_31_4(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vshl.u32 tmp0, W, #1; \ + +#define WPRECALC_16_31_5(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vext.8 tmp1, tmp1, W, #(16-12); \ + vshr.u32 W, W, #31; \ + +#define WPRECALC_16_31_6(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vorr tmp0, tmp0, W; \ + vshr.u32 W, tmp1, #30; \ + +#define WPRECALC_16_31_7(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vshl.u32 tmp1, tmp1, #2; \ + +#define WPRECALC_16_31_8(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + veor tmp0, tmp0, W; \ + +#define WPRECALC_16_31_9(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + veor W, tmp0, tmp1; \ + +#define WPRECALC_16_31_10(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vadd.u32 tmp0, W, curK; \ + +#define WPRECALC_16_31_11(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vst1.32 {tmp0}, [RWK]; + + +/********* Precalc macros for rounds 32-79 ************************************/ + +#define WPRECALC_32_79_0(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + veor W, W_m28; \ + +#define WPRECALC_32_79_1(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vext.8 tmp0, W_m08, W_m04, #8; \ + +#define WPRECALC_32_79_2(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + veor W, W_m16; \ + +#define WPRECALC_32_79_3(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + veor W, tmp0; \ + +#define WPRECALC_32_79_4(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + add RWK, sp, #(WK_offs(i&~3)); \ + +#define WPRECALC_32_79_5(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vshl.u32 tmp1, W, #2; \ + +#define WPRECALC_32_79_6(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vshr.u32 tmp0, W, #30; \ + +#define WPRECALC_32_79_7(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vorr W, tmp0, tmp1; \ + +#define WPRECALC_32_79_8(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vadd.u32 tmp0, W, curK; \ + +#define WPRECALC_32_79_9(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \ + vst1.32 {tmp0}, [RWK]; + + +/* + * Transform nblks*64 bytes (nblks*16 32-bit words) at DATA. + * + * unsigned int + * sha1_transform_neon (void *ctx, const unsigned char *data, + * unsigned int nblks) + */ +.align 3 +ENTRY(sha1_transform_neon) + /* input: + * r0: ctx, CTX + * r1: data (64*nblks bytes) + * r2: nblks + */ + + cmp RNBLKS, #0; + beq .Ldo_nothing; + + push {r4-r12, lr}; + /*vpush {q4-q7};*/ + + adr RT3, .LK_VEC; + + mov ROLDSTACK, sp; + + /* Align stack. */ + sub RT0, sp, #(16*4); + and RT0, #(~(16-1)); + mov sp, RT0; + + vld1.32 {qK1-qK2}, [RT3]!; /* Load K1,K2 */ + + /* Get the values of the chaining variables. */ + ldm RSTATE, {_a-_e}; + + vld1.32 {qK3-qK4}, [RT3]; /* Load K3,K4 */ + +#undef curK +#define curK qK1 + /* Precalc 0-15. */ + W_PRECALC_00_15(); + +.Loop: + /* Transform 0-15 + Precalc 16-31. */ + _R( _a, _b, _c, _d, _e, F1, 0, + WPRECALC_16_31_0, WPRECALC_16_31_1, WPRECALC_16_31_2, 16, + W4, W5, W6, W7, W0, _, _, _ ); + _R( _e, _a, _b, _c, _d, F1, 1, + WPRECALC_16_31_3, WPRECALC_16_31_4, WPRECALC_16_31_5, 16, + W4, W5, W6, W7, W0, _, _, _ ); + _R( _d, _e, _a, _b, _c, F1, 2, + WPRECALC_16_31_6, WPRECALC_16_31_7, WPRECALC_16_31_8, 16, + W4, W5, W6, W7, W0, _, _, _ ); + _R( _c, _d, _e, _a, _b, F1, 3, + WPRECALC_16_31_9, WPRECALC_16_31_10,WPRECALC_16_31_11,16, + W4, W5, W6, W7, W0, _, _, _ ); + +#undef curK +#define curK qK2 + _R( _b, _c, _d, _e, _a, F1, 4, + WPRECALC_16_31_0, WPRECALC_16_31_1, WPRECALC_16_31_2, 20, + W3, W4, W5, W6, W7, _, _, _ ); + _R( _a, _b, _c, _d, _e, F1, 5, + WPRECALC_16_31_3, WPRECALC_16_31_4, WPRECALC_16_31_5, 20, + W3, W4, W5, W6, W7, _, _, _ ); + _R( _e, _a, _b, _c, _d, F1, 6, + WPRECALC_16_31_6, WPRECALC_16_31_7, WPRECALC_16_31_8, 20, + W3, W4, W5, W6, W7, _, _, _ ); + _R( _d, _e, _a, _b, _c, F1, 7, + WPRECALC_16_31_9, WPRECALC_16_31_10,WPRECALC_16_31_11,20, + W3, W4, W5, W6, W7, _, _, _ ); + + _R( _c, _d, _e, _a, _b, F1, 8, + WPRECALC_16_31_0, WPRECALC_16_31_1, WPRECALC_16_31_2, 24, + W2, W3, W4, W5, W6, _, _, _ ); + _R( _b, _c, _d, _e, _a, F1, 9, + WPRECALC_16_31_3, WPRECALC_16_31_4, WPRECALC_16_31_5, 24, + W2, W3, W4, W5, W6, _, _, _ ); + _R( _a, _b, _c, _d, _e, F1, 10, + WPRECALC_16_31_6, WPRECALC_16_31_7, WPRECALC_16_31_8, 24, + W2, W3, W4, W5, W6, _, _, _ ); + _R( _e, _a, _b, _c, _d, F1, 11, + WPRECALC_16_31_9, WPRECALC_16_31_10,WPRECALC_16_31_11,24, + W2, W3, W4, W5, W6, _, _, _ ); + + _R( _d, _e, _a, _b, _c, F1, 12, + WPRECALC_16_31_0, WPRECALC_16_31_1, WPRECALC_16_31_2, 28, + W1, W2, W3, W4, W5, _, _, _ ); + _R( _c, _d, _e, _a, _b, F1, 13, + WPRECALC_16_31_3, WPRECALC_16_31_4, WPRECALC_16_31_5, 28, + W1, W2, W3, W4, W5, _, _, _ ); + _R( _b, _c, _d, _e, _a, F1, 14, + WPRECALC_16_31_6, WPRECALC_16_31_7, WPRECALC_16_31_8, 28, + W1, W2, W3, W4, W5, _, _, _ ); + _R( _a, _b, _c, _d, _e, F1, 15, + WPRECALC_16_31_9, WPRECALC_16_31_10,WPRECALC_16_31_11,28, + W1, W2, W3, W4, W5, _, _, _ ); + + /* Transform 16-63 + Precalc 32-79. */ + _R( _e, _a, _b, _c, _d, F1, 16, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 32, + W0, W1, W2, W3, W4, W5, W6, W7); + _R( _d, _e, _a, _b, _c, F1, 17, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 32, + W0, W1, W2, W3, W4, W5, W6, W7); + _R( _c, _d, _e, _a, _b, F1, 18, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 32, + W0, W1, W2, W3, W4, W5, W6, W7); + _R( _b, _c, _d, _e, _a, F1, 19, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 32, + W0, W1, W2, W3, W4, W5, W6, W7); + + _R( _a, _b, _c, _d, _e, F2, 20, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 36, + W7, W0, W1, W2, W3, W4, W5, W6); + _R( _e, _a, _b, _c, _d, F2, 21, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 36, + W7, W0, W1, W2, W3, W4, W5, W6); + _R( _d, _e, _a, _b, _c, F2, 22, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 36, + W7, W0, W1, W2, W3, W4, W5, W6); + _R( _c, _d, _e, _a, _b, F2, 23, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 36, + W7, W0, W1, W2, W3, W4, W5, W6); + +#undef curK +#define curK qK3 + _R( _b, _c, _d, _e, _a, F2, 24, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 40, + W6, W7, W0, W1, W2, W3, W4, W5); + _R( _a, _b, _c, _d, _e, F2, 25, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 40, + W6, W7, W0, W1, W2, W3, W4, W5); + _R( _e, _a, _b, _c, _d, F2, 26, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 40, + W6, W7, W0, W1, W2, W3, W4, W5); + _R( _d, _e, _a, _b, _c, F2, 27, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 40, + W6, W7, W0, W1, W2, W3, W4, W5); + + _R( _c, _d, _e, _a, _b, F2, 28, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 44, + W5, W6, W7, W0, W1, W2, W3, W4); + _R( _b, _c, _d, _e, _a, F2, 29, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 44, + W5, W6, W7, W0, W1, W2, W3, W4); + _R( _a, _b, _c, _d, _e, F2, 30, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 44, + W5, W6, W7, W0, W1, W2, W3, W4); + _R( _e, _a, _b, _c, _d, F2, 31, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 44, + W5, W6, W7, W0, W1, W2, W3, W4); + + _R( _d, _e, _a, _b, _c, F2, 32, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 48, + W4, W5, W6, W7, W0, W1, W2, W3); + _R( _c, _d, _e, _a, _b, F2, 33, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 48, + W4, W5, W6, W7, W0, W1, W2, W3); + _R( _b, _c, _d, _e, _a, F2, 34, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 48, + W4, W5, W6, W7, W0, W1, W2, W3); + _R( _a, _b, _c, _d, _e, F2, 35, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 48, + W4, W5, W6, W7, W0, W1, W2, W3); + + _R( _e, _a, _b, _c, _d, F2, 36, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 52, + W3, W4, W5, W6, W7, W0, W1, W2); + _R( _d, _e, _a, _b, _c, F2, 37, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 52, + W3, W4, W5, W6, W7, W0, W1, W2); + _R( _c, _d, _e, _a, _b, F2, 38, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 52, + W3, W4, W5, W6, W7, W0, W1, W2); + _R( _b, _c, _d, _e, _a, F2, 39, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 52, + W3, W4, W5, W6, W7, W0, W1, W2); + + _R( _a, _b, _c, _d, _e, F3, 40, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 56, + W2, W3, W4, W5, W6, W7, W0, W1); + _R( _e, _a, _b, _c, _d, F3, 41, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 56, + W2, W3, W4, W5, W6, W7, W0, W1); + _R( _d, _e, _a, _b, _c, F3, 42, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 56, + W2, W3, W4, W5, W6, W7, W0, W1); + _R( _c, _d, _e, _a, _b, F3, 43, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 56, + W2, W3, W4, W5, W6, W7, W0, W1); + +#undef curK +#define curK qK4 + _R( _b, _c, _d, _e, _a, F3, 44, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 60, + W1, W2, W3, W4, W5, W6, W7, W0); + _R( _a, _b, _c, _d, _e, F3, 45, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 60, + W1, W2, W3, W4, W5, W6, W7, W0); + _R( _e, _a, _b, _c, _d, F3, 46, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 60, + W1, W2, W3, W4, W5, W6, W7, W0); + _R( _d, _e, _a, _b, _c, F3, 47, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 60, + W1, W2, W3, W4, W5, W6, W7, W0); + + _R( _c, _d, _e, _a, _b, F3, 48, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 64, + W0, W1, W2, W3, W4, W5, W6, W7); + _R( _b, _c, _d, _e, _a, F3, 49, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 64, + W0, W1, W2, W3, W4, W5, W6, W7); + _R( _a, _b, _c, _d, _e, F3, 50, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 64, + W0, W1, W2, W3, W4, W5, W6, W7); + _R( _e, _a, _b, _c, _d, F3, 51, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 64, + W0, W1, W2, W3, W4, W5, W6, W7); + + _R( _d, _e, _a, _b, _c, F3, 52, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 68, + W7, W0, W1, W2, W3, W4, W5, W6); + _R( _c, _d, _e, _a, _b, F3, 53, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 68, + W7, W0, W1, W2, W3, W4, W5, W6); + _R( _b, _c, _d, _e, _a, F3, 54, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 68, + W7, W0, W1, W2, W3, W4, W5, W6); + _R( _a, _b, _c, _d, _e, F3, 55, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 68, + W7, W0, W1, W2, W3, W4, W5, W6); + + _R( _e, _a, _b, _c, _d, F3, 56, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 72, + W6, W7, W0, W1, W2, W3, W4, W5); + _R( _d, _e, _a, _b, _c, F3, 57, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 72, + W6, W7, W0, W1, W2, W3, W4, W5); + _R( _c, _d, _e, _a, _b, F3, 58, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 72, + W6, W7, W0, W1, W2, W3, W4, W5); + _R( _b, _c, _d, _e, _a, F3, 59, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 72, + W6, W7, W0, W1, W2, W3, W4, W5); + + subs RNBLKS, #1; + + _R( _a, _b, _c, _d, _e, F4, 60, + WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 76, + W5, W6, W7, W0, W1, W2, W3, W4); + _R( _e, _a, _b, _c, _d, F4, 61, + WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 76, + W5, W6, W7, W0, W1, W2, W3, W4); + _R( _d, _e, _a, _b, _c, F4, 62, + WPRECALC_32_79_6, dummy, WPRECALC_32_79_7, 76, + W5, W6, W7, W0, W1, W2, W3, W4); + _R( _c, _d, _e, _a, _b, F4, 63, + WPRECALC_32_79_8, dummy, WPRECALC_32_79_9, 76, + W5, W6, W7, W0, W1, W2, W3, W4); + + beq .Lend; + + /* Transform 64-79 + Precalc 0-15 of next block. */ +#undef curK +#define curK qK1 + _R( _b, _c, _d, _e, _a, F4, 64, + WPRECALC_00_15_0, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _a, _b, _c, _d, _e, F4, 65, + WPRECALC_00_15_1, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _e, _a, _b, _c, _d, F4, 66, + WPRECALC_00_15_2, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _d, _e, _a, _b, _c, F4, 67, + WPRECALC_00_15_3, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + + _R( _c, _d, _e, _a, _b, F4, 68, + dummy, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _b, _c, _d, _e, _a, F4, 69, + dummy, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _a, _b, _c, _d, _e, F4, 70, + WPRECALC_00_15_4, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _e, _a, _b, _c, _d, F4, 71, + WPRECALC_00_15_5, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + + _R( _d, _e, _a, _b, _c, F4, 72, + dummy, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _c, _d, _e, _a, _b, F4, 73, + dummy, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _b, _c, _d, _e, _a, F4, 74, + WPRECALC_00_15_6, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _a, _b, _c, _d, _e, F4, 75, + WPRECALC_00_15_7, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + + _R( _e, _a, _b, _c, _d, F4, 76, + WPRECALC_00_15_8, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _d, _e, _a, _b, _c, F4, 77, + WPRECALC_00_15_9, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _c, _d, _e, _a, _b, F4, 78, + WPRECALC_00_15_10, dummy, dummy, _, _, _, _, _, _, _, _, _ ); + _R( _b, _c, _d, _e, _a, F4, 79, + WPRECALC_00_15_11, dummy, WPRECALC_00_15_12, _, _, _, _, _, _, _, _, _ ); + + /* Update the chaining variables. */ + ldm RSTATE, {RT0-RT3}; + add _a, RT0; + ldr RT0, [RSTATE, #state_h4]; + add _b, RT1; + add _c, RT2; + add _d, RT3; + add _e, RT0; + stm RSTATE, {_a-_e}; + + b .Loop; + +.Lend: + /* Transform 64-79 */ + R( _b, _c, _d, _e, _a, F4, 64 ); + R( _a, _b, _c, _d, _e, F4, 65 ); + R( _e, _a, _b, _c, _d, F4, 66 ); + R( _d, _e, _a, _b, _c, F4, 67 ); + R( _c, _d, _e, _a, _b, F4, 68 ); + R( _b, _c, _d, _e, _a, F4, 69 ); + R( _a, _b, _c, _d, _e, F4, 70 ); + R( _e, _a, _b, _c, _d, F4, 71 ); + R( _d, _e, _a, _b, _c, F4, 72 ); + R( _c, _d, _e, _a, _b, F4, 73 ); + R( _b, _c, _d, _e, _a, F4, 74 ); + R( _a, _b, _c, _d, _e, F4, 75 ); + R( _e, _a, _b, _c, _d, F4, 76 ); + R( _d, _e, _a, _b, _c, F4, 77 ); + R( _c, _d, _e, _a, _b, F4, 78 ); + R( _b, _c, _d, _e, _a, F4, 79 ); + + mov sp, ROLDSTACK; + + /* Update the chaining variables. */ + ldm RSTATE, {RT0-RT3}; + add _a, RT0; + ldr RT0, [RSTATE, #state_h4]; + add _b, RT1; + add _c, RT2; + add _d, RT3; + /*vpop {q4-q7};*/ + add _e, RT0; + stm RSTATE, {_a-_e}; + + pop {r4-r12, pc}; + +.Ldo_nothing: + bx lr +ENDPROC(sha1_transform_neon) diff --git a/arch/arm/crypto/sha1_glue.c b/arch/arm/crypto/sha1_glue.c index c494e579ffc..84f2a756588 100644 --- a/arch/arm/crypto/sha1_glue.c +++ b/arch/arm/crypto/sha1_glue.c @@ -23,6 +23,7 @@ #include #include #include +#include asmlinkage void sha1_block_data_order(u32 *digest, @@ -65,8 +66,8 @@ static int __sha1_update(struct sha1_state *sctx, const u8 *data, } -static int sha1_update(struct shash_desc *desc, const u8 *data, - unsigned int len) +int sha1_update_arm(struct shash_desc *desc, const u8 *data, + unsigned int len) { struct sha1_state *sctx = shash_desc_ctx(desc); unsigned int partial = sctx->count % SHA1_BLOCK_SIZE; @@ -81,6 +82,7 @@ static int sha1_update(struct shash_desc *desc, const u8 *data, res = __sha1_update(sctx, data, len, partial); return res; } +EXPORT_SYMBOL_GPL(sha1_update_arm); /* Add padding and return the message digest. */ @@ -135,7 +137,7 @@ static int sha1_import(struct shash_desc *desc, const void *in) static struct shash_alg alg = { .digestsize = SHA1_DIGEST_SIZE, .init = sha1_init, - .update = sha1_update, + .update = sha1_update_arm, .final = sha1_final, .export = sha1_export, .import = sha1_import, diff --git a/arch/arm/crypto/sha1_neon_glue.c b/arch/arm/crypto/sha1_neon_glue.c new file mode 100644 index 00000000000..6f1b411b1d5 --- /dev/null +++ b/arch/arm/crypto/sha1_neon_glue.c @@ -0,0 +1,197 @@ +/* + * Glue code for the SHA1 Secure Hash Algorithm assembler implementation using + * ARM NEON instructions. + * + * Copyright © 2014 Jussi Kivilinna + * + * This file is based on sha1_generic.c and sha1_ssse3_glue.c: + * Copyright (c) Alan Smithee. + * Copyright (c) Andrew McDonald + * Copyright (c) Jean-Francois Dive + * Copyright (c) Mathias Krause + * Copyright (c) Chandramouli Narayanan + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +asmlinkage void sha1_transform_neon(void *state_h, const char *data, + unsigned int rounds); + + +static int sha1_neon_init(struct shash_desc *desc) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + + *sctx = (struct sha1_state){ + .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, + }; + + return 0; +} + +static int __sha1_neon_update(struct shash_desc *desc, const u8 *data, + unsigned int len, unsigned int partial) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + unsigned int done = 0; + + sctx->count += len; + + if (partial) { + done = SHA1_BLOCK_SIZE - partial; + memcpy(sctx->buffer + partial, data, done); + sha1_transform_neon(sctx->state, sctx->buffer, 1); + } + + if (len - done >= SHA1_BLOCK_SIZE) { + const unsigned int rounds = (len - done) / SHA1_BLOCK_SIZE; + + sha1_transform_neon(sctx->state, data + done, rounds); + done += rounds * SHA1_BLOCK_SIZE; + } + + memcpy(sctx->buffer, data + done, len - done); + + return 0; +} + +static int sha1_neon_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + unsigned int partial = sctx->count % SHA1_BLOCK_SIZE; + int res; + + /* Handle the fast case right here */ + if (partial + len < SHA1_BLOCK_SIZE) { + sctx->count += len; + memcpy(sctx->buffer + partial, data, len); + + return 0; + } + + if (!may_use_simd()) { + res = sha1_update_arm(desc, data, len); + } else { + kernel_neon_begin(); + res = __sha1_neon_update(desc, data, len, partial); + kernel_neon_end(); + } + + return res; +} + + +/* Add padding and return the message digest. */ +static int sha1_neon_final(struct shash_desc *desc, u8 *out) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + unsigned int i, index, padlen; + __be32 *dst = (__be32 *)out; + __be64 bits; + static const u8 padding[SHA1_BLOCK_SIZE] = { 0x80, }; + + bits = cpu_to_be64(sctx->count << 3); + + /* Pad out to 56 mod 64 and append length */ + index = sctx->count % SHA1_BLOCK_SIZE; + padlen = (index < 56) ? (56 - index) : ((SHA1_BLOCK_SIZE+56) - index); + if (!may_use_simd()) { + sha1_update_arm(desc, padding, padlen); + sha1_update_arm(desc, (const u8 *)&bits, sizeof(bits)); + } else { + kernel_neon_begin(); + /* We need to fill a whole block for __sha1_neon_update() */ + if (padlen <= 56) { + sctx->count += padlen; + memcpy(sctx->buffer + index, padding, padlen); + } else { + __sha1_neon_update(desc, padding, padlen, index); + } + __sha1_neon_update(desc, (const u8 *)&bits, sizeof(bits), 56); + kernel_neon_end(); + } + + /* Store state in digest */ + for (i = 0; i < 5; i++) + dst[i] = cpu_to_be32(sctx->state[i]); + + /* Wipe context */ + memset(sctx, 0, sizeof(*sctx)); + + return 0; +} + +static int sha1_neon_export(struct shash_desc *desc, void *out) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + + memcpy(out, sctx, sizeof(*sctx)); + + return 0; +} + +static int sha1_neon_import(struct shash_desc *desc, const void *in) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + + memcpy(sctx, in, sizeof(*sctx)); + + return 0; +} + +static struct shash_alg alg = { + .digestsize = SHA1_DIGEST_SIZE, + .init = sha1_neon_init, + .update = sha1_neon_update, + .final = sha1_neon_final, + .export = sha1_neon_export, + .import = sha1_neon_import, + .descsize = sizeof(struct sha1_state), + .statesize = sizeof(struct sha1_state), + .base = { + .cra_name = "sha1", + .cra_driver_name = "sha1-neon", + .cra_priority = 250, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + +static int __init sha1_neon_mod_init(void) +{ + if (!cpu_has_neon()) + return -ENODEV; + + return crypto_register_shash(&alg); +} + +static void __exit sha1_neon_mod_fini(void) +{ + crypto_unregister_shash(&alg); +} + +module_init(sha1_neon_mod_init); +module_exit(sha1_neon_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, NEON accelerated"); +MODULE_ALIAS("sha1"); diff --git a/arch/arm/include/asm/crypto/sha1.h b/arch/arm/include/asm/crypto/sha1.h new file mode 100644 index 00000000000..75e6a417416 --- /dev/null +++ b/arch/arm/include/asm/crypto/sha1.h @@ -0,0 +1,10 @@ +#ifndef ASM_ARM_CRYPTO_SHA1_H +#define ASM_ARM_CRYPTO_SHA1_H + +#include +#include + +extern int sha1_update_arm(struct shash_desc *desc, const u8 *data, + unsigned int len); + +#endif diff --git a/crypto/Kconfig b/crypto/Kconfig index f7f9204f7ea..6da0c6515f6 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -432,6 +432,17 @@ config CRYPTO_SHA1_ARM SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented using optimized ARM assembler. +config CRYPTO_SHA1_ARM_NEON + tristate "SHA1 digest algorithm (ARM NEON)" + depends on ARM && KERNEL_MODE_NEON && !CPU_BIG_ENDIAN + select CRYPTO_SHA1_ARM + select CRYPTO_SHA1 + select CRYPTO_HASH + help + SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented + using optimized ARM NEON assembly, when NEON instructions are + available. + config CRYPTO_SHA256 tristate "SHA224 and SHA256 digest algorithm" select CRYPTO_HASH From 3ce262027dd21bd1319d02f9e2c33183f51f9222 Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Tue, 29 Jul 2014 17:15:24 +0100 Subject: [PATCH 060/168] ARM: 8120/1: crypto: sha512: add ARM NEON implementation This patch adds ARM NEON assembly implementation of SHA-512 and SHA-384 algorithms. tcrypt benchmark results on Cortex-A8, sha512-generic vs sha512-neon-asm: block-size bytes/update old-vs-new 16 16 2.99x 64 16 2.67x 64 64 3.00x 256 16 2.64x 256 64 3.06x 256 256 3.33x 1024 16 2.53x 1024 256 3.39x 1024 1024 3.52x 2048 16 2.50x 2048 256 3.41x 2048 1024 3.54x 2048 2048 3.57x 4096 16 2.49x 4096 256 3.42x 4096 1024 3.56x 4096 4096 3.59x 8192 16 2.48x 8192 256 3.42x 8192 1024 3.56x 8192 4096 3.60x 8192 8192 3.60x Change-Id: I8471b7baf6ca8973759c0d0cf993ba772b3edcb5 Acked-by: Ard Biesheuvel Tested-by: Ard Biesheuvel Signed-off-by: Jussi Kivilinna Signed-off-by: Russell King Git-commit: a2e5f7fdc58387bdc6a626c8c762c0df478bc071 Git-repo: https://android.googlesource.com/kernel/common/ Signed-off-by: Samir Mehta --- arch/arm/crypto/Makefile | 2 + arch/arm/crypto/sha512-armv7-neon.S | 455 ++++++++++++++++++++++++++++ arch/arm/crypto/sha512_neon_glue.c | 305 +++++++++++++++++++ crypto/Kconfig | 15 + 4 files changed, 777 insertions(+) create mode 100644 arch/arm/crypto/sha512-armv7-neon.S create mode 100644 arch/arm/crypto/sha512_neon_glue.c diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile index 374956d2f89..b48fa341648 100644 --- a/arch/arm/crypto/Makefile +++ b/arch/arm/crypto/Makefile @@ -6,11 +6,13 @@ obj-$(CONFIG_CRYPTO_AES_ARM) += aes-arm.o obj-$(CONFIG_CRYPTO_AES_ARM_BS) += aes-arm-bs.o obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o obj-$(CONFIG_CRYPTO_SHA1_ARM_NEON) += sha1-arm-neon.o +obj-$(CONFIG_CRYPTO_SHA512_ARM_NEON) += sha512-arm-neon.o aes-arm-y := aes-armv4.o aes_glue.o aes-arm-bs-y := aesbs-core.o aesbs-glue.o sha1-arm-y := sha1-armv4-large.o sha1_glue.o sha1-arm-neon-y := sha1-armv7-neon.o sha1_neon_glue.o +sha512-arm-neon-y := sha512-armv7-neon.o sha512_neon_glue.o quiet_cmd_perl = PERL $@ cmd_perl = $(PERL) $(<) > $(@) diff --git a/arch/arm/crypto/sha512-armv7-neon.S b/arch/arm/crypto/sha512-armv7-neon.S new file mode 100644 index 00000000000..fe99472e507 --- /dev/null +++ b/arch/arm/crypto/sha512-armv7-neon.S @@ -0,0 +1,455 @@ +/* sha512-armv7-neon.S - ARM/NEON assembly implementation of SHA-512 transform + * + * Copyright © 2013-2014 Jussi Kivilinna + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + */ + +#include + + +.syntax unified +.code 32 +.fpu neon + +.text + +/* structure of SHA512_CONTEXT */ +#define hd_a 0 +#define hd_b ((hd_a) + 8) +#define hd_c ((hd_b) + 8) +#define hd_d ((hd_c) + 8) +#define hd_e ((hd_d) + 8) +#define hd_f ((hd_e) + 8) +#define hd_g ((hd_f) + 8) + +/* register macros */ +#define RK %r2 + +#define RA d0 +#define RB d1 +#define RC d2 +#define RD d3 +#define RE d4 +#define RF d5 +#define RG d6 +#define RH d7 + +#define RT0 d8 +#define RT1 d9 +#define RT2 d10 +#define RT3 d11 +#define RT4 d12 +#define RT5 d13 +#define RT6 d14 +#define RT7 d15 + +#define RT01q q4 +#define RT23q q5 +#define RT45q q6 +#define RT67q q7 + +#define RW0 d16 +#define RW1 d17 +#define RW2 d18 +#define RW3 d19 +#define RW4 d20 +#define RW5 d21 +#define RW6 d22 +#define RW7 d23 +#define RW8 d24 +#define RW9 d25 +#define RW10 d26 +#define RW11 d27 +#define RW12 d28 +#define RW13 d29 +#define RW14 d30 +#define RW15 d31 + +#define RW01q q8 +#define RW23q q9 +#define RW45q q10 +#define RW67q q11 +#define RW89q q12 +#define RW1011q q13 +#define RW1213q q14 +#define RW1415q q15 + +/*********************************************************************** + * ARM assembly implementation of sha512 transform + ***********************************************************************/ +#define rounds2_0_63(ra, rb, rc, rd, re, rf, rg, rh, rw0, rw1, rw01q, rw2, \ + rw23q, rw1415q, rw9, rw10, interleave_op, arg1) \ + /* t1 = h + Sum1 (e) + Ch (e, f, g) + k[t] + w[t]; */ \ + vshr.u64 RT2, re, #14; \ + vshl.u64 RT3, re, #64 - 14; \ + interleave_op(arg1); \ + vshr.u64 RT4, re, #18; \ + vshl.u64 RT5, re, #64 - 18; \ + vld1.64 {RT0}, [RK]!; \ + veor.64 RT23q, RT23q, RT45q; \ + vshr.u64 RT4, re, #41; \ + vshl.u64 RT5, re, #64 - 41; \ + vadd.u64 RT0, RT0, rw0; \ + veor.64 RT23q, RT23q, RT45q; \ + vmov.64 RT7, re; \ + veor.64 RT1, RT2, RT3; \ + vbsl.64 RT7, rf, rg; \ + \ + vadd.u64 RT1, RT1, rh; \ + vshr.u64 RT2, ra, #28; \ + vshl.u64 RT3, ra, #64 - 28; \ + vadd.u64 RT1, RT1, RT0; \ + vshr.u64 RT4, ra, #34; \ + vshl.u64 RT5, ra, #64 - 34; \ + vadd.u64 RT1, RT1, RT7; \ + \ + /* h = Sum0 (a) + Maj (a, b, c); */ \ + veor.64 RT23q, RT23q, RT45q; \ + vshr.u64 RT4, ra, #39; \ + vshl.u64 RT5, ra, #64 - 39; \ + veor.64 RT0, ra, rb; \ + veor.64 RT23q, RT23q, RT45q; \ + vbsl.64 RT0, rc, rb; \ + vadd.u64 rd, rd, RT1; /* d+=t1; */ \ + veor.64 rh, RT2, RT3; \ + \ + /* t1 = g + Sum1 (d) + Ch (d, e, f) + k[t] + w[t]; */ \ + vshr.u64 RT2, rd, #14; \ + vshl.u64 RT3, rd, #64 - 14; \ + vadd.u64 rh, rh, RT0; \ + vshr.u64 RT4, rd, #18; \ + vshl.u64 RT5, rd, #64 - 18; \ + vadd.u64 rh, rh, RT1; /* h+=t1; */ \ + vld1.64 {RT0}, [RK]!; \ + veor.64 RT23q, RT23q, RT45q; \ + vshr.u64 RT4, rd, #41; \ + vshl.u64 RT5, rd, #64 - 41; \ + vadd.u64 RT0, RT0, rw1; \ + veor.64 RT23q, RT23q, RT45q; \ + vmov.64 RT7, rd; \ + veor.64 RT1, RT2, RT3; \ + vbsl.64 RT7, re, rf; \ + \ + vadd.u64 RT1, RT1, rg; \ + vshr.u64 RT2, rh, #28; \ + vshl.u64 RT3, rh, #64 - 28; \ + vadd.u64 RT1, RT1, RT0; \ + vshr.u64 RT4, rh, #34; \ + vshl.u64 RT5, rh, #64 - 34; \ + vadd.u64 RT1, RT1, RT7; \ + \ + /* g = Sum0 (h) + Maj (h, a, b); */ \ + veor.64 RT23q, RT23q, RT45q; \ + vshr.u64 RT4, rh, #39; \ + vshl.u64 RT5, rh, #64 - 39; \ + veor.64 RT0, rh, ra; \ + veor.64 RT23q, RT23q, RT45q; \ + vbsl.64 RT0, rb, ra; \ + vadd.u64 rc, rc, RT1; /* c+=t1; */ \ + veor.64 rg, RT2, RT3; \ + \ + /* w[0] += S1 (w[14]) + w[9] + S0 (w[1]); */ \ + /* w[1] += S1 (w[15]) + w[10] + S0 (w[2]); */ \ + \ + /**** S0(w[1:2]) */ \ + \ + /* w[0:1] += w[9:10] */ \ + /* RT23q = rw1:rw2 */ \ + vext.u64 RT23q, rw01q, rw23q, #1; \ + vadd.u64 rw0, rw9; \ + vadd.u64 rg, rg, RT0; \ + vadd.u64 rw1, rw10;\ + vadd.u64 rg, rg, RT1; /* g+=t1; */ \ + \ + vshr.u64 RT45q, RT23q, #1; \ + vshl.u64 RT67q, RT23q, #64 - 1; \ + vshr.u64 RT01q, RT23q, #8; \ + veor.u64 RT45q, RT45q, RT67q; \ + vshl.u64 RT67q, RT23q, #64 - 8; \ + veor.u64 RT45q, RT45q, RT01q; \ + vshr.u64 RT01q, RT23q, #7; \ + veor.u64 RT45q, RT45q, RT67q; \ + \ + /**** S1(w[14:15]) */ \ + vshr.u64 RT23q, rw1415q, #6; \ + veor.u64 RT01q, RT01q, RT45q; \ + vshr.u64 RT45q, rw1415q, #19; \ + vshl.u64 RT67q, rw1415q, #64 - 19; \ + veor.u64 RT23q, RT23q, RT45q; \ + vshr.u64 RT45q, rw1415q, #61; \ + veor.u64 RT23q, RT23q, RT67q; \ + vshl.u64 RT67q, rw1415q, #64 - 61; \ + veor.u64 RT23q, RT23q, RT45q; \ + vadd.u64 rw01q, RT01q; /* w[0:1] += S(w[1:2]) */ \ + veor.u64 RT01q, RT23q, RT67q; +#define vadd_RT01q(rw01q) \ + /* w[0:1] += S(w[14:15]) */ \ + vadd.u64 rw01q, RT01q; + +#define dummy(_) /*_*/ + +#define rounds2_64_79(ra, rb, rc, rd, re, rf, rg, rh, rw0, rw1, \ + interleave_op1, arg1, interleave_op2, arg2) \ + /* t1 = h + Sum1 (e) + Ch (e, f, g) + k[t] + w[t]; */ \ + vshr.u64 RT2, re, #14; \ + vshl.u64 RT3, re, #64 - 14; \ + interleave_op1(arg1); \ + vshr.u64 RT4, re, #18; \ + vshl.u64 RT5, re, #64 - 18; \ + interleave_op2(arg2); \ + vld1.64 {RT0}, [RK]!; \ + veor.64 RT23q, RT23q, RT45q; \ + vshr.u64 RT4, re, #41; \ + vshl.u64 RT5, re, #64 - 41; \ + vadd.u64 RT0, RT0, rw0; \ + veor.64 RT23q, RT23q, RT45q; \ + vmov.64 RT7, re; \ + veor.64 RT1, RT2, RT3; \ + vbsl.64 RT7, rf, rg; \ + \ + vadd.u64 RT1, RT1, rh; \ + vshr.u64 RT2, ra, #28; \ + vshl.u64 RT3, ra, #64 - 28; \ + vadd.u64 RT1, RT1, RT0; \ + vshr.u64 RT4, ra, #34; \ + vshl.u64 RT5, ra, #64 - 34; \ + vadd.u64 RT1, RT1, RT7; \ + \ + /* h = Sum0 (a) + Maj (a, b, c); */ \ + veor.64 RT23q, RT23q, RT45q; \ + vshr.u64 RT4, ra, #39; \ + vshl.u64 RT5, ra, #64 - 39; \ + veor.64 RT0, ra, rb; \ + veor.64 RT23q, RT23q, RT45q; \ + vbsl.64 RT0, rc, rb; \ + vadd.u64 rd, rd, RT1; /* d+=t1; */ \ + veor.64 rh, RT2, RT3; \ + \ + /* t1 = g + Sum1 (d) + Ch (d, e, f) + k[t] + w[t]; */ \ + vshr.u64 RT2, rd, #14; \ + vshl.u64 RT3, rd, #64 - 14; \ + vadd.u64 rh, rh, RT0; \ + vshr.u64 RT4, rd, #18; \ + vshl.u64 RT5, rd, #64 - 18; \ + vadd.u64 rh, rh, RT1; /* h+=t1; */ \ + vld1.64 {RT0}, [RK]!; \ + veor.64 RT23q, RT23q, RT45q; \ + vshr.u64 RT4, rd, #41; \ + vshl.u64 RT5, rd, #64 - 41; \ + vadd.u64 RT0, RT0, rw1; \ + veor.64 RT23q, RT23q, RT45q; \ + vmov.64 RT7, rd; \ + veor.64 RT1, RT2, RT3; \ + vbsl.64 RT7, re, rf; \ + \ + vadd.u64 RT1, RT1, rg; \ + vshr.u64 RT2, rh, #28; \ + vshl.u64 RT3, rh, #64 - 28; \ + vadd.u64 RT1, RT1, RT0; \ + vshr.u64 RT4, rh, #34; \ + vshl.u64 RT5, rh, #64 - 34; \ + vadd.u64 RT1, RT1, RT7; \ + \ + /* g = Sum0 (h) + Maj (h, a, b); */ \ + veor.64 RT23q, RT23q, RT45q; \ + vshr.u64 RT4, rh, #39; \ + vshl.u64 RT5, rh, #64 - 39; \ + veor.64 RT0, rh, ra; \ + veor.64 RT23q, RT23q, RT45q; \ + vbsl.64 RT0, rb, ra; \ + vadd.u64 rc, rc, RT1; /* c+=t1; */ \ + veor.64 rg, RT2, RT3; +#define vadd_rg_RT0(rg) \ + vadd.u64 rg, rg, RT0; +#define vadd_rg_RT1(rg) \ + vadd.u64 rg, rg, RT1; /* g+=t1; */ + +.align 3 +ENTRY(sha512_transform_neon) + /* Input: + * %r0: SHA512_CONTEXT + * %r1: data + * %r2: u64 k[] constants + * %r3: nblks + */ + push {%lr}; + + mov %lr, #0; + + /* Load context to d0-d7 */ + vld1.64 {RA-RD}, [%r0]!; + vld1.64 {RE-RH}, [%r0]; + sub %r0, #(4*8); + + /* Load input to w[16], d16-d31 */ + /* NOTE: Assumes that on ARMv7 unaligned accesses are always allowed. */ + vld1.64 {RW0-RW3}, [%r1]!; + vld1.64 {RW4-RW7}, [%r1]!; + vld1.64 {RW8-RW11}, [%r1]!; + vld1.64 {RW12-RW15}, [%r1]!; +#ifdef __ARMEL__ + /* byteswap */ + vrev64.8 RW01q, RW01q; + vrev64.8 RW23q, RW23q; + vrev64.8 RW45q, RW45q; + vrev64.8 RW67q, RW67q; + vrev64.8 RW89q, RW89q; + vrev64.8 RW1011q, RW1011q; + vrev64.8 RW1213q, RW1213q; + vrev64.8 RW1415q, RW1415q; +#endif + + /* EABI says that d8-d15 must be preserved by callee. */ + /*vpush {RT0-RT7};*/ + +.Loop: + rounds2_0_63(RA, RB, RC, RD, RE, RF, RG, RH, RW0, RW1, RW01q, RW2, + RW23q, RW1415q, RW9, RW10, dummy, _); + b .Lenter_rounds; + +.Loop_rounds: + rounds2_0_63(RA, RB, RC, RD, RE, RF, RG, RH, RW0, RW1, RW01q, RW2, + RW23q, RW1415q, RW9, RW10, vadd_RT01q, RW1415q); +.Lenter_rounds: + rounds2_0_63(RG, RH, RA, RB, RC, RD, RE, RF, RW2, RW3, RW23q, RW4, + RW45q, RW01q, RW11, RW12, vadd_RT01q, RW01q); + rounds2_0_63(RE, RF, RG, RH, RA, RB, RC, RD, RW4, RW5, RW45q, RW6, + RW67q, RW23q, RW13, RW14, vadd_RT01q, RW23q); + rounds2_0_63(RC, RD, RE, RF, RG, RH, RA, RB, RW6, RW7, RW67q, RW8, + RW89q, RW45q, RW15, RW0, vadd_RT01q, RW45q); + rounds2_0_63(RA, RB, RC, RD, RE, RF, RG, RH, RW8, RW9, RW89q, RW10, + RW1011q, RW67q, RW1, RW2, vadd_RT01q, RW67q); + rounds2_0_63(RG, RH, RA, RB, RC, RD, RE, RF, RW10, RW11, RW1011q, RW12, + RW1213q, RW89q, RW3, RW4, vadd_RT01q, RW89q); + add %lr, #16; + rounds2_0_63(RE, RF, RG, RH, RA, RB, RC, RD, RW12, RW13, RW1213q, RW14, + RW1415q, RW1011q, RW5, RW6, vadd_RT01q, RW1011q); + cmp %lr, #64; + rounds2_0_63(RC, RD, RE, RF, RG, RH, RA, RB, RW14, RW15, RW1415q, RW0, + RW01q, RW1213q, RW7, RW8, vadd_RT01q, RW1213q); + bne .Loop_rounds; + + subs %r3, #1; + + rounds2_64_79(RA, RB, RC, RD, RE, RF, RG, RH, RW0, RW1, + vadd_RT01q, RW1415q, dummy, _); + rounds2_64_79(RG, RH, RA, RB, RC, RD, RE, RF, RW2, RW3, + vadd_rg_RT0, RG, vadd_rg_RT1, RG); + beq .Lhandle_tail; + vld1.64 {RW0-RW3}, [%r1]!; + rounds2_64_79(RE, RF, RG, RH, RA, RB, RC, RD, RW4, RW5, + vadd_rg_RT0, RE, vadd_rg_RT1, RE); + rounds2_64_79(RC, RD, RE, RF, RG, RH, RA, RB, RW6, RW7, + vadd_rg_RT0, RC, vadd_rg_RT1, RC); +#ifdef __ARMEL__ + vrev64.8 RW01q, RW01q; + vrev64.8 RW23q, RW23q; +#endif + vld1.64 {RW4-RW7}, [%r1]!; + rounds2_64_79(RA, RB, RC, RD, RE, RF, RG, RH, RW8, RW9, + vadd_rg_RT0, RA, vadd_rg_RT1, RA); + rounds2_64_79(RG, RH, RA, RB, RC, RD, RE, RF, RW10, RW11, + vadd_rg_RT0, RG, vadd_rg_RT1, RG); +#ifdef __ARMEL__ + vrev64.8 RW45q, RW45q; + vrev64.8 RW67q, RW67q; +#endif + vld1.64 {RW8-RW11}, [%r1]!; + rounds2_64_79(RE, RF, RG, RH, RA, RB, RC, RD, RW12, RW13, + vadd_rg_RT0, RE, vadd_rg_RT1, RE); + rounds2_64_79(RC, RD, RE, RF, RG, RH, RA, RB, RW14, RW15, + vadd_rg_RT0, RC, vadd_rg_RT1, RC); +#ifdef __ARMEL__ + vrev64.8 RW89q, RW89q; + vrev64.8 RW1011q, RW1011q; +#endif + vld1.64 {RW12-RW15}, [%r1]!; + vadd_rg_RT0(RA); + vadd_rg_RT1(RA); + + /* Load context */ + vld1.64 {RT0-RT3}, [%r0]!; + vld1.64 {RT4-RT7}, [%r0]; + sub %r0, #(4*8); + +#ifdef __ARMEL__ + vrev64.8 RW1213q, RW1213q; + vrev64.8 RW1415q, RW1415q; +#endif + + vadd.u64 RA, RT0; + vadd.u64 RB, RT1; + vadd.u64 RC, RT2; + vadd.u64 RD, RT3; + vadd.u64 RE, RT4; + vadd.u64 RF, RT5; + vadd.u64 RG, RT6; + vadd.u64 RH, RT7; + + /* Store the first half of context */ + vst1.64 {RA-RD}, [%r0]!; + sub RK, $(8*80); + vst1.64 {RE-RH}, [%r0]; /* Store the last half of context */ + mov %lr, #0; + sub %r0, #(4*8); + + b .Loop; + +.Lhandle_tail: + rounds2_64_79(RE, RF, RG, RH, RA, RB, RC, RD, RW4, RW5, + vadd_rg_RT0, RE, vadd_rg_RT1, RE); + rounds2_64_79(RC, RD, RE, RF, RG, RH, RA, RB, RW6, RW7, + vadd_rg_RT0, RC, vadd_rg_RT1, RC); + rounds2_64_79(RA, RB, RC, RD, RE, RF, RG, RH, RW8, RW9, + vadd_rg_RT0, RA, vadd_rg_RT1, RA); + rounds2_64_79(RG, RH, RA, RB, RC, RD, RE, RF, RW10, RW11, + vadd_rg_RT0, RG, vadd_rg_RT1, RG); + rounds2_64_79(RE, RF, RG, RH, RA, RB, RC, RD, RW12, RW13, + vadd_rg_RT0, RE, vadd_rg_RT1, RE); + rounds2_64_79(RC, RD, RE, RF, RG, RH, RA, RB, RW14, RW15, + vadd_rg_RT0, RC, vadd_rg_RT1, RC); + + /* Load context to d16-d23 */ + vld1.64 {RW0-RW3}, [%r0]!; + vadd_rg_RT0(RA); + vld1.64 {RW4-RW7}, [%r0]; + vadd_rg_RT1(RA); + sub %r0, #(4*8); + + vadd.u64 RA, RW0; + vadd.u64 RB, RW1; + vadd.u64 RC, RW2; + vadd.u64 RD, RW3; + vadd.u64 RE, RW4; + vadd.u64 RF, RW5; + vadd.u64 RG, RW6; + vadd.u64 RH, RW7; + + /* Store the first half of context */ + vst1.64 {RA-RD}, [%r0]!; + + /* Clear used registers */ + /* d16-d31 */ + veor.u64 RW01q, RW01q; + veor.u64 RW23q, RW23q; + veor.u64 RW45q, RW45q; + veor.u64 RW67q, RW67q; + vst1.64 {RE-RH}, [%r0]; /* Store the last half of context */ + veor.u64 RW89q, RW89q; + veor.u64 RW1011q, RW1011q; + veor.u64 RW1213q, RW1213q; + veor.u64 RW1415q, RW1415q; + /* d8-d15 */ + /*vpop {RT0-RT7};*/ + /* d0-d7 (q0-q3) */ + veor.u64 %q0, %q0; + veor.u64 %q1, %q1; + veor.u64 %q2, %q2; + veor.u64 %q3, %q3; + + pop {%pc}; +ENDPROC(sha512_transform_neon) diff --git a/arch/arm/crypto/sha512_neon_glue.c b/arch/arm/crypto/sha512_neon_glue.c new file mode 100644 index 00000000000..0d2758ff5e1 --- /dev/null +++ b/arch/arm/crypto/sha512_neon_glue.c @@ -0,0 +1,305 @@ +/* + * Glue code for the SHA512 Secure Hash Algorithm assembly implementation + * using NEON instructions. + * + * Copyright © 2014 Jussi Kivilinna + * + * This file is based on sha512_ssse3_glue.c: + * Copyright (C) 2013 Intel Corporation + * Author: Tim Chen + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +static const u64 sha512_k[] = { + 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, + 0xb5c0fbcfec4d3b2fULL, 0xe9b5dba58189dbbcULL, + 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL, + 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, + 0xd807aa98a3030242ULL, 0x12835b0145706fbeULL, + 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL, + 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, + 0x9bdc06a725c71235ULL, 0xc19bf174cf692694ULL, + 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL, + 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, + 0x2de92c6f592b0275ULL, 0x4a7484aa6ea6e483ULL, + 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL, + 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, + 0xb00327c898fb213fULL, 0xbf597fc7beef0ee4ULL, + 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL, + 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, + 0x27b70a8546d22ffcULL, 0x2e1b21385c26c926ULL, + 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL, + 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, + 0x81c2c92e47edaee6ULL, 0x92722c851482353bULL, + 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL, + 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, + 0xd192e819d6ef5218ULL, 0xd69906245565a910ULL, + 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL, + 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, + 0x2748774cdf8eeb99ULL, 0x34b0bcb5e19b48a8ULL, + 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL, + 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, + 0x748f82ee5defb2fcULL, 0x78a5636f43172f60ULL, + 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL, + 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, + 0xbef9a3f7b2c67915ULL, 0xc67178f2e372532bULL, + 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL, + 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, + 0x06f067aa72176fbaULL, 0x0a637dc5a2c898a6ULL, + 0x113f9804bef90daeULL, 0x1b710b35131c471bULL, + 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, + 0x3c9ebe0a15c9bebcULL, 0x431d67c49c100d4cULL, + 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, + 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL +}; + + +asmlinkage void sha512_transform_neon(u64 *digest, const void *data, + const u64 k[], unsigned int num_blks); + + +static int sha512_neon_init(struct shash_desc *desc) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + + sctx->state[0] = SHA512_H0; + sctx->state[1] = SHA512_H1; + sctx->state[2] = SHA512_H2; + sctx->state[3] = SHA512_H3; + sctx->state[4] = SHA512_H4; + sctx->state[5] = SHA512_H5; + sctx->state[6] = SHA512_H6; + sctx->state[7] = SHA512_H7; + sctx->count[0] = sctx->count[1] = 0; + + return 0; +} + +static int __sha512_neon_update(struct shash_desc *desc, const u8 *data, + unsigned int len, unsigned int partial) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + unsigned int done = 0; + + sctx->count[0] += len; + if (sctx->count[0] < len) + sctx->count[1]++; + + if (partial) { + done = SHA512_BLOCK_SIZE - partial; + memcpy(sctx->buf + partial, data, done); + sha512_transform_neon(sctx->state, sctx->buf, sha512_k, 1); + } + + if (len - done >= SHA512_BLOCK_SIZE) { + const unsigned int rounds = (len - done) / SHA512_BLOCK_SIZE; + + sha512_transform_neon(sctx->state, data + done, sha512_k, + rounds); + + done += rounds * SHA512_BLOCK_SIZE; + } + + memcpy(sctx->buf, data + done, len - done); + + return 0; +} + +static int sha512_neon_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + unsigned int partial = sctx->count[0] % SHA512_BLOCK_SIZE; + int res; + + /* Handle the fast case right here */ + if (partial + len < SHA512_BLOCK_SIZE) { + sctx->count[0] += len; + if (sctx->count[0] < len) + sctx->count[1]++; + memcpy(sctx->buf + partial, data, len); + + return 0; + } + + if (!may_use_simd()) { + res = crypto_sha512_update(desc, data, len); + } else { + kernel_neon_begin(); + res = __sha512_neon_update(desc, data, len, partial); + kernel_neon_end(); + } + + return res; +} + + +/* Add padding and return the message digest. */ +static int sha512_neon_final(struct shash_desc *desc, u8 *out) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + unsigned int i, index, padlen; + __be64 *dst = (__be64 *)out; + __be64 bits[2]; + static const u8 padding[SHA512_BLOCK_SIZE] = { 0x80, }; + + /* save number of bits */ + bits[1] = cpu_to_be64(sctx->count[0] << 3); + bits[0] = cpu_to_be64(sctx->count[1] << 3 | sctx->count[0] >> 61); + + /* Pad out to 112 mod 128 and append length */ + index = sctx->count[0] & 0x7f; + padlen = (index < 112) ? (112 - index) : ((128+112) - index); + + if (!may_use_simd()) { + crypto_sha512_update(desc, padding, padlen); + crypto_sha512_update(desc, (const u8 *)&bits, sizeof(bits)); + } else { + kernel_neon_begin(); + /* We need to fill a whole block for __sha512_neon_update() */ + if (padlen <= 112) { + sctx->count[0] += padlen; + if (sctx->count[0] < padlen) + sctx->count[1]++; + memcpy(sctx->buf + index, padding, padlen); + } else { + __sha512_neon_update(desc, padding, padlen, index); + } + __sha512_neon_update(desc, (const u8 *)&bits, + sizeof(bits), 112); + kernel_neon_end(); + } + + /* Store state in digest */ + for (i = 0; i < 8; i++) + dst[i] = cpu_to_be64(sctx->state[i]); + + /* Wipe context */ + memset(sctx, 0, sizeof(*sctx)); + + return 0; +} + +static int sha512_neon_export(struct shash_desc *desc, void *out) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + + memcpy(out, sctx, sizeof(*sctx)); + + return 0; +} + +static int sha512_neon_import(struct shash_desc *desc, const void *in) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + + memcpy(sctx, in, sizeof(*sctx)); + + return 0; +} + +static int sha384_neon_init(struct shash_desc *desc) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + + sctx->state[0] = SHA384_H0; + sctx->state[1] = SHA384_H1; + sctx->state[2] = SHA384_H2; + sctx->state[3] = SHA384_H3; + sctx->state[4] = SHA384_H4; + sctx->state[5] = SHA384_H5; + sctx->state[6] = SHA384_H6; + sctx->state[7] = SHA384_H7; + + sctx->count[0] = sctx->count[1] = 0; + + return 0; +} + +static int sha384_neon_final(struct shash_desc *desc, u8 *hash) +{ + u8 D[SHA512_DIGEST_SIZE]; + + sha512_neon_final(desc, D); + + memcpy(hash, D, SHA384_DIGEST_SIZE); + memset(D, 0, SHA512_DIGEST_SIZE); + + return 0; +} + +static struct shash_alg algs[] = { { + .digestsize = SHA512_DIGEST_SIZE, + .init = sha512_neon_init, + .update = sha512_neon_update, + .final = sha512_neon_final, + .export = sha512_neon_export, + .import = sha512_neon_import, + .descsize = sizeof(struct sha512_state), + .statesize = sizeof(struct sha512_state), + .base = { + .cra_name = "sha512", + .cra_driver_name = "sha512-neon", + .cra_priority = 250, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}, { + .digestsize = SHA384_DIGEST_SIZE, + .init = sha384_neon_init, + .update = sha512_neon_update, + .final = sha384_neon_final, + .export = sha512_neon_export, + .import = sha512_neon_import, + .descsize = sizeof(struct sha512_state), + .statesize = sizeof(struct sha512_state), + .base = { + .cra_name = "sha384", + .cra_driver_name = "sha384-neon", + .cra_priority = 250, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +} }; + +static int __init sha512_neon_mod_init(void) +{ + if (!cpu_has_neon()) + return -ENODEV; + + return crypto_register_shashes(algs, ARRAY_SIZE(algs)); +} + +static void __exit sha512_neon_mod_fini(void) +{ + crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); +} + +module_init(sha512_neon_mod_init); +module_exit(sha512_neon_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA512 Secure Hash Algorithm, NEON accelerated"); + +MODULE_ALIAS("sha512"); +MODULE_ALIAS("sha384"); diff --git a/crypto/Kconfig b/crypto/Kconfig index 6da0c6515f6..df3ed20338c 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -467,6 +467,21 @@ config CRYPTO_SHA512 This code also includes SHA-384, a 384 bit hash with 192 bits of security against collision attacks. +config CRYPTO_SHA512_ARM_NEON + tristate "SHA384 and SHA512 digest algorithm (ARM NEON)" + depends on ARM && KERNEL_MODE_NEON && !CPU_BIG_ENDIAN + select CRYPTO_SHA512 + select CRYPTO_HASH + help + SHA-512 secure hash standard (DFIPS 180-2) implemented + using ARM NEON instructions, when available. + + This version of SHA implements a 512 bit hash with 256 bits of + security against collision attacks. + + This code also includes SHA-384, a 384 bit hash with 192 bits + of security against collision attacks. + config CRYPTO_TGR192 tristate "Tiger digest algorithms" select CRYPTO_HASH From 70142c5f3fcb811fa00c9f79eafe29af8bf12a3c Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 16 May 2013 11:41:48 +0200 Subject: [PATCH 061/168] ARM: add support for kernel mode NEON In order to safely support the use of NEON instructions in kernel mode, some precautions need to be taken: - the userland context that may be present in the registers (even if the NEON/VFP is currently disabled) must be stored under the correct task (which may not be 'current' in the UP case), - to avoid having to keep track of additional vfpstates for the kernel side, disallow the use of NEON in interrupt context and run with preemption disabled, - after use, re-enable preemption and re-enable the lazy restore machinery by disabling the NEON/VFP unit. This patch adds the functions kernel_neon_begin() and kernel_neon_end() which take care of the above. It also adds the Kconfig symbol KERNEL_MODE_NEON to enable it. Change-Id: I595345a0eabcc55685582ed3df1c2968bf49e037 Signed-off-by: Ard Biesheuvel Acked-by: Nicolas Pitre Git-commit: 3ad7b1bb81d15ab510cce9dc9df7f1a18185ee36 Git-repo: https://android.googlesource.com/kernel/common/ Signed-off-by: Samir Mehta --- arch/arm/Kconfig | 7 ++++++ arch/arm/include/asm/neon.h | 36 ++++++++++++++++++++++++++++ arch/arm/vfp/vfpmodule.c | 47 +++++++++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+) create mode 100644 arch/arm/include/asm/neon.h diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 9468df594cd..66de0a921be 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -2436,6 +2436,13 @@ config NEON Say Y to include support code for NEON, the ARMv7 Advanced SIMD Extension. +config KERNEL_MODE_NEON + bool "Support for NEON in kernel mode" + default n + depends on NEON + help + Say Y to include support for NEON in kernel mode. + endmenu menu "Userspace binary formats" diff --git a/arch/arm/include/asm/neon.h b/arch/arm/include/asm/neon.h new file mode 100644 index 00000000000..8f730fe7009 --- /dev/null +++ b/arch/arm/include/asm/neon.h @@ -0,0 +1,36 @@ +/* + * linux/arch/arm/include/asm/neon.h + * + * Copyright (C) 2013 Linaro Ltd + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include + +#define cpu_has_neon() (!!(elf_hwcap & HWCAP_NEON)) + +#ifdef __ARM_NEON__ + +/* + * If you are affected by the BUILD_BUG below, it probably means that you are + * using NEON code /and/ calling the kernel_neon_begin() function from the same + * compilation unit. To prevent issues that may arise from GCC reordering or + * generating(1) NEON instructions outside of these begin/end functions, the + * only supported way of using NEON code in the kernel is by isolating it in a + * separate compilation unit, and calling it from another unit from inside a + * kernel_neon_begin/kernel_neon_end pair. + * + * (1) Current GCC (4.7) might generate NEON instructions at O3 level if + * -mpfu=neon is set. + */ + +#define kernel_neon_begin() \ + BUILD_BUG_ON_MSG(1, "kernel_neon_begin() called from NEON code") + +#else +void kernel_neon_begin(void); +#endif +void kernel_neon_end(void); diff --git a/arch/arm/vfp/vfpmodule.c b/arch/arm/vfp/vfpmodule.c index 9a1a2eb1c1e..89e3b817780 100644 --- a/arch/arm/vfp/vfpmodule.c +++ b/arch/arm/vfp/vfpmodule.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -636,6 +637,52 @@ int vfp_restore_user_hwstate(struct user_vfp __user *ufp, return err ? -EFAULT : 0; } +#ifdef CONFIG_KERNEL_MODE_NEON + +/* + * Kernel-side NEON support functions + */ +void kernel_neon_begin(void) +{ + struct thread_info *thread = current_thread_info(); + unsigned int cpu; + u32 fpexc; + + /* + * Kernel mode NEON is only allowed outside of interrupt context + * with preemption disabled. This will make sure that the kernel + * mode NEON register contents never need to be preserved. + */ + BUG_ON(in_interrupt()); + cpu = get_cpu(); + + fpexc = fmrx(FPEXC) | FPEXC_EN; + fmxr(FPEXC, fpexc); + + /* + * Save the userland NEON/VFP state. Under UP, + * the owner could be a task other than 'current' + */ + if (vfp_state_in_hw(cpu, thread)) + vfp_save_state(&thread->vfpstate, fpexc); +#ifndef CONFIG_SMP + else if (vfp_current_hw_state[cpu] != NULL) + vfp_save_state(vfp_current_hw_state[cpu], fpexc); +#endif + vfp_current_hw_state[cpu] = NULL; +} +EXPORT_SYMBOL(kernel_neon_begin); + +void kernel_neon_end(void) +{ + /* Disable the NEON/VFP unit. */ + fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN); + put_cpu(); +} +EXPORT_SYMBOL(kernel_neon_end); + +#endif /* CONFIG_KERNEL_MODE_NEON */ + /* * VFP hardware can lose all context when a CPU goes offline. * As we will be running in SMP mode with CPU hotplug, we will save the From 40290259f3e250cb0de3b95815f9a1163c5ba132 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 20 Sep 2013 09:55:40 +0200 Subject: [PATCH 062/168] crypto: create generic version of ablk_helper Create a generic version of ablk_helper so it can be reused by other architectures. Change-Id: I3bc45cde3c736b82e752ae8a9ea6a2f9465963fc Acked-by: Jussi Kivilinna Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu Git-commit: 0df9ee0ee0dd0f50f1c0c03f14f44dc76ad7dd60 Git-repo: https://android.googlesource.com/kernel/common/ Signed-off-by: Samir Mehta --- crypto/Kconfig | 4 + crypto/Makefile | 1 + crypto/ablk_helper.c | 150 +++++++++++++++++++++++++++++++++++ include/asm-generic/simd.h | 14 ++++ include/crypto/ablk_helper.h | 31 ++++++++ 5 files changed, 200 insertions(+) create mode 100644 crypto/ablk_helper.c create mode 100644 include/asm-generic/simd.h create mode 100644 include/crypto/ablk_helper.h diff --git a/crypto/Kconfig b/crypto/Kconfig index df3ed20338c..77e3752897a 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -100,6 +100,10 @@ config CRYPTO_MANAGER2 select CRYPTO_BLKCIPHER2 select CRYPTO_PCOMP2 +config CRYPTO_ABLK_HELPER + tristate + select CRYPTO_CRYPTD + config CRYPTO_USER tristate "Userspace cryptographic algorithm configuration" depends on NET diff --git a/crypto/Makefile b/crypto/Makefile index 30f33d67533..a34d1453012 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -96,3 +96,4 @@ obj-$(CONFIG_CRYPTO_USER_API_SKCIPHER) += algif_skcipher.o # obj-$(CONFIG_XOR_BLOCKS) += xor.o obj-$(CONFIG_ASYNC_CORE) += async_tx/ +obj-$(CONFIG_CRYPTO_ABLK_HELPER) += ablk_helper.o diff --git a/crypto/ablk_helper.c b/crypto/ablk_helper.c new file mode 100644 index 00000000000..62568b1fc88 --- /dev/null +++ b/crypto/ablk_helper.c @@ -0,0 +1,150 @@ +/* + * Shared async block cipher helpers + * + * Copyright (c) 2012 Jussi Kivilinna + * + * Based on aesni-intel_glue.c by: + * Copyright (C) 2008, Intel Corp. + * Author: Huang Ying + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int key_len) +{ + struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm); + struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base; + int err; + + crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); + crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm) + & CRYPTO_TFM_REQ_MASK); + err = crypto_ablkcipher_setkey(child, key, key_len); + crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child) + & CRYPTO_TFM_RES_MASK); + return err; +} +EXPORT_SYMBOL_GPL(ablk_set_key); + +int __ablk_encrypt(struct ablkcipher_request *req) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm); + struct blkcipher_desc desc; + + desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm); + desc.info = req->info; + desc.flags = 0; + + return crypto_blkcipher_crt(desc.tfm)->encrypt( + &desc, req->dst, req->src, req->nbytes); +} +EXPORT_SYMBOL_GPL(__ablk_encrypt); + +int ablk_encrypt(struct ablkcipher_request *req) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm); + + if (!may_use_simd()) { + struct ablkcipher_request *cryptd_req = + ablkcipher_request_ctx(req); + + memcpy(cryptd_req, req, sizeof(*req)); + ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); + + return crypto_ablkcipher_encrypt(cryptd_req); + } else { + return __ablk_encrypt(req); + } +} +EXPORT_SYMBOL_GPL(ablk_encrypt); + +int ablk_decrypt(struct ablkcipher_request *req) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm); + + if (!may_use_simd()) { + struct ablkcipher_request *cryptd_req = + ablkcipher_request_ctx(req); + + memcpy(cryptd_req, req, sizeof(*req)); + ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); + + return crypto_ablkcipher_decrypt(cryptd_req); + } else { + struct blkcipher_desc desc; + + desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm); + desc.info = req->info; + desc.flags = 0; + + return crypto_blkcipher_crt(desc.tfm)->decrypt( + &desc, req->dst, req->src, req->nbytes); + } +} +EXPORT_SYMBOL_GPL(ablk_decrypt); + +void ablk_exit(struct crypto_tfm *tfm) +{ + struct async_helper_ctx *ctx = crypto_tfm_ctx(tfm); + + cryptd_free_ablkcipher(ctx->cryptd_tfm); +} +EXPORT_SYMBOL_GPL(ablk_exit); + +int ablk_init_common(struct crypto_tfm *tfm, const char *drv_name) +{ + struct async_helper_ctx *ctx = crypto_tfm_ctx(tfm); + struct cryptd_ablkcipher *cryptd_tfm; + + cryptd_tfm = cryptd_alloc_ablkcipher(drv_name, 0, 0); + if (IS_ERR(cryptd_tfm)) + return PTR_ERR(cryptd_tfm); + + ctx->cryptd_tfm = cryptd_tfm; + tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) + + crypto_ablkcipher_reqsize(&cryptd_tfm->base); + + return 0; +} +EXPORT_SYMBOL_GPL(ablk_init_common); + +int ablk_init(struct crypto_tfm *tfm) +{ + char drv_name[CRYPTO_MAX_ALG_NAME]; + + snprintf(drv_name, sizeof(drv_name), "__driver-%s", + crypto_tfm_alg_driver_name(tfm)); + + return ablk_init_common(tfm, drv_name); +} +EXPORT_SYMBOL_GPL(ablk_init); + +MODULE_LICENSE("GPL"); diff --git a/include/asm-generic/simd.h b/include/asm-generic/simd.h new file mode 100644 index 00000000000..f57eb7b5c23 --- /dev/null +++ b/include/asm-generic/simd.h @@ -0,0 +1,14 @@ + +#include + +/* + * may_use_simd - whether it is allowable at this time to issue SIMD + * instructions or access the SIMD register file + * + * As architectures typically don't preserve the SIMD register file when + * taking an interrupt, !in_interrupt() should be a reasonable default. + */ +static __must_check inline bool may_use_simd(void) +{ + return !in_interrupt(); +} diff --git a/include/crypto/ablk_helper.h b/include/crypto/ablk_helper.h new file mode 100644 index 00000000000..4f93df50c23 --- /dev/null +++ b/include/crypto/ablk_helper.h @@ -0,0 +1,31 @@ +/* + * Shared async block cipher helpers + */ + +#ifndef _CRYPTO_ABLK_HELPER_H +#define _CRYPTO_ABLK_HELPER_H + +#include +#include +#include + +struct async_helper_ctx { + struct cryptd_ablkcipher *cryptd_tfm; +}; + +extern int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int key_len); + +extern int __ablk_encrypt(struct ablkcipher_request *req); + +extern int ablk_encrypt(struct ablkcipher_request *req); + +extern int ablk_decrypt(struct ablkcipher_request *req); + +extern void ablk_exit(struct crypto_tfm *tfm); + +extern int ablk_init_common(struct crypto_tfm *tfm, const char *drv_name); + +extern int ablk_init(struct crypto_tfm *tfm); + +#endif /* _CRYPTO_ABLK_HELPER_H */ From 53cd26560905c545da8c2fa51d367e36fc595cf0 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 22 May 2013 10:38:53 +0200 Subject: [PATCH 063/168] ARM: move VFP init to an earlier boot stage In order to use the NEON unit in the kernel, we should initialize it a bit earlier in the boot process so NEON users that like to do a quick benchmark at load time (like the xor_blocks or RAID-6 code) find the NEON/VFP unit already enabled. Replaced late_initcall() with core_initcall(). Change-Id: I09b1f48a0a4bfb2fec98aa548718a40cd82ea51e Signed-off-by: Ard Biesheuvel Acked-by: Nicolas Pitre Git-commit: be194465a238c9b19e0cb46729b8a3b5deb56ee5 Git-repo: https://android.googlesource.com/kernel/common/ Signed-off-by: Samir Mehta --- arch/arm/vfp/vfpmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/vfp/vfpmodule.c b/arch/arm/vfp/vfpmodule.c index 89e3b817780..117f6d137f7 100644 --- a/arch/arm/vfp/vfpmodule.c +++ b/arch/arm/vfp/vfpmodule.c @@ -813,4 +813,4 @@ static int __init vfp_init(void) return 0; } -late_initcall(vfp_init); +core_initcall(vfp_init); From 3e1ae5f8455adc69ac66c4aebc87fe059efb16a7 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 20 Sep 2013 09:57:37 +0200 Subject: [PATCH 064/168] ARM: pull in from asm-generic Change-Id: I2749ede160a1f99ddef0bb126cea1c7dbdd779e0 Signed-off-by: Ard Biesheuvel Git-commit: ca768fdb1a4f79fa6ed0f8400b9b7f11e8c8464e Git-repo: https://android.googlesource.com/kernel/common/ Signed-off-by: Samir Mehta --- arch/arm/include/asm/Kbuild | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm/include/asm/Kbuild b/arch/arm/include/asm/Kbuild index 960abceb8e1..9b7ea8f32ec 100644 --- a/arch/arm/include/asm/Kbuild +++ b/arch/arm/include/asm/Kbuild @@ -17,4 +17,5 @@ generic-y += poll.h generic-y += resource.h generic-y += sections.h generic-y += siginfo.h +generic-y += simd.h generic-y += sizes.h From db5cb41a12398bc626c6a211463d93bf31af15a9 Mon Sep 17 00:00:00 2001 From: Jussi Kivilinna Date: Wed, 11 Jul 2012 14:20:20 +0300 Subject: [PATCH 065/168] crypto: add crypto_[un]register_shashes for [un]registering multiple shash entries at once Add crypto_[un]register_shashes() to allow simplifying init/exit code of shash crypto modules that register multiple algorithms. Change-Id: Iaf5fe0ecce13ee6bd8018f25f59983b466692ceb Signed-off-by: Jussi Kivilinna Signed-off-by: Herbert Xu Git-commit: 50fc3e8d2c9d1ee72c67b751e5ac5d76ebc5a12e Git-repo: https://android.googlesource.com/kernel/common/ Signed-off-by: Samir Mehta --- crypto/shash.c | 36 ++++++++++++++++++++++++++++++++++ include/crypto/internal/hash.h | 2 ++ 2 files changed, 38 insertions(+) diff --git a/crypto/shash.c b/crypto/shash.c index 21fc12e2378..8f2b9a06957 100644 --- a/crypto/shash.c +++ b/crypto/shash.c @@ -629,6 +629,42 @@ int crypto_unregister_shash(struct shash_alg *alg) } EXPORT_SYMBOL_GPL(crypto_unregister_shash); +int crypto_register_shashes(struct shash_alg *algs, int count) +{ + int i, ret; + + for (i = 0; i < count; i++) { + ret = crypto_register_shash(&algs[i]); + if (ret) + goto err; + } + + return 0; + +err: + for (--i; i >= 0; --i) + crypto_unregister_shash(&algs[i]); + + return ret; +} +EXPORT_SYMBOL_GPL(crypto_register_shashes); + +int crypto_unregister_shashes(struct shash_alg *algs, int count) +{ + int i, ret; + + for (i = count - 1; i >= 0; --i) { + ret = crypto_unregister_shash(&algs[i]); + if (ret) + pr_err("Failed to unregister %s %s: %d\n", + algs[i].base.cra_driver_name, + algs[i].base.cra_name, ret); + } + + return 0; +} +EXPORT_SYMBOL_GPL(crypto_unregister_shashes); + int shash_register_instance(struct crypto_template *tmpl, struct shash_instance *inst) { diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index 5bfad8c8059..821eae8cbd8 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -83,6 +83,8 @@ struct hash_alg_common *ahash_attr_alg(struct rtattr *rta, u32 type, u32 mask); int crypto_register_shash(struct shash_alg *alg); int crypto_unregister_shash(struct shash_alg *alg); +int crypto_register_shashes(struct shash_alg *algs, int count); +int crypto_unregister_shashes(struct shash_alg *algs, int count); int shash_register_instance(struct crypto_template *tmpl, struct shash_instance *inst); void shash_free_instance(struct crypto_instance *inst); From 1fa5619d6ad7d17f26282ac34914845d682144ea Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Tue, 26 Mar 2013 13:58:49 -0700 Subject: [PATCH 066/168] crypto: sha256 - Expose SHA256 generic routine to be callable externally. Other SHA256 routine may need to use the generic routine when FPU is not available. Change-Id: I9301e29ef34cd2597d9e418f43a07c2ce9d6eafa Signed-off-by: Tim Chen Signed-off-by: Herbert Xu Git-commit: 35d2c9d0c3cfd90850dc647250610587743e1f29 Git-repo: https://android.googlesource.com/kernel/common/ Signed-off-by: Samir Mehta --- crypto/sha256_generic.c | 11 ++++++----- include/crypto/sha.h | 2 ++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/crypto/sha256_generic.c b/crypto/sha256_generic.c index c48459ebf05..02f089a7080 100644 --- a/crypto/sha256_generic.c +++ b/crypto/sha256_generic.c @@ -246,7 +246,7 @@ static int sha256_init(struct shash_desc *desc) return 0; } -static int sha256_update(struct shash_desc *desc, const u8 *data, +int crypto_sha256_update(struct shash_desc *desc, const u8 *data, unsigned int len) { struct sha256_state *sctx = shash_desc_ctx(desc); @@ -277,6 +277,7 @@ static int sha256_update(struct shash_desc *desc, const u8 *data, return 0; } +EXPORT_SYMBOL(crypto_sha256_update); static int sha256_final(struct shash_desc *desc, u8 *out) { @@ -293,10 +294,10 @@ static int sha256_final(struct shash_desc *desc, u8 *out) /* Pad out to 56 mod 64. */ index = sctx->count & 0x3f; pad_len = (index < 56) ? (56 - index) : ((64+56) - index); - sha256_update(desc, padding, pad_len); + crypto_sha256_update(desc, padding, pad_len); /* Append length (before padding) */ - sha256_update(desc, (const u8 *)&bits, sizeof(bits)); + crypto_sha256_update(desc, (const u8 *)&bits, sizeof(bits)); /* Store state in digest */ for (i = 0; i < 8; i++) @@ -339,7 +340,7 @@ static int sha256_import(struct shash_desc *desc, const void *in) static struct shash_alg sha256 = { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_init, - .update = sha256_update, + .update = crypto_sha256_update, .final = sha256_final, .export = sha256_export, .import = sha256_import, @@ -357,7 +358,7 @@ static struct shash_alg sha256 = { static struct shash_alg sha224 = { .digestsize = SHA224_DIGEST_SIZE, .init = sha224_init, - .update = sha256_update, + .update = crypto_sha256_update, .final = sha224_final, .descsize = sizeof(struct sha256_state), .base = { diff --git a/include/crypto/sha.h b/include/crypto/sha.h index c6c9c1fe460..f46ff61e378 100644 --- a/include/crypto/sha.h +++ b/include/crypto/sha.h @@ -87,4 +87,6 @@ struct shash_desc; extern int crypto_sha1_update(struct shash_desc *desc, const u8 *data, unsigned int len); +extern int crypto_sha256_update(struct shash_desc *desc, const u8 *data, + unsigned int len); #endif From e068b323163b5af862b0c66e9e59f75ebf577cb0 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Tue, 26 Mar 2013 13:59:25 -0700 Subject: [PATCH 067/168] crypto: sha512 - Expose generic sha512 routine to be callable from other modules Other SHA512 routines may need to use the generic routine when FPU is not available. Change-Id: Ie610359fc36474c936cf63a57d7de74bb64a40cd Signed-off-by: Tim Chen Signed-off-by: Herbert Xu Git-commit: bf70fa9d9ee07aa175453b19a39b2b9dab602d97 Git-repo: https://android.googlesource.com/kernel/common/ Signed-off-by: Samir Mehta --- crypto/sha512_generic.c | 13 +++++++------ include/crypto/sha.h | 3 +++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/crypto/sha512_generic.c b/crypto/sha512_generic.c index dd30f40af9f..fa0ae51d07d 100644 --- a/crypto/sha512_generic.c +++ b/crypto/sha512_generic.c @@ -163,8 +163,8 @@ sha384_init(struct shash_desc *desc) return 0; } -static int -sha512_update(struct shash_desc *desc, const u8 *data, unsigned int len) +int crypto_sha512_update(struct shash_desc *desc, const u8 *data, + unsigned int len) { struct sha512_state *sctx = shash_desc_ctx(desc); @@ -197,6 +197,7 @@ sha512_update(struct shash_desc *desc, const u8 *data, unsigned int len) return 0; } +EXPORT_SYMBOL(crypto_sha512_update); static int sha512_final(struct shash_desc *desc, u8 *hash) @@ -215,10 +216,10 @@ sha512_final(struct shash_desc *desc, u8 *hash) /* Pad out to 112 mod 128. */ index = sctx->count[0] & 0x7f; pad_len = (index < 112) ? (112 - index) : ((128+112) - index); - sha512_update(desc, padding, pad_len); + crypto_sha512_update(desc, padding, pad_len); /* Append length (before padding) */ - sha512_update(desc, (const u8 *)bits, sizeof(bits)); + crypto_sha512_update(desc, (const u8 *)bits, sizeof(bits)); /* Store state in digest */ for (i = 0; i < 8; i++) @@ -245,7 +246,7 @@ static int sha384_final(struct shash_desc *desc, u8 *hash) static struct shash_alg sha512 = { .digestsize = SHA512_DIGEST_SIZE, .init = sha512_init, - .update = sha512_update, + .update = crypto_sha512_update, .final = sha512_final, .descsize = sizeof(struct sha512_state), .base = { @@ -259,7 +260,7 @@ static struct shash_alg sha512 = { static struct shash_alg sha384 = { .digestsize = SHA384_DIGEST_SIZE, .init = sha384_init, - .update = sha512_update, + .update = crypto_sha512_update, .final = sha384_final, .descsize = sizeof(struct sha512_state), .base = { diff --git a/include/crypto/sha.h b/include/crypto/sha.h index f46ff61e378..190f8a0e024 100644 --- a/include/crypto/sha.h +++ b/include/crypto/sha.h @@ -89,4 +89,7 @@ extern int crypto_sha1_update(struct shash_desc *desc, const u8 *data, extern int crypto_sha256_update(struct shash_desc *desc, const u8 *data, unsigned int len); + +extern int crypto_sha512_update(struct shash_desc *desc, const u8 *data, + unsigned int len); #endif From dbe7add8a7a439e9d480d4bd4e0bd3ab318227fe Mon Sep 17 00:00:00 2001 From: Ravinder Konka Date: Thu, 17 Dec 2015 21:24:02 -0800 Subject: [PATCH 068/168] Revert "net/ipv6/route.c: packets originating on device match lo" This reverts commit 274f0b63bc3e6d7a7d1b6e3af015221830fd574b Change-Id: I080ab5c017dfca3b55f03ca4784b32369431238e Signed-off-by: Ravinder Konka Signed-off-by: Shreyas Narayan --- net/ipv6/route.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/net/ipv6/route.c b/net/ipv6/route.c index ab71b41e139..3e6b53b981a 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -928,8 +928,6 @@ struct dst_entry * ip6_route_output(struct net *net, const struct sock *sk, { int flags = 0; - fl6->flowi6_iif = net->loopback_dev->ifindex; - if ((sk && sk->sk_bound_dev_if) || rt6_need_strict(&fl6->daddr)) flags |= RT6_LOOKUP_F_IFACE; From 87564c19a43fc99d413c455e9b4bf44a79040ff4 Mon Sep 17 00:00:00 2001 From: David McCullough Date: Mon, 25 Jun 2012 15:42:26 +0000 Subject: [PATCH 069/168] net/ipv6/route.c: packets originating on device match lo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix to allow IPv6 packets originating locally to match rules with the "iff" set to "lo". This allows IPv6 rule matching work the same as it does for IPv4. From the iproute2 man page: iif NAME select the incoming device to match. If the interface is loop‐ back, the rule only matches packets originating from this host. This means that you may create separate routing tables for for‐ warded and local packets and, hence, completely segregate them. Change-Id: Id044383eaa17b24b99b167b0b14d88c7dcb3292d Signed-off-by: David McCullough Signed-off-by: David S. Miller Git-commit: 4dc27d1cf3b3027b9ce654221c559e88b2f41b33 Git-repo: https://android.googlesource.com/kernel/msm/ Signed-off-by: Ravinder Konka --- net/ipv6/route.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/net/ipv6/route.c b/net/ipv6/route.c index 3e6b53b981a..ab71b41e139 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -928,6 +928,8 @@ struct dst_entry * ip6_route_output(struct net *net, const struct sock *sk, { int flags = 0; + fl6->flowi6_iif = net->loopback_dev->ifindex; + if ((sk && sk->sk_bound_dev_if) || rt6_need_strict(&fl6->daddr)) flags |= RT6_LOOKUP_F_IFACE; From 39647537806b1dbc7ce5e4c14c8e309d487569cf Mon Sep 17 00:00:00 2001 From: "Padma, Santhosh Kumar" Date: Wed, 25 Nov 2015 12:30:19 +0530 Subject: [PATCH 070/168] wcnss: Expose dump stack functionality Add changes to expose dump stack functionality which can be used by driver to dump stack information when it requires CR Fixed: 943322 Change-Id: I0fde7142dea2c18daf6b1fb0c5ee4bb8a31a6be0 Signed-off-by: Padma, Santhosh Kumar Signed-off-by: Pradosh Das --- drivers/net/wireless/wcnss/wcnss_wlan.c | 10 ++++++++++ include/linux/wcnss_wlan.h | 2 ++ 2 files changed, 12 insertions(+) diff --git a/drivers/net/wireless/wcnss/wcnss_wlan.c b/drivers/net/wireless/wcnss/wcnss_wlan.c index 131c5279e81..3ea7eebc4b0 100644 --- a/drivers/net/wireless/wcnss/wcnss_wlan.c +++ b/drivers/net/wireless/wcnss/wcnss_wlan.c @@ -2687,6 +2687,16 @@ void wcnss_flush_work(struct work_struct *work) } EXPORT_SYMBOL(wcnss_flush_work); +/* wlan prop driver cannot invoke show_stack + * function directly, so to invoke this function it + * call wcnss_dump_stack function + */ +void wcnss_dump_stack(struct task_struct *task) +{ + show_stack(task, NULL); +} +EXPORT_SYMBOL(wcnss_dump_stack); + /* wlan prop driver cannot invoke cancel_delayed_work_sync * function directly, so to invoke this function it call * wcnss_flush_delayed_work function diff --git a/include/linux/wcnss_wlan.h b/include/linux/wcnss_wlan.h index cfa92854d95..6218c336e69 100644 --- a/include/linux/wcnss_wlan.h +++ b/include/linux/wcnss_wlan.h @@ -15,6 +15,7 @@ #define _WCNSS_WLAN_H_ #include +#include enum wcnss_opcode { WCNSS_WLAN_SWITCH_OFF = 0, @@ -94,6 +95,7 @@ int wcnss_wlan_iris_xo_mode(void); void wcnss_flush_work(struct work_struct *work); void wcnss_flush_delayed_work(struct delayed_work *dwork); int wcnss_get_iris_name(char *iris_version); +void wcnss_dump_stack(struct task_struct *task); #ifdef CONFIG_WCNSS_REGISTER_DUMP_ON_BITE void wcnss_log_debug_regs_on_bite(void); From 8d1603876ad7c604547ef24b35d01636c4196a5b Mon Sep 17 00:00:00 2001 From: Samir Mehta Date: Tue, 22 Dec 2015 12:03:12 +0530 Subject: [PATCH 071/168] arm: defconfig: Enable NEON config Enable NEON related configs. Change-Id: I2dcb41e6782cbd2347d5472b18ab397d16cae44b Signed-off-by: Samir Mehta --- arch/arm/configs/msm8226-perf_defconfig | 9 +++++++++ arch/arm/configs/msm8226_defconfig | 9 +++++++++ arch/arm/configs/msm8974-perf_defconfig | 9 +++++++++ arch/arm/configs/msm8974_defconfig | 9 +++++++++ 4 files changed, 36 insertions(+) diff --git a/arch/arm/configs/msm8226-perf_defconfig b/arch/arm/configs/msm8226-perf_defconfig index 51ee11d2b4f..7a29f307a4a 100644 --- a/arch/arm/configs/msm8226-perf_defconfig +++ b/arch/arm/configs/msm8226-perf_defconfig @@ -476,6 +476,15 @@ CONFIG_NFC_QNCI=y CONFIG_CRYPTO_DEV_QCRYPTO=m CONFIG_CRYPTO_DEV_QCE=y CONFIG_CRYPTO_DEV_QCEDEV=m +CONFIG_KERNEL_MODE_NEON=y +CONFIG_CRYPTO_AES_ARM=y +CONFIG_CRYPTO_ALGAPI=y +CONFIG_CRYPTO_AES=y +CONFIG_CRYPTO_ALGAPI2=y +CONFIG_CRYPTO_SHA1_ARM_NEON=y +CONFIG_CRYPTO_SHA512_ARM_NEON=y +CONFIG_CRYPTO_AES_ARM_BS=y +CONFIG_CRYPTO_SHA1_ARM=y CONFIG_MOBICORE_SUPPORT=m CONFIG_MOBICORE_API=m CONFIG_MSM_RPM_RBCPR_STATS_V2_LOG=y diff --git a/arch/arm/configs/msm8226_defconfig b/arch/arm/configs/msm8226_defconfig index 52c6ca170e5..8fa182ceb91 100644 --- a/arch/arm/configs/msm8226_defconfig +++ b/arch/arm/configs/msm8226_defconfig @@ -529,6 +529,15 @@ CONFIG_CRYPTO_MD4=y CONFIG_CRYPTO_ARC4=y CONFIG_CRYPTO_XCBC=y CONFIG_CRYPTO_TWOFISH=y +CONFIG_KERNEL_MODE_NEON=y +CONFIG_CRYPTO_AES_ARM=y +CONFIG_CRYPTO_ALGAPI=y +CONFIG_CRYPTO_AES=y +CONFIG_CRYPTO_ALGAPI2=y +CONFIG_CRYPTO_SHA1_ARM_NEON=y +CONFIG_CRYPTO_SHA512_ARM_NEON=y +CONFIG_CRYPTO_AES_ARM_BS=y +CONFIG_CRYPTO_SHA1_ARM=y CONFIG_NFC_QNCI=y CONFIG_CRYPTO_DEV_QCRYPTO=m CONFIG_CRYPTO_DEV_QCE=y diff --git a/arch/arm/configs/msm8974-perf_defconfig b/arch/arm/configs/msm8974-perf_defconfig index 5e51adb38fa..4f9154713d2 100755 --- a/arch/arm/configs/msm8974-perf_defconfig +++ b/arch/arm/configs/msm8974-perf_defconfig @@ -511,6 +511,15 @@ CONFIG_NFC_QNCI=y CONFIG_CRYPTO_DEV_QCRYPTO=m CONFIG_CRYPTO_DEV_QCE=y CONFIG_CRYPTO_DEV_QCEDEV=y +CONFIG_KERNEL_MODE_NEON=y +CONFIG_CRYPTO_AES_ARM=y +CONFIG_CRYPTO_ALGAPI=y +CONFIG_CRYPTO_AES=y +CONFIG_CRYPTO_ALGAPI2=y +CONFIG_CRYPTO_SHA1_ARM_NEON=y +CONFIG_CRYPTO_SHA512_ARM_NEON=y +CONFIG_CRYPTO_AES_ARM_BS=y +CONFIG_CRYPTO_SHA1_ARM=y CONFIG_PFT=y CONFIG_SND_SOC_MSM_HDMI_CODEC_RX=y CONFIG_BUILD_ARM_APPENDED_DTB_IMAGE=y diff --git a/arch/arm/configs/msm8974_defconfig b/arch/arm/configs/msm8974_defconfig index 82791a332bb..0e255e0b1b7 100755 --- a/arch/arm/configs/msm8974_defconfig +++ b/arch/arm/configs/msm8974_defconfig @@ -565,6 +565,15 @@ CONFIG_CRYPTO_MD4=y CONFIG_CRYPTO_ARC4=y CONFIG_CRYPTO_TWOFISH=y CONFIG_CRYPTO_XTS=y +CONFIG_KERNEL_MODE_NEON=y +CONFIG_CRYPTO_AES_ARM=y +CONFIG_CRYPTO_ALGAPI=y +CONFIG_CRYPTO_AES=y +CONFIG_CRYPTO_ALGAPI2=y +CONFIG_CRYPTO_SHA1_ARM_NEON=y +CONFIG_CRYPTO_SHA512_ARM_NEON=y +CONFIG_CRYPTO_AES_ARM_BS=y +CONFIG_CRYPTO_SHA1_ARM=y CONFIG_NFC_QNCI=y CONFIG_CRYPTO_DEV_QCRYPTO=y CONFIG_CRYPTO_DEV_QCE=y From 4a98d423b1308188a6c8a0c166f114ec118bed24 Mon Sep 17 00:00:00 2001 From: Harout Hedeshian Date: Wed, 15 Apr 2015 14:59:27 -0600 Subject: [PATCH 072/168] net: udp: Adjust UDP socket state for encapsulation sockets UDP IPv4 encapsulation sockets will have their state printed as 0xF0 binary ORed with the actual state such that they can be distinguished from regular UDP sockets in /proc/net/udp. CRs-Fixed: 821341 Change-Id: I240ab1526a4280e5e996d9577a904581684fc84a Signed-off-by: Harout Hedeshian Signed-off-by: Shreyas Narayan --- net/ipv4/udp.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index 72f2bb907ca..0d8a5a4a74c 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -2084,14 +2084,18 @@ static void udp4_format_sock(struct sock *sp, struct seq_file *f, int bucket) { struct inet_sock *inet = inet_sk(sp); + struct udp_sock *up = udp_sk(sp); __be32 dest = inet->inet_daddr; __be32 src = inet->inet_rcv_saddr; __u16 destp = ntohs(inet->inet_dport); __u16 srcp = ntohs(inet->inet_sport); + __u8 state = sp->sk_state; + if (up->encap_rcv) + state |= 0xF0; seq_printf(f, "%5d: %08X:%04X %08X:%04X" - " %02X %08X:%08X %02X:%08lX %08X %5d %8d %lu %d %pK %d", - bucket, src, srcp, dest, destp, sp->sk_state, + " %02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %d", + bucket, src, srcp, dest, destp, state, sk_wmem_alloc_get(sp), sk_rmem_alloc_get(sp), 0, 0L, 0, sock_i_uid(sp), 0, sock_i_ino(sp), From e595866cddd6e3813ad872cc0b66fefd66bf720b Mon Sep 17 00:00:00 2001 From: Ravinder Konka Date: Tue, 5 Jan 2016 16:54:16 +0530 Subject: [PATCH 073/168] net: msm_rmnet: Add an IOCTL definition to deregister a net device Platform drivers may be registered on driver startup for multiple channels which may cause probe and remove events to be called corresponding to driver events. It is possible that not all channels need to be enumerated for normal device operation. The unused channels also add unnecessary delay during sub system restart. Add an extended IOCTL which provides user space an option to unregister a platform driver. CRs-Fixed: 738039 Change-Id: Ifa6284563551ecdd297ae457ea2b35027f60844e Acked-by: Nagarjuna Chaganti Signed-off-by: Ravinder Konka Signed-off-by: Subash Abhinov Kasiviswanathan --- include/linux/msm_rmnet.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/linux/msm_rmnet.h b/include/linux/msm_rmnet.h index 08cd3b67ece..ff5eaa1f378 100644 --- a/include/linux/msm_rmnet.h +++ b/include/linux/msm_rmnet.h @@ -58,7 +58,8 @@ enum rmnet_ioctl_extended_cmds_e { RMNET_IOCTL_GET_SUPPORTED_QOS_MODES = 0x0013, /* Get QoS modes */ RMNET_IOCTL_SET_SLEEP_STATE = 0x0014, /* Set sleep state */ RMNET_IOCTL_SET_XLAT_DEV_INFO = 0x0015, /* xlat dev name */ - RMNET_IOCTL_EXTENDED_MAX = 0x0016 + RMNET_IOCTL_DEREGISTER_DEV = 0x0016, /* Dereg a net dev */ + RMNET_IOCTL_EXTENDED_MAX = 0x0017 }; /* Return values for the RMNET_IOCTL_GET_SUPPORTED_FEATURES IOCTL */ From acb0cece3a6399967f41f8b8cd4ea439ac8f7bac Mon Sep 17 00:00:00 2001 From: Balamurugan Alagarsamy Date: Thu, 14 Jan 2016 09:35:30 +0530 Subject: [PATCH 074/168] msm: vidc: initialize the session list when device initialized When multiple sessions are running and if there is a SYS_ERROR all clients will be notified with hardware error and clients are expected to close the sessions. If new session is opened before all sessions are closed results in list corruption because session open will do list initialization. To avoid list corruption issue initialize the session list at the time of device initialization instead of first session open. Change-Id: Ifd5c62ff7ae0547ecf73029fadbcac66f1e189bc Signed-off-by: Balamurugan Alagarsamy --- drivers/media/platform/msm/vidc/venus_hfi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/media/platform/msm/vidc/venus_hfi.c b/drivers/media/platform/msm/vidc/venus_hfi.c index fdc61c74ffc..8e043ece812 100644 --- a/drivers/media/platform/msm/vidc/venus_hfi.c +++ b/drivers/media/platform/msm/vidc/venus_hfi.c @@ -1977,7 +1977,6 @@ static int venus_hfi_core_init(void *device) VENUS_SET_STATE(dev, VENUS_STATE_INIT); dev->intr_status = 0; - INIT_LIST_HEAD(&dev->sess_head); venus_hfi_set_registers(dev); if (!dev->hal_client) { @@ -3922,6 +3921,7 @@ static void *venus_hfi_add_device(u32 device_id, mutex_init(&hdevice->session_lock); mutex_init(&hdevice->clk_pwr_lock); + INIT_LIST_HEAD(&hdevice->sess_head); if (hal_ctxt.dev_count == 0) INIT_LIST_HEAD(&hal_ctxt.dev_head); From 95d90a5c6210e08a3ff81a0ab7d8b26f8f6ac6d9 Mon Sep 17 00:00:00 2001 From: Pradosh Das Date: Thu, 21 Jan 2016 10:41:20 +0530 Subject: [PATCH 075/168] Revert "mmc: queue: exclude asynchronous transfer for special request" This reverts commit e6ca25a2eb80c3e95e5cc9b1a9c039b37cec2cbb. Change-Id: If4b4dee9bd0d1d5126c910cd49dafb64a751e227 Signed-off-by: Pradosh Das --- drivers/mmc/card/queue.c | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/drivers/mmc/card/queue.c b/drivers/mmc/card/queue.c index 4edea18b2b0..896cc2f496b 100644 --- a/drivers/mmc/card/queue.c +++ b/drivers/mmc/card/queue.c @@ -24,8 +24,6 @@ #define MMC_QUEUE_BOUNCESZ 65536 -#define MMC_REQ_SPECIAL_MASK (REQ_DISCARD | REQ_FLUSH) - /* * Based on benchmark tests the default num of requests to trigger the write * packing was determined, to keep the read latency as low as possible and @@ -68,7 +66,6 @@ static int mmc_queue_thread(void *d) do { struct mmc_queue_req *tmp; struct request *req = NULL; - unsigned int cmd_flags = 0; spin_lock_irq(q->queue_lock); set_current_state(TASK_INTERRUPTIBLE); @@ -78,7 +75,6 @@ static int mmc_queue_thread(void *d) if (req || mq->mqrq_prev->req) { set_current_state(TASK_RUNNING); - cmd_flags = req ? req->cmd_flags : 0; mq->issue_fn(mq, req); if (test_bit(MMC_QUEUE_NEW_REQUEST, &mq->flags)) { continue; /* fetch again */ @@ -99,13 +95,7 @@ static int mmc_queue_thread(void *d) /* * Current request becomes previous request * and vice versa. - * In case of special requests, current request - * has been finished. Do not assign it to previous - * request. */ - if (cmd_flags & MMC_REQ_SPECIAL_MASK) - mq->mqrq_cur->req = NULL; - mq->mqrq_prev->brq.mrq.data = NULL; mq->mqrq_prev->req = NULL; tmp = mq->mqrq_prev; From a0b88907e71207dd5c31ca94ae937558ffdab22f Mon Sep 17 00:00:00 2001 From: Pradosh Das Date: Thu, 21 Jan 2016 11:56:38 +0530 Subject: [PATCH 076/168] Revert "mmc: queue: use cached attributes of special requests" This reverts commit 0fe4ec363c22ca736da710b8d31ba86cbe583dc1. Change-Id: I6dd901e12816ab4bf8ba019c1fede04e5837665f Signed-off-by: Pradosh Das --- drivers/mmc/card/queue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/mmc/card/queue.c b/drivers/mmc/card/queue.c index 896cc2f496b..cceb8d96318 100644 --- a/drivers/mmc/card/queue.c +++ b/drivers/mmc/card/queue.c @@ -80,7 +80,7 @@ static int mmc_queue_thread(void *d) continue; /* fetch again */ } else if (test_bit(MMC_QUEUE_URGENT_REQUEST, &mq->flags) && (mq->mqrq_cur->req && - !(cmd_flags & + !(mq->mqrq_cur->req->cmd_flags & MMC_REQ_NOREINSERT_MASK))) { /* * clean current request when urgent request From b5398952f999df5d72013e081a02d83f76a5e504 Mon Sep 17 00:00:00 2001 From: Pradosh Das Date: Thu, 21 Jan 2016 10:41:20 +0530 Subject: [PATCH 077/168] Revert "mmc: queue: exclude asynchronous transfer for special request" This reverts commit e6ca25a2eb80c3e95e5cc9b1a9c039b37cec2cbb. Change-Id: If4b4dee9bd0d1d5126c910cd49dafb64a751e227 Signed-off-by: Pradosh Das --- drivers/mmc/card/queue.c | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/drivers/mmc/card/queue.c b/drivers/mmc/card/queue.c index 4edea18b2b0..896cc2f496b 100644 --- a/drivers/mmc/card/queue.c +++ b/drivers/mmc/card/queue.c @@ -24,8 +24,6 @@ #define MMC_QUEUE_BOUNCESZ 65536 -#define MMC_REQ_SPECIAL_MASK (REQ_DISCARD | REQ_FLUSH) - /* * Based on benchmark tests the default num of requests to trigger the write * packing was determined, to keep the read latency as low as possible and @@ -68,7 +66,6 @@ static int mmc_queue_thread(void *d) do { struct mmc_queue_req *tmp; struct request *req = NULL; - unsigned int cmd_flags = 0; spin_lock_irq(q->queue_lock); set_current_state(TASK_INTERRUPTIBLE); @@ -78,7 +75,6 @@ static int mmc_queue_thread(void *d) if (req || mq->mqrq_prev->req) { set_current_state(TASK_RUNNING); - cmd_flags = req ? req->cmd_flags : 0; mq->issue_fn(mq, req); if (test_bit(MMC_QUEUE_NEW_REQUEST, &mq->flags)) { continue; /* fetch again */ @@ -99,13 +95,7 @@ static int mmc_queue_thread(void *d) /* * Current request becomes previous request * and vice versa. - * In case of special requests, current request - * has been finished. Do not assign it to previous - * request. */ - if (cmd_flags & MMC_REQ_SPECIAL_MASK) - mq->mqrq_cur->req = NULL; - mq->mqrq_prev->brq.mrq.data = NULL; mq->mqrq_prev->req = NULL; tmp = mq->mqrq_prev; From 5b1b77820e71b157a9b63988ef57d2544badca95 Mon Sep 17 00:00:00 2001 From: Pradosh Das Date: Thu, 21 Jan 2016 11:56:38 +0530 Subject: [PATCH 078/168] Revert "mmc: queue: use cached attributes of special requests" This reverts commit 0fe4ec363c22ca736da710b8d31ba86cbe583dc1. Change-Id: I6dd901e12816ab4bf8ba019c1fede04e5837665f Signed-off-by: Pradosh Das --- drivers/mmc/card/queue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/mmc/card/queue.c b/drivers/mmc/card/queue.c index 896cc2f496b..cceb8d96318 100644 --- a/drivers/mmc/card/queue.c +++ b/drivers/mmc/card/queue.c @@ -80,7 +80,7 @@ static int mmc_queue_thread(void *d) continue; /* fetch again */ } else if (test_bit(MMC_QUEUE_URGENT_REQUEST, &mq->flags) && (mq->mqrq_cur->req && - !(cmd_flags & + !(mq->mqrq_cur->req->cmd_flags & MMC_REQ_NOREINSERT_MASK))) { /* * clean current request when urgent request From 1b3a408ff27fc62c4ec436eb43f398eb1eaa61b5 Mon Sep 17 00:00:00 2001 From: Zhen Kong Date: Tue, 10 Mar 2015 16:29:53 -0700 Subject: [PATCH 079/168] qseecom: Change to work with appsbl qseecom Do not send app regin notification if qseecom in appsbl has already done it Change-Id: I81ae9a991a5d8ec582d9320e18be1b6f8e8ee7fd Signed-off-by: Zhen Kong --- Documentation/devicetree/bindings/qseecom/qseecom.txt | 2 ++ drivers/misc/qseecom.c | 10 +++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/qseecom/qseecom.txt b/Documentation/devicetree/bindings/qseecom/qseecom.txt index c0426a7b082..fcca2e5c834 100644 --- a/Documentation/devicetree/bindings/qseecom/qseecom.txt +++ b/Documentation/devicetree/bindings/qseecom/qseecom.txt @@ -16,6 +16,7 @@ Optional properties: - qcom,support-bus-scaling : indicates if driver support scaling the bus for crypto operation. - qcom,support-fde : indicates if driver support key managing for full disk encryption feature. - qcom,support-pfe : indicates if driver support key managing for per file encryption feature. + - qcom,appsbl-qseecom-support : indicates if there is qseecom support in appsbootloader Example: qcom,qseecom@fe806000 { @@ -32,6 +33,7 @@ Example: qcom,msm_bus,num_cases = <4>; qcom,msm_bus,active_only = <0>; qcom,msm_bus,num_paths = <1>; + qcom,appsbl-qseecom-support; qcom,msm_bus,vectors = <55 512 0 0>, <55 512 3936000000 393600000>, diff --git a/drivers/misc/qseecom.c b/drivers/misc/qseecom.c index c9bf7f8a7a4..42b725f650f 100644 --- a/drivers/misc/qseecom.c +++ b/drivers/misc/qseecom.c @@ -177,6 +177,7 @@ struct qseecom_control { struct work_struct bw_inactive_req_ws; struct cdev cdev; bool timer_running; + bool appsbl_qseecom_support; }; struct qseecom_client_handle { @@ -4444,6 +4445,12 @@ static int __devinit qseecom_probe(struct platform_device *pdev) qseecom.ce_info.qsee_ce_hw_instance); } + qseecom.appsbl_qseecom_support = + of_property_read_bool((&pdev->dev)->of_node, + "qcom,appsbl-qseecom-support"); + pr_info("qseecom.appsbl_qseecom_support = 0x%x", + qseecom.appsbl_qseecom_support); + qseecom.qsee.instance = qseecom.ce_info.qsee_ce_hw_instance; qseecom.ce_drv.instance = qseecom.ce_info.hlos_ce_hw_instance; @@ -4470,7 +4477,8 @@ static int __devinit qseecom_probe(struct platform_device *pdev) qseecom_platform_support = (struct msm_bus_scale_pdata *) msm_bus_cl_get_pdata(pdev); - if (qseecom.qsee_version >= (QSEE_VERSION_02)) { + if (qseecom.qsee_version >= (QSEE_VERSION_02) && + !qseecom.appsbl_qseecom_support) { struct resource *resource = NULL; struct qsee_apps_region_info_ireq req; struct qseecom_command_scm_resp resp; From 17f8de2051bd60fc550abc0fe7221bee41882900 Mon Sep 17 00:00:00 2001 From: Ravinder Konka Date: Fri, 22 Jan 2016 18:08:51 +0530 Subject: [PATCH 080/168] msm8226_defconfig: Enable NFLOG target support StrictModeTest#testCleartextNetwork CTS testcase is failing due to missing NFLOG target support related defconfig. Enabling the defconfig for 8926. Change-Id: I70d9597523a6a500e5b9dde0e6ae3028b4847236 CRs-Fixed: 916129 Signed-off-by: Ravinder Konka --- arch/arm/configs/msm8226-perf_defconfig | 1 + arch/arm/configs/msm8226_defconfig | 1 + 2 files changed, 2 insertions(+) diff --git a/arch/arm/configs/msm8226-perf_defconfig b/arch/arm/configs/msm8226-perf_defconfig index 7a29f307a4a..67f44ed9705 100644 --- a/arch/arm/configs/msm8226-perf_defconfig +++ b/arch/arm/configs/msm8226-perf_defconfig @@ -151,6 +151,7 @@ CONFIG_NETFILTER_XT_TARGET_CONNMARK=y CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=y CONFIG_NETFILTER_XT_TARGET_LOG=y CONFIG_NETFILTER_XT_TARGET_MARK=y +CONFIG_NETFILTER_XT_TARGET_NFLOG=y CONFIG_NETFILTER_XT_TARGET_NFQUEUE=y CONFIG_NETFILTER_XT_TARGET_NOTRACK=y CONFIG_NETFILTER_XT_TARGET_SECMARK=y diff --git a/arch/arm/configs/msm8226_defconfig b/arch/arm/configs/msm8226_defconfig index 8fa182ceb91..992916f50c7 100644 --- a/arch/arm/configs/msm8226_defconfig +++ b/arch/arm/configs/msm8226_defconfig @@ -151,6 +151,7 @@ CONFIG_NETFILTER_XT_TARGET_CONNMARK=y CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=y CONFIG_NETFILTER_XT_TARGET_LOG=y CONFIG_NETFILTER_XT_TARGET_MARK=y +CONFIG_NETFILTER_XT_TARGET_NFLOG=y CONFIG_NETFILTER_XT_TARGET_NFQUEUE=y CONFIG_NETFILTER_XT_TARGET_NOTRACK=y CONFIG_NETFILTER_XT_TARGET_SECMARK=y From 9c5310cff3fa0f31069a27ce3e6670e1f507b6db Mon Sep 17 00:00:00 2001 From: Ravinder Konka Date: Fri, 22 Jan 2016 18:08:51 +0530 Subject: [PATCH 081/168] msm8226_defconfig: Enable NFLOG target support StrictModeTest#testCleartextNetwork CTS testcase is failing due to missing NFLOG target support related defconfig. Enabling the defconfig for 8926. Change-Id: I70d9597523a6a500e5b9dde0e6ae3028b4847236 CRs-Fixed: 916129 Signed-off-by: Ravinder Konka --- arch/arm/configs/msm8226-perf_defconfig | 1 + arch/arm/configs/msm8226_defconfig | 1 + 2 files changed, 2 insertions(+) diff --git a/arch/arm/configs/msm8226-perf_defconfig b/arch/arm/configs/msm8226-perf_defconfig index 7a29f307a4a..67f44ed9705 100644 --- a/arch/arm/configs/msm8226-perf_defconfig +++ b/arch/arm/configs/msm8226-perf_defconfig @@ -151,6 +151,7 @@ CONFIG_NETFILTER_XT_TARGET_CONNMARK=y CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=y CONFIG_NETFILTER_XT_TARGET_LOG=y CONFIG_NETFILTER_XT_TARGET_MARK=y +CONFIG_NETFILTER_XT_TARGET_NFLOG=y CONFIG_NETFILTER_XT_TARGET_NFQUEUE=y CONFIG_NETFILTER_XT_TARGET_NOTRACK=y CONFIG_NETFILTER_XT_TARGET_SECMARK=y diff --git a/arch/arm/configs/msm8226_defconfig b/arch/arm/configs/msm8226_defconfig index 8fa182ceb91..992916f50c7 100644 --- a/arch/arm/configs/msm8226_defconfig +++ b/arch/arm/configs/msm8226_defconfig @@ -151,6 +151,7 @@ CONFIG_NETFILTER_XT_TARGET_CONNMARK=y CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=y CONFIG_NETFILTER_XT_TARGET_LOG=y CONFIG_NETFILTER_XT_TARGET_MARK=y +CONFIG_NETFILTER_XT_TARGET_NFLOG=y CONFIG_NETFILTER_XT_TARGET_NFQUEUE=y CONFIG_NETFILTER_XT_TARGET_NOTRACK=y CONFIG_NETFILTER_XT_TARGET_SECMARK=y From 64cba04f70a670fd46966fb79854041088cba560 Mon Sep 17 00:00:00 2001 From: Zhen Kong Date: Wed, 20 May 2015 13:50:04 -0700 Subject: [PATCH 082/168] qseecom: Register the existing app if it is loaded by appsbl. Register the app in qseecom driver if it has been already loaded by appsbl before Change-Id: Iec39137a7e18dc703c731e55955ab84d1b9c97f3 Signed-off-by: Zhen Kong Signed-off-by: Shreyas Narayan --- drivers/misc/qseecom.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/drivers/misc/qseecom.c b/drivers/misc/qseecom.c index 42b725f650f..a107577b44d 100644 --- a/drivers/misc/qseecom.c +++ b/drivers/misc/qseecom.c @@ -3008,6 +3008,7 @@ static int qseecom_query_app_loaded(struct qseecom_dev_handle *data, struct qseecom_check_app_ireq req; struct qseecom_registered_app_list *entry = NULL; unsigned long flags = 0; + bool found_app = false; /* Copy the relevant information needed for loading the image */ if (copy_from_user(&query_req, @@ -3034,6 +3035,7 @@ static int qseecom_query_app_loaded(struct qseecom_dev_handle *data, &qseecom.registered_app_list_head, list){ if (entry->app_id == ret) { entry->ref_cnt++; + found_app = true; break; } } @@ -3043,6 +3045,29 @@ static int qseecom_query_app_loaded(struct qseecom_dev_handle *data, query_req.app_id = ret; strlcpy(data->client.app_name, query_req.app_name, MAX_APP_NAME_SIZE); + /* + * If app was loaded by appsbl or kernel client before + * and was not registered, regiser this app now. + */ + if (!found_app) { + pr_debug("Register app %d [%s] which was loaded before\n", + ret, (char *)query_req.app_name); + entry = kmalloc(sizeof(*entry), GFP_KERNEL); + if (!entry) { + pr_err("kmalloc for app entry failed\n"); + return -ENOMEM; + } + entry->app_id = ret; + entry->ref_cnt = 1; + strlcpy(entry->app_name, data->client.app_name, + MAX_APP_NAME_SIZE); + spin_lock_irqsave(&qseecom.registered_app_list_lock, + flags); + list_add_tail(&entry->list, + &qseecom.registered_app_list_head); + spin_unlock_irqrestore( + &qseecom.registered_app_list_lock, flags); + } if (copy_to_user(argp, &query_req, sizeof(query_req))) { pr_err("copy_to_user failed\n"); return -EFAULT; From f76a68ca2b789b7e5503ac6f915d554972fe2d5e Mon Sep 17 00:00:00 2001 From: Swetha Chikkaboraiah Date: Wed, 27 Jan 2016 11:46:54 +0530 Subject: [PATCH 083/168] msm: perf: Protect buffer overflow due to malicious user In function krait_pmu_disable_event, parameter hwc comes from userspace and is untrusted.The function krait_clearpmu is called after the function get_krait_evtinfo. Function get_krait_evtinfo as parameter krait_evt_type variable which is used to extract the groupcode(reg) which is bound to KRAIT_MAX_L1_REG (is 3). After validation,one code path modifies groupcode(reg):If this code path executes, groupcode(reg) can be 3,4, 5, or 6. In krait_clearpmu groupcode used to access array krait_functions whose size is 3. Since groupcode can be 3,4,5,6 accessing array krait_functions lead to bufferoverlflow. This change will validate groupcode not to exceed 3 . Change-Id: I48c92adda137d8a074b4e1a367a468195a810ca1 CRs-fixed: 962450 Signed-off-by: Swetha Chikkaboraiah --- arch/arm/kernel/perf_event_msm_krait.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/arm/kernel/perf_event_msm_krait.c b/arch/arm/kernel/perf_event_msm_krait.c index 1c338f79bab..3f09c4c0754 100644 --- a/arch/arm/kernel/perf_event_msm_krait.c +++ b/arch/arm/kernel/perf_event_msm_krait.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2012, 2014 The Linux Foundation. All rights reserved. + * Copyright (c) 2011-2012, 2014,2016 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -219,9 +219,6 @@ static unsigned int get_krait_evtinfo(unsigned int krait_evt_type, code = (krait_evt_type & 0x00FF0) >> 4; group = krait_evt_type & 0x0000F; - if ((group > 3) || (reg > krait_max_l1_reg)) - return -EINVAL; - if (prefix != KRAIT_EVT_PREFIX && prefix != KRAIT_VENUMEVT_PREFIX) return -EINVAL; @@ -232,6 +229,9 @@ static unsigned int get_krait_evtinfo(unsigned int krait_evt_type, reg += VENUM_BASE_OFFSET; } + if ((group > 3) || (reg > krait_max_l1_reg)) + return -EINVAL; + evtinfo->group_setval = 0x80000000 | (code << (group * 8)); evtinfo->groupcode = reg; evtinfo->armv7_evt_type = evt_type_base[evt_index][reg] | group; From 6c0f34bc604f5bd37bda4baa31e3265a1fd925a6 Mon Sep 17 00:00:00 2001 From: Xu Han Date: Mon, 2 Nov 2015 17:38:47 -0800 Subject: [PATCH 084/168] msm: camera: Add NULL check in msm_actuator NULL check on function pointers and actuator control struct is necessary to prevent errors. Change-Id: I212e8558c649e0d432d232dbb99e58b7a064829b Signed-off-by: Xu Han Signed-off-by: VijayaKumar T M --- .../camera_v2/sensor/actuator/msm_actuator.c | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/drivers/media/platform/msm/camera_v2/sensor/actuator/msm_actuator.c b/drivers/media/platform/msm/camera_v2/sensor/actuator/msm_actuator.c index 4f69122b3b3..35997eda8d3 100644 --- a/drivers/media/platform/msm/camera_v2/sensor/actuator/msm_actuator.c +++ b/drivers/media/platform/msm/camera_v2/sensor/actuator/msm_actuator.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2015, The Linux Foundation. All rights reserved. +/* Copyright (c) 2011-2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -75,13 +75,23 @@ static int32_t msm_actuator_piezo_set_default_focus( static void msm_actuator_parse_i2c_params(struct msm_actuator_ctrl_t *a_ctrl, int16_t next_lens_position, uint32_t hw_params, uint16_t delay) { - struct msm_actuator_reg_params_t *write_arr = a_ctrl->reg_tbl; + struct msm_actuator_reg_params_t *write_arr = NULL; uint32_t hw_dword = hw_params; uint16_t i2c_byte1 = 0, i2c_byte2 = 0; uint16_t value = 0; - uint32_t size = a_ctrl->reg_tbl_size, i = 0; - struct msm_camera_i2c_reg_array *i2c_tbl = a_ctrl->i2c_reg_tbl; + uint32_t size = 0, i = 0; + struct msm_camera_i2c_reg_array *i2c_tbl = NULL; CDBG("Enter\n"); + + if (a_ctrl == NULL) { + pr_err("failed. actuator ctrl is NULL"); + return; + } + + size = a_ctrl->reg_tbl_size; + write_arr = a_ctrl->reg_tbl; + i2c_tbl = a_ctrl->i2c_reg_tbl; + for (i = 0; i < size; i++) { /* check that the index into i2c_tbl cannot grow larger that the allocated size of i2c_tbl */ @@ -530,12 +540,23 @@ static int32_t msm_actuator_set_position( return -EFAULT; } + if (!a_ctrl || !a_ctrl->func_tbl || + !a_ctrl->func_tbl->actuator_parse_i2c_params) { + pr_err("failed. NULL actuator pointers."); + return -EFAULT; + } + + if (a_ctrl->actuator_state != ACTUATOR_POWER_UP) { + pr_err("failed. Invalid actuator state."); + return -EFAULT; + } + a_ctrl->i2c_tbl_index = 0; for (index = 0; index < set_pos->number_of_steps; index++) { next_lens_position = set_pos->pos[index]; delay = set_pos->delay[index]; a_ctrl->func_tbl->actuator_parse_i2c_params(a_ctrl, - next_lens_position, hw_params, delay); + next_lens_position, hw_params, delay); reg_setting.reg_setting = a_ctrl->i2c_reg_tbl; reg_setting.size = a_ctrl->i2c_tbl_index; From 117891550ba801c1464ece19d4d55d4dfe493aec Mon Sep 17 00:00:00 2001 From: Kishor PK Date: Fri, 29 Jan 2016 11:13:25 +0530 Subject: [PATCH 085/168] trace: prevent NULL pointer dereference Prevent unintended NULL pointer dereference in trace_event_perf. Change-Id: I35151c460b4350ebd414b67c655684c2019f799f Signed-off-by: Kishor PK Signed-off-by: Srinivasarao P --- kernel/trace/trace_event_perf.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/trace/trace_event_perf.c b/kernel/trace/trace_event_perf.c index fee3752ae8f..3cc9a399830 100644 --- a/kernel/trace/trace_event_perf.c +++ b/kernel/trace/trace_event_perf.c @@ -222,7 +222,8 @@ int perf_trace_add(struct perf_event *p_event, int flags) void perf_trace_del(struct perf_event *p_event, int flags) { struct ftrace_event_call *tp_event = p_event->tp_event; - hlist_del_rcu(&p_event->hlist_entry); + if (!hlist_unhashed(&p_event->hlist_entry)) + hlist_del_rcu(&p_event->hlist_entry); tp_event->class->reg(tp_event, TRACE_REG_PERF_DEL, p_event); } From 777e823462a299981586d19c3cf6f053a411aca7 Mon Sep 17 00:00:00 2001 From: Balamurugan Alagarsamy Date: Thu, 11 Feb 2016 19:28:38 +0530 Subject: [PATCH 086/168] msm: vidc: Fix a race condition in error handling Currently video driver handles session init and clean in one thread and response handling from HW in another thread. Due to race conditions response handler thread may access the session which was already freed by forward thread. Hence synchronize the forward thread and response handler with session_lock. Ex : When driver detects SYS_ERROR, it informs client and client tries to clean the session. In the meantime if HW sends a response, the response handler thread tries to access the inst struct and this struct might have freed as part of session close. This causes use-after-free exception. CRs-Fixed: 765521 Change-Id: I11519087e9c0dcb95587c6a048853e47032bd1be Signed-off-by: Balamurugan Alagarsamy --- .../media/platform/msm/vidc/hfi_response_handler.c | 4 ++-- drivers/media/platform/msm/vidc/msm_vidc.c | 3 ++- drivers/media/platform/msm/vidc/msm_vidc_common.c | 1 - drivers/media/platform/msm/vidc/venus_hfi.c | 13 +++++++++---- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/drivers/media/platform/msm/vidc/hfi_response_handler.c b/drivers/media/platform/msm/vidc/hfi_response_handler.c index 93656810b66..bf69acf3614 100644 --- a/drivers/media/platform/msm/vidc/hfi_response_handler.c +++ b/drivers/media/platform/msm/vidc/hfi_response_handler.c @@ -1302,8 +1302,8 @@ u32 hfi_process_msg_packet( { u32 rc = 0; struct hal_session *sess = NULL; - if (!callback || !msg_hdr || msg_hdr->size < - VIDC_IFACEQ_MIN_PKT_SIZE) { + if (!callback || !session_lock || !msg_hdr || + msg_hdr->size < VIDC_IFACEQ_MIN_PKT_SIZE) { dprintk(VIDC_ERR, "hal_process_msg_packet:bad" "packet/packet size: %d", msg_hdr->size); rc = -EINVAL; diff --git a/drivers/media/platform/msm/vidc/msm_vidc.c b/drivers/media/platform/msm/vidc/msm_vidc.c index 0a4a4660530..0b279cc02fc 100644 --- a/drivers/media/platform/msm/vidc/msm_vidc.c +++ b/drivers/media/platform/msm/vidc/msm_vidc.c @@ -1378,7 +1378,6 @@ int msm_vidc_close(void *instance) } core = inst->core; - msm_comm_session_clean(inst); mutex_lock(&core->lock); list_for_each_safe(ptr, next, &core->instances) { @@ -1406,6 +1405,8 @@ int msm_vidc_close(void *instance) dprintk(VIDC_ERR, "Failed to move video instance to uninit state\n"); + msm_comm_session_clean(inst); + msm_smem_delete_client(inst->mem_client); pr_info(VIDC_DBG_TAG "Closed video instance: %p\n", VIDC_INFO, inst); kfree(inst); diff --git a/drivers/media/platform/msm/vidc/msm_vidc_common.c b/drivers/media/platform/msm/vidc/msm_vidc_common.c index dc925d82b1e..1f7d2f367ca 100644 --- a/drivers/media/platform/msm/vidc/msm_vidc_common.c +++ b/drivers/media/platform/msm/vidc/msm_vidc_common.c @@ -1000,7 +1000,6 @@ static void handle_session_close(enum command_response cmd, void *data) dprintk(VIDC_ERR, "%s invalid params\n", __func__); return; } - msm_comm_session_clean(inst); signal_session_msg_receipt(cmd, inst); show_stats(inst); } else { diff --git a/drivers/media/platform/msm/vidc/venus_hfi.c b/drivers/media/platform/msm/vidc/venus_hfi.c index a54f33f7e72..b8656536b61 100644 --- a/drivers/media/platform/msm/vidc/venus_hfi.c +++ b/drivers/media/platform/msm/vidc/venus_hfi.c @@ -136,6 +136,11 @@ static void venus_hfi_sim_modify_cmd_packet(u8 *packet) sys_init = (struct hfi_cmd_sys_session_init_packet *)packet; sess = (struct hal_session *) sys_init->session_id; + if (!sess) { + dprintk(VIDC_DBG, "%s :Invalid session id : %x\n", + __func__, sys_init->session_id); + return; + } switch (sys_init->packet_type) { case HFI_CMD_SESSION_EMPTY_BUFFER: if (sess->is_decoder) { @@ -2495,19 +2500,19 @@ static int venus_hfi_session_abort(void *session) static int venus_hfi_session_clean(void *session) { struct hal_session *sess_close; + struct venus_hfi_device *device; if (!session) { dprintk(VIDC_ERR, "Invalid Params %s", __func__); return -EINVAL; } sess_close = session; + device = sess_close->device; dprintk(VIDC_DBG, "deleted the session: 0x%p", sess_close); - mutex_lock(&((struct venus_hfi_device *) - sess_close->device)->session_lock); + mutex_lock(&device->session_lock); list_del(&sess_close->list); - mutex_unlock(&((struct venus_hfi_device *) - sess_close->device)->session_lock); kfree(sess_close); + mutex_unlock(&device->session_lock); return 0; } From ac13c00f94a4f834dca17f67b6effac70754f33c Mon Sep 17 00:00:00 2001 From: Balamurugan Alagarsamy Date: Wed, 4 Nov 2015 15:43:16 +0530 Subject: [PATCH 087/168] msm: vidc: skip fps round off when client sets 1 fps sometimes fps is rounded off one lesser, makes it to zero, when the client is setting fps as 1. This exposes division by zero crash when calculating the load for non realtime usescases. Crash is observed in recent monkey runs. Change-Id: I4ec824d658ed3a14ab43ca031e989b01adfddc11 CRs-Fixed: 933641 Signed-off-by: Balamurugan Alagarsamy --- drivers/media/platform/msm/vidc/msm_vdec.c | 2 +- drivers/media/platform/msm/vidc/msm_venc.c | 2 +- drivers/media/platform/msm/vidc/msm_vidc_common.c | 9 +++++++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/media/platform/msm/vidc/msm_vdec.c b/drivers/media/platform/msm/vidc/msm_vdec.c index a30aa42cdd4..2cf59c6626f 100644 --- a/drivers/media/platform/msm/vidc/msm_vdec.c +++ b/drivers/media/platform/msm/vidc/msm_vdec.c @@ -934,7 +934,7 @@ int msm_vdec_s_parm(struct msm_vidc_inst *inst, struct v4l2_streamparm *a) if ((fps % 15 == 14) || (fps % 24 == 23)) fps = fps + 1; - else if ((fps % 24 == 1) || (fps % 15 == 1)) + else if ((fps > 1) && ((fps % 24 == 1) || (fps % 15 == 1))) fps = fps - 1; if (inst->prop.fps != fps) { diff --git a/drivers/media/platform/msm/vidc/msm_venc.c b/drivers/media/platform/msm/vidc/msm_venc.c index 814d161e052..c6fc3a69cc7 100644 --- a/drivers/media/platform/msm/vidc/msm_venc.c +++ b/drivers/media/platform/msm/vidc/msm_venc.c @@ -2571,7 +2571,7 @@ int msm_venc_s_parm(struct msm_vidc_inst *inst, struct v4l2_streamparm *a) if ((fps % 15 == 14) || (fps % 24 == 23)) fps = fps + 1; - else if ((fps % 24 == 1) || (fps % 15 == 1)) + else if ((fps > 1) && ((fps % 24 == 1) || (fps % 15 == 1))) fps = fps - 1; if (inst->prop.fps != fps) { diff --git a/drivers/media/platform/msm/vidc/msm_vidc_common.c b/drivers/media/platform/msm/vidc/msm_vidc_common.c index 1f7d2f367ca..8558f3b3b71 100644 --- a/drivers/media/platform/msm/vidc/msm_vidc_common.c +++ b/drivers/media/platform/msm/vidc/msm_vidc_common.c @@ -140,8 +140,13 @@ static int msm_comm_get_inst_load(struct msm_vidc_inst *inst, } if (is_non_realtime_session(inst) && - (quirks & LOAD_CALC_IGNORE_NON_REALTIME_LOAD)) - load = msm_comm_get_mbs_per_sec(inst) / inst->prop.fps; + (quirks & LOAD_CALC_IGNORE_NON_REALTIME_LOAD)) { + if (!inst->prop.fps) { + dprintk(VIDC_INFO, "%s: instance:%p prop->fps is set 0\n", __func__, inst); + load = 0; + } else + load = msm_comm_get_mbs_per_sec(inst) / inst->prop.fps; + } return load; } From 6885e0ba5e81adf6525d453406af3c99d693a147 Mon Sep 17 00:00:00 2001 From: Laxminath Kasam Date: Thu, 26 Nov 2015 10:11:07 +0530 Subject: [PATCH 088/168] avtimer: update ioctl call to query avtimer current avtimer ioctl uses deprecated logic to query avtimer. Use the updated API to query avtimer to avoid wrong timestamp values when query using ioctl interface. Change-Id: I16e4373226903b6121c911ed9f2ca31851de80e5 Signed-off-by: Laxminath Kasam --- drivers/platform/msm/avtimer.c | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/drivers/platform/msm/avtimer.c b/drivers/platform/msm/avtimer.c index bb9b2bfc75b..8d73f144289 100644 --- a/drivers/platform/msm/avtimer.c +++ b/drivers/platform/msm/avtimer.c @@ -307,7 +307,8 @@ int avcs_core_query_timer(uint64_t *avtimer_tick) | avtimer_lsw; res = do_div(avtimer_tick_temp, avtimer.clk_div); *avtimer_tick = avtimer_tick_temp; - pr_debug("%s:Avtimer: msw: %u, lsw: %u, tick: %llu\n", __func__, + pr_debug_ratelimited("%s:Avtimer: msw: %u, lsw: %u, tick: %llu\n", + __func__, avtimer_msw, avtimer_lsw, *avtimer_tick); return 0; } @@ -332,21 +333,11 @@ static long avtimer_ioctl(struct file *file, unsigned int ioctl_num, switch (ioctl_num) { case IOCTL_GET_AVTIMER_TICK: { - uint32_t avtimer_msw_1st = 0, avtimer_lsw = 0; - uint32_t avtimer_msw_2nd = 0; uint64_t avtimer_tick; - do { - avtimer_msw_1st = ioread32(avtimer.p_avtimer_msw); - avtimer_lsw = ioread32(avtimer.p_avtimer_lsw); - avtimer_msw_2nd = ioread32(avtimer.p_avtimer_msw); - } while (avtimer_msw_1st != avtimer_msw_2nd); - avtimer_lsw = avtimer_lsw/avtimer.clk_div; - avtimer_tick = - ((uint64_t) avtimer_msw_1st << 32) | avtimer_lsw; - - pr_debug("%s: AV Timer tick: msw: %x, lsw: %x time %llx\n", - __func__, avtimer_msw_1st, avtimer_lsw, avtimer_tick); + avcs_core_query_timer(&avtimer_tick); + pr_debug_ratelimited("%s: AV Timer tick: time %llx\n", + __func__, avtimer_tick); if (copy_to_user((void *) ioctl_param, &avtimer_tick, sizeof(avtimer_tick))) { pr_err("copy_to_user failed\n"); From 9b298d5198fcffbd25759c9d9d19114f6bd5a078 Mon Sep 17 00:00:00 2001 From: Kishor PK Date: Thu, 18 Feb 2016 15:26:50 +0530 Subject: [PATCH 089/168] msm: perf: validate input argument of ev_constraints functions Validate input argument before writing into pmu_constraints_codes array. CRs-Fixed: 975404 Change-Id: Id68b1d2201ab1af783af2236833b1dc894e08cc7 Signed-off-by: Kishor PK --- arch/arm/mach-msm/perf_event_msm_krait_l2.c | 23 +++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/arch/arm/mach-msm/perf_event_msm_krait_l2.c b/arch/arm/mach-msm/perf_event_msm_krait_l2.c index c9e2b8f8faf..562e90e4737 100644 --- a/arch/arm/mach-msm/perf_event_msm_krait_l2.c +++ b/arch/arm/mach-msm/perf_event_msm_krait_l2.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2013 The Linux Foundation. All rights reserved. + * Copyright (c) 2011-2013,2016 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -18,13 +18,15 @@ #include +#define PMU_CODES_SIZE 64 + /* * The L2 PMU is shared between all CPU's, so protect * its bitmap access. */ struct pmu_constraints { u64 pmu_bitmap; - u8 codes[64]; + u8 codes[PMU_CODES_SIZE]; raw_spinlock_t lock; } l2_pmu_constraints = { .pmu_bitmap = 0, @@ -419,10 +421,9 @@ static int msm_l2_test_set_ev_constraint(struct perf_event *event) u8 group = evt_type & 0x0000F; u8 code = (evt_type & 0x00FF0) >> 4; unsigned long flags; - u32 err = 0; + int err = 0; u64 bitmap_t; u32 shift_idx; - if (evt_prefix == L2_TRACECTR_PREFIX) return err; /* @@ -436,6 +437,11 @@ static int msm_l2_test_set_ev_constraint(struct perf_event *event) shift_idx = ((reg * 4) + group); + if (shift_idx >= PMU_CODES_SIZE) { + err = -EINVAL; + goto out; + } + bitmap_t = 1 << shift_idx; if (!(l2_pmu_constraints.pmu_bitmap & bitmap_t)) { @@ -476,6 +482,7 @@ static int msm_l2_clear_ev_constraint(struct perf_event *event) unsigned long flags; u64 bitmap_t; u32 shift_idx; + int err = 1; if (evt_prefix == L2_TRACECTR_PREFIX) return 1; @@ -483,6 +490,10 @@ static int msm_l2_clear_ev_constraint(struct perf_event *event) shift_idx = ((reg * 4) + group); + if (shift_idx >= PMU_CODES_SIZE) { + err = -EINVAL; + goto out; + } bitmap_t = 1 << shift_idx; /* Clear constraint bit. */ @@ -490,9 +501,9 @@ static int msm_l2_clear_ev_constraint(struct perf_event *event) /* Clear code. */ l2_pmu_constraints.codes[shift_idx] = -1; - +out: raw_spin_unlock_irqrestore(&l2_pmu_constraints.lock, flags); - return 1; + return err; } int get_num_events(void) From b01f6f93c3a5d422da869a4eebe0de078097fb05 Mon Sep 17 00:00:00 2001 From: George Cherian Date: Thu, 25 Feb 2016 14:22:32 +0530 Subject: [PATCH 090/168] usb: host: xhci: Enable XHCI_SPURIOUS_SUCCESS for all controllers with xhci 1.0 commit 07f3cb7c28bf3f4dd80bfb136cf45810c46ac474 upstream. Xhci controllers with hci_version > 0.96 gives spurious success events on short packet completion. During webcam capture the "ERROR Transfer event TRB DMA ptr not part of current TD" was observed. The same application works fine with synopsis controllers hci_version 0.96. The same issue is seen with Intel Pantherpoint xhci controller. So enabling this quirk in xhci_gen_setup if controller verion is greater than 0.96. For xhci-pci move the quirk to much generic place xhci_gen_setup. Note from Sarah: The xHCI 1.0 spec changed how hardware handles short packets. The HW will notify SW of the TRB where the short packet occurred, and it will also give a successful status for the last TRB in a TD (the one with the IOC flag set). On the second successful status, that warning will be triggered in the driver. Software is now supposed to not assume the TD is not completed until it gets that last successful status. That means we have a slight race condition, although it should have little practical impact. This patch papers over that issue. It's on my long-term to-do list to fix this race condition, but it is a much more involved patch that will probably be too big for stable. This patch is needed for stable to avoid serious log spam. This patch should be backported to kernels as old as 3.0, that contain the commit ad808333d8201d53075a11bc8dd83b81f3d68f0b "Intel xhci: Ignore spurious successful event." The patch will have to be modified for kernels older than 3.2, since that kernel added the xhci_gen_setup function for xhci platform devices. The correct conflict resolution for kernels older than 3.2 is to set XHCI_SPURIOUS_SUCCESS in xhci_pci_quirks for all xHCI 1.0 hosts. Signed-off-by: George Cherian Signed-off-by: Sarah Sharp Signed-off-by: Greg Kroah-Hartman Change-Id: I309533a7b6338fa0fa353f99a61ba94622291640 Git-commit: a875d82eda58712d81caf78ebb515fb072187138 Signed-off-by: Vijayavardhan Vennapusa --- drivers/usb/host/xhci-pci.c | 1 - drivers/usb/host/xhci.c | 7 +++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/usb/host/xhci-pci.c b/drivers/usb/host/xhci-pci.c index 7a856a767e7..8b1874d0314 100644 --- a/drivers/usb/host/xhci-pci.c +++ b/drivers/usb/host/xhci-pci.c @@ -85,7 +85,6 @@ static void xhci_pci_quirks(struct device *dev, struct xhci_hcd *xhci) xhci->quirks |= XHCI_AMD_PLL_FIX; if (pdev->vendor == PCI_VENDOR_ID_INTEL && pdev->device == PCI_DEVICE_ID_INTEL_PANTHERPOINT_XHCI) { - xhci->quirks |= XHCI_SPURIOUS_SUCCESS; xhci->quirks |= XHCI_EP_LIMIT_QUIRK; xhci->limit_active_eps = 64; xhci->quirks |= XHCI_SW_BW_CHECKING; diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c index f8bc1c50616..5ab13243a25 100644 --- a/drivers/usb/host/xhci.c +++ b/drivers/usb/host/xhci.c @@ -4047,6 +4047,13 @@ int xhci_gen_setup(struct usb_hcd *hcd, xhci_get_quirks_t get_quirks) get_quirks(dev, xhci); + /* In xhci controllers which follow xhci 1.0 spec gives a spurious + * success event after a short transfer. This quirk will ignore such + * spurious event. + */ + if (xhci->hci_version > 0x96) + xhci->quirks |= XHCI_SPURIOUS_SUCCESS; + /* Make sure the HC is halted. */ retval = xhci_halt(xhci); if (retval) From 0abc3d7ab9b71e2e13d4a489cc062e0cae95eb64 Mon Sep 17 00:00:00 2001 From: George Cherian Date: Thu, 25 Feb 2016 14:22:32 +0530 Subject: [PATCH 091/168] usb: host: xhci: Enable XHCI_SPURIOUS_SUCCESS for all controllers with xhci 1.0 commit 07f3cb7c28bf3f4dd80bfb136cf45810c46ac474 upstream. Xhci controllers with hci_version > 0.96 gives spurious success events on short packet completion. During webcam capture the "ERROR Transfer event TRB DMA ptr not part of current TD" was observed. The same application works fine with synopsis controllers hci_version 0.96. The same issue is seen with Intel Pantherpoint xhci controller. So enabling this quirk in xhci_gen_setup if controller verion is greater than 0.96. For xhci-pci move the quirk to much generic place xhci_gen_setup. Note from Sarah: The xHCI 1.0 spec changed how hardware handles short packets. The HW will notify SW of the TRB where the short packet occurred, and it will also give a successful status for the last TRB in a TD (the one with the IOC flag set). On the second successful status, that warning will be triggered in the driver. Software is now supposed to not assume the TD is not completed until it gets that last successful status. That means we have a slight race condition, although it should have little practical impact. This patch papers over that issue. It's on my long-term to-do list to fix this race condition, but it is a much more involved patch that will probably be too big for stable. This patch is needed for stable to avoid serious log spam. This patch should be backported to kernels as old as 3.0, that contain the commit ad808333d8201d53075a11bc8dd83b81f3d68f0b "Intel xhci: Ignore spurious successful event." The patch will have to be modified for kernels older than 3.2, since that kernel added the xhci_gen_setup function for xhci platform devices. The correct conflict resolution for kernels older than 3.2 is to set XHCI_SPURIOUS_SUCCESS in xhci_pci_quirks for all xHCI 1.0 hosts. Signed-off-by: George Cherian Signed-off-by: Sarah Sharp Signed-off-by: Greg Kroah-Hartman Change-Id: I309533a7b6338fa0fa353f99a61ba94622291640 Git-commit: a875d82eda58712d81caf78ebb515fb072187138 Signed-off-by: Vijayavardhan Vennapusa --- drivers/usb/host/xhci-pci.c | 1 - drivers/usb/host/xhci.c | 7 +++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/usb/host/xhci-pci.c b/drivers/usb/host/xhci-pci.c index 7a856a767e7..8b1874d0314 100644 --- a/drivers/usb/host/xhci-pci.c +++ b/drivers/usb/host/xhci-pci.c @@ -85,7 +85,6 @@ static void xhci_pci_quirks(struct device *dev, struct xhci_hcd *xhci) xhci->quirks |= XHCI_AMD_PLL_FIX; if (pdev->vendor == PCI_VENDOR_ID_INTEL && pdev->device == PCI_DEVICE_ID_INTEL_PANTHERPOINT_XHCI) { - xhci->quirks |= XHCI_SPURIOUS_SUCCESS; xhci->quirks |= XHCI_EP_LIMIT_QUIRK; xhci->limit_active_eps = 64; xhci->quirks |= XHCI_SW_BW_CHECKING; diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c index f8bc1c50616..5ab13243a25 100644 --- a/drivers/usb/host/xhci.c +++ b/drivers/usb/host/xhci.c @@ -4047,6 +4047,13 @@ int xhci_gen_setup(struct usb_hcd *hcd, xhci_get_quirks_t get_quirks) get_quirks(dev, xhci); + /* In xhci controllers which follow xhci 1.0 spec gives a spurious + * success event after a short transfer. This quirk will ignore such + * spurious event. + */ + if (xhci->hci_version > 0x96) + xhci->quirks |= XHCI_SPURIOUS_SUCCESS; + /* Make sure the HC is halted. */ retval = xhci_halt(xhci); if (retval) From 15445a9cb2f476fd1aafa7541f20bff631afaf39 Mon Sep 17 00:00:00 2001 From: Vinayak Menon Date: Fri, 26 Dec 2014 19:29:41 +0530 Subject: [PATCH 092/168] mm: vmscan: fix the page state calculation in too_many_isolated It is observed that sometimes multiple tasks get blocked in the congestion_wait loop below, in shrink_inactive_list. (__schedule) from [] (schedule_timeout) from [] (io_schedule_timeout) from [] (congestion_wait) from [] (shrink_inactive_list) from [] (shrink_zone) from [] (try_to_free_pages) from [] (__alloc_pages_nodemask) from [] (new_slab) from [] (__slab_alloc) from [] In one such instance, zone_page_state(zone, NR_ISOLATED_FILE) had returned 14, zone_page_state(zone, NR_INACTIVE_FILE) returned 92, and the gfp_flag was GFP_KERNEL which resulted in too_many_isolated to return true. But one of the CPU pageset vmstat diff had NR_ISOLATED_FILE as -14. As there weren't any more update to per cpu pageset, the threshold wasn't met, and the tasks were blocked in the congestion wait. This patch uses zone_page_state_snapshot instead, but restricts its usage to avoid performance penalty. Change-Id: Iec767a548e524729c7ed79a92fe4718cdd08ce69 Signed-off-by: Vinayak Menon Signed-off-by: Srinivasarao P --- mm/vmscan.c | 60 +++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 11 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index e2e43aaff6c..b1f791bd5b8 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1195,29 +1195,64 @@ int isolate_lru_page(struct page *page) return ret; } +static int __too_many_isolated(struct zone *zone, int file, + struct scan_control *sc, int safe) +{ + unsigned long inactive, isolated; + + if (file) { + if (safe) { + inactive = zone_page_state_snapshot(zone, + NR_INACTIVE_FILE); + isolated = zone_page_state_snapshot(zone, + NR_ISOLATED_FILE); + } else { + inactive = zone_page_state(zone, NR_INACTIVE_FILE); + isolated = zone_page_state(zone, NR_ISOLATED_FILE); + } + } else { + if (safe) { + inactive = zone_page_state_snapshot(zone, + NR_INACTIVE_ANON); + isolated = zone_page_state_snapshot(zone, + NR_ISOLATED_ANON); + } else { + inactive = zone_page_state(zone, NR_INACTIVE_ANON); + isolated = zone_page_state(zone, NR_ISOLATED_ANON); + } + } + + /* + * GFP_NOIO/GFP_NOFS callers are allowed to isolate more pages, so they + * won't get blocked by normal direct-reclaimers, forming a circular + * deadlock. + */ + if ((sc->gfp_mask & GFP_IOFS) == GFP_IOFS) + inactive >>= 3; + + return isolated > inactive; +} + /* * Are there way too many processes in the direct reclaim path already? */ static int too_many_isolated(struct zone *zone, int file, - struct scan_control *sc) + struct scan_control *sc, int safe) { - unsigned long inactive, isolated; - if (current_is_kswapd()) return 0; if (!global_reclaim(sc)) return 0; - if (file) { - inactive = zone_page_state(zone, NR_INACTIVE_FILE); - isolated = zone_page_state(zone, NR_ISOLATED_FILE); - } else { - inactive = zone_page_state(zone, NR_INACTIVE_ANON); - isolated = zone_page_state(zone, NR_ISOLATED_ANON); + if (unlikely(__too_many_isolated(zone, file, sc, 0))) { + if (safe) + return __too_many_isolated(zone, file, sc, safe); + else + return 1; } - return isolated > inactive; + return 0; } static noinline_for_stack void @@ -1287,16 +1322,19 @@ shrink_inactive_list(unsigned long nr_to_scan, struct mem_cgroup_zone *mz, unsigned long nr_writeback = 0; isolate_mode_t isolate_mode = 0; int file = is_file_lru(lru); + int safe = 0; struct zone *zone = mz->zone; struct zone_reclaim_stat *reclaim_stat = get_reclaim_stat(mz); struct lruvec *lruvec = mem_cgroup_zone_lruvec(zone, mz->mem_cgroup); - while (unlikely(too_many_isolated(zone, file, sc))) { + while (unlikely(too_many_isolated(zone, file, sc, safe))) { congestion_wait(BLK_RW_ASYNC, HZ/10); /* We are about to die and free our memory. Return now. */ if (fatal_signal_pending(current)) return SWAP_CLUSTER_MAX; + + safe = 1; } lru_add_drain(); From 927d2fde1f4235cb9377c031d0d9f7587f223355 Mon Sep 17 00:00:00 2001 From: Rama Krishna Phani A Date: Wed, 6 May 2015 20:58:27 +0530 Subject: [PATCH 093/168] soc: qcom: bam_dmux: Use SPS hold mem flag Bam_dmux sometimes calls sps_connect() during time sensitive operations. Sps_connect() internally has some memory allocations which may stall during low memory conditions and can cause a timeout in bam_dmux. SPS supports a configuration flag which changes these memory allocations to hold memory. Use this flag to prevent allocations from stalling and leading to timeouts. Change-Id: I506d5664e07baf56f4caa10d1868b1795a4439d9 Signed-off-by: Rama Krishna Phani A Signed-off-by: Srinivasarao P --- arch/arm/mach-msm/bam_dmux.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/mach-msm/bam_dmux.c b/arch/arm/mach-msm/bam_dmux.c index 084f377017b..79634412865 100644 --- a/arch/arm/mach-msm/bam_dmux.c +++ b/arch/arm/mach-msm/bam_dmux.c @@ -2084,7 +2084,7 @@ static int bam_init(void) a2_props.virt_addr = a2_virt_addr; a2_props.virt_size = a2_phys_size; a2_props.irq = a2_bam_irq; - a2_props.options = SPS_BAM_OPT_IRQ_WAKEUP; + a2_props.options = SPS_BAM_OPT_IRQ_WAKEUP | SPS_BAM_HOLD_MEM; a2_props.num_pipes = A2_NUM_PIPES; a2_props.summing_threshold = A2_SUMMING_THRESHOLD; a2_props.constrained_logging = true; From abf7d198bcc0b51683e3d420109f66f7d317f46a Mon Sep 17 00:00:00 2001 From: Rama Krishna Phani A Date: Wed, 6 May 2015 17:28:20 +0530 Subject: [PATCH 094/168] msm: sps: Handle low memory condition in SPS driver In low memory condition some bam client is sensitive to timing and may get time out while allocating memory through kzalloc. Add support to hold memory during connect and disconnect and to release memory during client driver's deregistration of its BAM. Change-Id: Iaf89dc542b62c95f055e7b06d788d896809b0bf8 Signed-off-by: Rama Krishna Phani A Signed-off-by: Srinivasarao P --- arch/arm/mach-msm/include/mach/sps.h | 2 ++ drivers/platform/msm/sps/sps.c | 7 ++++ drivers/platform/msm/sps/sps_bam.c | 52 +++++++++++++++++++--------- drivers/platform/msm/sps/sps_bam.h | 5 ++- 4 files changed, 48 insertions(+), 18 deletions(-) diff --git a/arch/arm/mach-msm/include/mach/sps.h b/arch/arm/mach-msm/include/mach/sps.h index 6cd4a2ee8cb..e35f29a08b8 100644 --- a/arch/arm/mach-msm/include/mach/sps.h +++ b/arch/arm/mach-msm/include/mach/sps.h @@ -109,6 +109,8 @@ #define SPS_BAM_NO_LOCAL_CLK_GATING (1UL << 5) /* Don't enable writeback cancel*/ #define SPS_BAM_CANCEL_WB (1UL << 6) +/* Hold memory for BAM DMUX */ +#define SPS_BAM_HOLD_MEM (1UL << 8) /* BAM device management flags */ diff --git a/drivers/platform/msm/sps/sps.c b/drivers/platform/msm/sps/sps.c index 55a5a2d7c27..a00ed3eace4 100644 --- a/drivers/platform/msm/sps/sps.c +++ b/drivers/platform/msm/sps/sps.c @@ -2193,6 +2193,7 @@ EXPORT_SYMBOL(sps_register_bam_device); int sps_deregister_bam_device(u32 dev_handle) { struct sps_bam *bam; + int n; SPS_DBG2("sps:%s.", __func__); @@ -2209,6 +2210,12 @@ int sps_deregister_bam_device(u32 dev_handle) SPS_DBG2("sps:SPS deregister BAM: phys 0x%x.", bam->props.phys_addr); + if (bam->props.options & SPS_BAM_HOLD_MEM) { + for (n = 0; n < BAM_MAX_PIPES; n++) + if (bam->desc_cache_pointers[n] != NULL) + kfree(bam->desc_cache_pointers[n]); + } + /* If this BAM is attached to a BAM-DMA, init the BAM-DMA device */ #ifdef CONFIG_SPS_SUPPORT_BAMDMA if ((bam->props.options & SPS_BAM_OPT_BAMDMA)) { diff --git a/drivers/platform/msm/sps/sps_bam.c b/drivers/platform/msm/sps/sps_bam.c index ad5e2419d4a..09c4f1eaba7 100644 --- a/drivers/platform/msm/sps/sps_bam.c +++ b/drivers/platform/msm/sps/sps_bam.c @@ -959,10 +959,15 @@ int sps_bam_pipe_disconnect(struct sps_bam *dev, u32 pipe_index) bam_pipe_exit(dev->base, pipe_index, dev->props.ee); if (pipe->sys.desc_cache != NULL) { u32 size = pipe->num_descs * sizeof(void *); - if (pipe->desc_size + size <= PAGE_SIZE) - kfree(pipe->sys.desc_cache); - else + if (pipe->desc_size + size <= PAGE_SIZE) { + if (dev->props.options & SPS_BAM_HOLD_MEM) + memset(pipe->sys.desc_cache, 0, + pipe->desc_size + size); + else + kfree(pipe->sys.desc_cache); + } else { vfree(pipe->sys.desc_cache); + } pipe->sys.desc_cache = NULL; } dev->pipes[pipe_index] = BAM_PIPE_UNASSIGNED; @@ -1086,30 +1091,43 @@ int sps_bam_pipe_set_params(struct sps_bam *dev, u32 pipe_index, u32 options) /* Allocate both descriptor cache and user pointer array */ size = pipe->num_descs * sizeof(void *); - if (pipe->desc_size + size <= PAGE_SIZE) - pipe->sys.desc_cache = - kzalloc(pipe->desc_size + size, GFP_KERNEL); - else { + if (pipe->desc_size + size <= PAGE_SIZE) { + if ((dev->props.options & + SPS_BAM_HOLD_MEM)) { + if (dev->desc_cache_pointers[pipe_index]) { + pipe->sys.desc_cache = + dev->desc_cache_pointers + [pipe_index]; + } else { + pipe->sys.desc_cache = + kzalloc(pipe->desc_size + size, + GFP_KERNEL); + dev->desc_cache_pointers[pipe_index] = + pipe->sys.desc_cache; + } + } else { + pipe->sys.desc_cache = + kzalloc(pipe->desc_size + size, + GFP_KERNEL); + } + if (pipe->sys.desc_cache == NULL) { + SPS_ERR("sps:No memory for pipe%d of BAM 0x%x\n", + pipe_index, BAM_ID(dev)); + return -ENOMEM; + } + } else { pipe->sys.desc_cache = vmalloc(pipe->desc_size + size); if (pipe->sys.desc_cache == NULL) { - SPS_ERR( - "sps:No memory for pipe %d of BAM 0x%x\n", - pipe_index, BAM_ID(dev)); + SPS_ERR("sps:No memory for pipe %d of BAM 0x%x\n", + pipe_index, BAM_ID(dev)); return -ENOMEM; } memset(pipe->sys.desc_cache, 0, pipe->desc_size + size); } - if (pipe->sys.desc_cache == NULL) { - /*** MUST BE LAST POINT OF FAILURE (see below) *****/ - SPS_ERR("sps:Desc cache error: BAM 0x%x pipe %d: %d\n", - BAM_ID(dev), pipe_index, - pipe->desc_size + size); - return SPS_ERROR; - } pipe->sys.user_ptrs = (void **)(pipe->sys.desc_cache + pipe->desc_size); pipe->sys.cache_offset = pipe->sys.acked_offset; diff --git a/drivers/platform/msm/sps/sps_bam.h b/drivers/platform/msm/sps/sps_bam.h index a20156b4f93..84c01d8715d 100644 --- a/drivers/platform/msm/sps/sps_bam.h +++ b/drivers/platform/msm/sps/sps_bam.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2014, The Linux Foundation. All rights reserved. +/* Copyright (c) 2011-2015, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -214,6 +214,9 @@ struct sps_bam { u32 irq_from_disabled_pipe; u32 event_trigger_failures; + /* Desc cache pointers */ + u8 *desc_cache_pointers[BAM_MAX_PIPES]; + }; /** From 96afa97d12f6fb788d6c2dabd3923a68b942cef8 Mon Sep 17 00:00:00 2001 From: Pavan Anamula Date: Tue, 8 Mar 2016 12:55:37 +0530 Subject: [PATCH 095/168] Revert "Revert "mmc: queue: exclude asynchronous transfer for special request"" This reverts commit 95d90a5c6210e08a3ff81a0ab7d8b26f8f6ac6d9. Change-Id: Ib925bb8d7dbcf0fc623db74a7a72e9bdf9e97166 Signed-off-by: Pavan Anamula --- drivers/mmc/card/queue.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/mmc/card/queue.c b/drivers/mmc/card/queue.c index cceb8d96318..fd57f6e3d84 100644 --- a/drivers/mmc/card/queue.c +++ b/drivers/mmc/card/queue.c @@ -24,6 +24,8 @@ #define MMC_QUEUE_BOUNCESZ 65536 +#define MMC_REQ_SPECIAL_MASK (REQ_DISCARD | REQ_FLUSH) + /* * Based on benchmark tests the default num of requests to trigger the write * packing was determined, to keep the read latency as low as possible and @@ -66,6 +68,7 @@ static int mmc_queue_thread(void *d) do { struct mmc_queue_req *tmp; struct request *req = NULL; + unsigned int cmd_flags = 0; spin_lock_irq(q->queue_lock); set_current_state(TASK_INTERRUPTIBLE); @@ -75,6 +78,7 @@ static int mmc_queue_thread(void *d) if (req || mq->mqrq_prev->req) { set_current_state(TASK_RUNNING); + cmd_flags = req ? req->cmd_flags : 0; mq->issue_fn(mq, req); if (test_bit(MMC_QUEUE_NEW_REQUEST, &mq->flags)) { continue; /* fetch again */ @@ -95,7 +99,13 @@ static int mmc_queue_thread(void *d) /* * Current request becomes previous request * and vice versa. + * In case of special requests, current request + * has been finished. Do not assign it to previous + * request. */ + if (cmd_flags & MMC_REQ_SPECIAL_MASK) + mq->mqrq_cur->req = NULL; + mq->mqrq_prev->brq.mrq.data = NULL; mq->mqrq_prev->req = NULL; tmp = mq->mqrq_prev; From 244bf389b16e85fbe110a479e759bc8b007c75bd Mon Sep 17 00:00:00 2001 From: Pavan Anamula Date: Tue, 8 Mar 2016 13:06:56 +0530 Subject: [PATCH 096/168] Revert "Revert "mmc: queue: use cached attributes of special requests"" This reverts commit a0b88907e71207dd5c31ca94ae937558ffdab22f. Change-Id: I0569e89508b64244be31d0d32a9593eb9f09a573 Signed-off-by: Pavan Anamula --- drivers/mmc/card/queue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/mmc/card/queue.c b/drivers/mmc/card/queue.c index fd57f6e3d84..4edea18b2b0 100644 --- a/drivers/mmc/card/queue.c +++ b/drivers/mmc/card/queue.c @@ -84,7 +84,7 @@ static int mmc_queue_thread(void *d) continue; /* fetch again */ } else if (test_bit(MMC_QUEUE_URGENT_REQUEST, &mq->flags) && (mq->mqrq_cur->req && - !(mq->mqrq_cur->req->cmd_flags & + !(cmd_flags & MMC_REQ_NOREINSERT_MASK))) { /* * clean current request when urgent request From 31ab7dc66c2ff4f02ddd3451d7179c3fa282dff9 Mon Sep 17 00:00:00 2001 From: Pavan Anamula Date: Wed, 10 Feb 2016 18:28:22 +0530 Subject: [PATCH 097/168] mmc: fix host release issue after discard/flush operation Under function mmc_blk_issue_rq, relese the host when there are no more requests after special request(discard, flush), To fix reboot stuck issue. Change-Id: Ie8d1024a4403ed8f8a3f6a2dfcd3874465b28c3b Signed-off-by: Pavan Anamula --- drivers/mmc/card/block.c | 1 + drivers/mmc/card/queue.h | 2 ++ 2 files changed, 3 insertions(+) diff --git a/drivers/mmc/card/block.c b/drivers/mmc/card/block.c index 9bf4804d646..3ef098cd94e 100644 --- a/drivers/mmc/card/block.c +++ b/drivers/mmc/card/block.c @@ -2716,6 +2716,7 @@ out: * (all existing requests completed or reinserted to the block layer) */ if ((!req && !(test_bit(MMC_QUEUE_NEW_REQUEST, &mq->flags))) || + (cmd_flags & MMC_REQ_SPECIAL_MASK) || ((test_bit(MMC_QUEUE_URGENT_REQUEST, &mq->flags)) && !(cmd_flags & MMC_REQ_NOREINSERT_MASK))) { if (mmc_card_need_bkops(card)) diff --git a/drivers/mmc/card/queue.h b/drivers/mmc/card/queue.h index 0ed7f366b7c..5d423797098 100644 --- a/drivers/mmc/card/queue.h +++ b/drivers/mmc/card/queue.h @@ -1,6 +1,8 @@ #ifndef MMC_QUEUE_H #define MMC_QUEUE_H +#define MMC_REQ_SPECIAL_MASK (REQ_DISCARD | REQ_FLUSH) + struct request; struct task_struct; From 2f25e5a246019cbbf99ba23cbf654f17eb26bbd1 Mon Sep 17 00:00:00 2001 From: Fengguang Wu Date: Tue, 18 Dec 2012 14:23:31 -0800 Subject: [PATCH 098/168] mm/vmscan.c: avoid possible deadlock caused by too_many_isolated() Neil found that if too_many_isolated() returns true while performing direct reclaim we can end up waiting for other threads to complete their direct reclaim. If those threads are allowed to enter the FS or IO to free memory, but this thread is not, then it is possible that those threads will be waiting on this thread and so we get a circular deadlock. some task enters direct reclaim with GFP_KERNEL => too_many_isolated() false => vmscan and run into dirty pages => pageout() => take some FS lock => fs/block code does GFP_NOIO allocation => enter direct reclaim again => too_many_isolated() true => waiting for others to progress, however the other tasks may be circular waiting for the FS lock.. The fix is to let !__GFP_IO and !__GFP_FS direct reclaims enjoy higher priority than normal ones, by lowering the throttle threshold for the latter. Allowing ~1/8 isolated pages in normal is large enough. For example, for a 1GB LRU list, that's ~128MB isolated pages, or 1k blocked tasks (each isolates 32 4KB pages), or 64 blocked tasks per logical CPU (assuming 16 logical CPUs per NUMA node). So it's not likely some CPU goes idle waiting (when it could make progress) because of this limit: there are much more sleeping reclaim tasks than the number of CPU, so the task may well be blocked by some low level queue/lock anyway. Now !GFP_IOFS reclaims won't be waiting for GFP_IOFS reclaims to progress. They will be blocked only when there are too many concurrent !GFP_IOFS reclaims, however that's very unlikely because the IO-less direct reclaims is able to progress much more faster, and they won't deadlock each other. The threshold is raised high enough for them, so that there can be sufficient parallel progress of !GFP_IOFS reclaims. [akpm@linux-foundation.org: tweak comment] Signed-off-by: Wu Fengguang Cc: Torsten Kaiser Tested-by: NeilBrown Reviewed-by: Minchan Kim Acked-by: KOSAKI Motohiro Acked-by: Rik van Riel Cc: Mel Gorman Cc: Johannes Weiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds Change-Id: I4bba11ffcec1a61a57dafa24486497bc89f5aab0 Git-commit: 3cf23841b4b76eb94d3f8d0fb3627690e4431413 Git-repo: https://android.googlesource.com/kernel/common/ Signed-off-by: Srinivasarao P --- mm/vmscan.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mm/vmscan.c b/mm/vmscan.c index e2e43aaff6c..85c3b639563 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1217,6 +1217,14 @@ static int too_many_isolated(struct zone *zone, int file, isolated = zone_page_state(zone, NR_ISOLATED_ANON); } + /* + * GFP_NOIO/GFP_NOFS callers are allowed to isolate more pages, so they + * won't get blocked by normal direct-reclaimers, forming a circular + * deadlock. + */ + if ((sc->gfp_mask & GFP_IOFS) == GFP_IOFS) + inactive >>= 3; + return isolated > inactive; } From c702aaaddd8423c97b780bf46eed4de3d5e2690f Mon Sep 17 00:00:00 2001 From: Vinayak Menon Date: Fri, 26 Dec 2014 19:29:41 +0530 Subject: [PATCH 099/168] mm: vmscan: fix the page state calculation in too_many_isolated It is observed that sometimes multiple tasks get blocked in the congestion_wait loop below, in shrink_inactive_list. (__schedule) from [] (schedule_timeout) from [] (io_schedule_timeout) from [] (congestion_wait) from [] (shrink_inactive_list) from [] (shrink_zone) from [] (try_to_free_pages) from [] (__alloc_pages_nodemask) from [] (new_slab) from [] (__slab_alloc) from [] In one such instance, zone_page_state(zone, NR_ISOLATED_FILE) had returned 14, zone_page_state(zone, NR_INACTIVE_FILE) returned 92, and the gfp_flag was GFP_KERNEL which resulted in too_many_isolated to return true. But one of the CPU pageset vmstat diff had NR_ISOLATED_FILE as -14. As there weren't any more update to per cpu pageset, the threshold wasn't met, and the tasks were blocked in the congestion wait. This patch uses zone_page_state_snapshot instead, but restricts its usage to avoid performance penalty. Change-Id: Iec767a548e524729c7ed79a92fe4718cdd08ce69 Signed-off-by: Vinayak Menon Signed-off-by: Srinivasarao P --- mm/vmscan.c | 62 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index 85c3b639563..b1f791bd5b8 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1195,26 +1195,31 @@ int isolate_lru_page(struct page *page) return ret; } -/* - * Are there way too many processes in the direct reclaim path already? - */ -static int too_many_isolated(struct zone *zone, int file, - struct scan_control *sc) +static int __too_many_isolated(struct zone *zone, int file, + struct scan_control *sc, int safe) { unsigned long inactive, isolated; - if (current_is_kswapd()) - return 0; - - if (!global_reclaim(sc)) - return 0; - if (file) { - inactive = zone_page_state(zone, NR_INACTIVE_FILE); - isolated = zone_page_state(zone, NR_ISOLATED_FILE); + if (safe) { + inactive = zone_page_state_snapshot(zone, + NR_INACTIVE_FILE); + isolated = zone_page_state_snapshot(zone, + NR_ISOLATED_FILE); + } else { + inactive = zone_page_state(zone, NR_INACTIVE_FILE); + isolated = zone_page_state(zone, NR_ISOLATED_FILE); + } } else { - inactive = zone_page_state(zone, NR_INACTIVE_ANON); - isolated = zone_page_state(zone, NR_ISOLATED_ANON); + if (safe) { + inactive = zone_page_state_snapshot(zone, + NR_INACTIVE_ANON); + isolated = zone_page_state_snapshot(zone, + NR_ISOLATED_ANON); + } else { + inactive = zone_page_state(zone, NR_INACTIVE_ANON); + isolated = zone_page_state(zone, NR_ISOLATED_ANON); + } } /* @@ -1228,6 +1233,28 @@ static int too_many_isolated(struct zone *zone, int file, return isolated > inactive; } +/* + * Are there way too many processes in the direct reclaim path already? + */ +static int too_many_isolated(struct zone *zone, int file, + struct scan_control *sc, int safe) +{ + if (current_is_kswapd()) + return 0; + + if (!global_reclaim(sc)) + return 0; + + if (unlikely(__too_many_isolated(zone, file, sc, 0))) { + if (safe) + return __too_many_isolated(zone, file, sc, safe); + else + return 1; + } + + return 0; +} + static noinline_for_stack void putback_inactive_pages(struct mem_cgroup_zone *mz, struct list_head *page_list) @@ -1295,16 +1322,19 @@ shrink_inactive_list(unsigned long nr_to_scan, struct mem_cgroup_zone *mz, unsigned long nr_writeback = 0; isolate_mode_t isolate_mode = 0; int file = is_file_lru(lru); + int safe = 0; struct zone *zone = mz->zone; struct zone_reclaim_stat *reclaim_stat = get_reclaim_stat(mz); struct lruvec *lruvec = mem_cgroup_zone_lruvec(zone, mz->mem_cgroup); - while (unlikely(too_many_isolated(zone, file, sc))) { + while (unlikely(too_many_isolated(zone, file, sc, safe))) { congestion_wait(BLK_RW_ASYNC, HZ/10); /* We are about to die and free our memory. Return now. */ if (fatal_signal_pending(current)) return SWAP_CLUSTER_MAX; + + safe = 1; } lru_add_drain(); From 4308ec38e348da866cea4e5e43d103c6b337095c Mon Sep 17 00:00:00 2001 From: Srinivasarao P Date: Tue, 8 Mar 2016 14:32:46 +0530 Subject: [PATCH 100/168] msm: sps: Handle low memory condition in SPS driver In low memory condition some bam client is sensitive to timing and may get time out while allocating memory through kzalloc. Add support to hold memory during connect and disconnect and to release memory during client driver's deregistration of its BAM. Change-Id: Iaf89dc542b62c95f055e7b06d788d896809b0bf8 Signed-off-by: Rama Krishna Phani A Signed-off-by: Srinivasarao P --- arch/arm/mach-msm/include/mach/sps.h | 4 ++- drivers/platform/msm/sps/sps.c | 9 ++++- drivers/platform/msm/sps/sps_bam.c | 54 ++++++++++++++++++---------- drivers/platform/msm/sps/sps_bam.h | 5 ++- 4 files changed, 51 insertions(+), 21 deletions(-) diff --git a/arch/arm/mach-msm/include/mach/sps.h b/arch/arm/mach-msm/include/mach/sps.h index 6cd4a2ee8cb..15e823fbaa0 100644 --- a/arch/arm/mach-msm/include/mach/sps.h +++ b/arch/arm/mach-msm/include/mach/sps.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2013, The Linux Foundation. All rights reserved. +/* Copyright (c) 2011-2013, 2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -109,6 +109,8 @@ #define SPS_BAM_NO_LOCAL_CLK_GATING (1UL << 5) /* Don't enable writeback cancel*/ #define SPS_BAM_CANCEL_WB (1UL << 6) +/* Hold memory for BAM DMUX */ +#define SPS_BAM_HOLD_MEM (1UL << 8) /* BAM device management flags */ diff --git a/drivers/platform/msm/sps/sps.c b/drivers/platform/msm/sps/sps.c index 55a5a2d7c27..10e47b98c1a 100644 --- a/drivers/platform/msm/sps/sps.c +++ b/drivers/platform/msm/sps/sps.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2013, The Linux Foundation. All rights reserved. +/* Copyright (c) 2011-2013, 2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -2193,6 +2193,7 @@ EXPORT_SYMBOL(sps_register_bam_device); int sps_deregister_bam_device(u32 dev_handle) { struct sps_bam *bam; + int n; SPS_DBG2("sps:%s.", __func__); @@ -2209,6 +2210,12 @@ int sps_deregister_bam_device(u32 dev_handle) SPS_DBG2("sps:SPS deregister BAM: phys 0x%x.", bam->props.phys_addr); + if (bam->props.options & SPS_BAM_HOLD_MEM) { + for (n = 0; n < BAM_MAX_PIPES; n++) + if (bam->desc_cache_pointers[n] != NULL) + kfree(bam->desc_cache_pointers[n]); + } + /* If this BAM is attached to a BAM-DMA, init the BAM-DMA device */ #ifdef CONFIG_SPS_SUPPORT_BAMDMA if ((bam->props.options & SPS_BAM_OPT_BAMDMA)) { diff --git a/drivers/platform/msm/sps/sps_bam.c b/drivers/platform/msm/sps/sps_bam.c index ad5e2419d4a..40b21bd0b9e 100644 --- a/drivers/platform/msm/sps/sps_bam.c +++ b/drivers/platform/msm/sps/sps_bam.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2014, The Linux Foundation. All rights reserved. +/* Copyright (c) 2011-2014, 2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -959,10 +959,15 @@ int sps_bam_pipe_disconnect(struct sps_bam *dev, u32 pipe_index) bam_pipe_exit(dev->base, pipe_index, dev->props.ee); if (pipe->sys.desc_cache != NULL) { u32 size = pipe->num_descs * sizeof(void *); - if (pipe->desc_size + size <= PAGE_SIZE) - kfree(pipe->sys.desc_cache); - else + if (pipe->desc_size + size <= PAGE_SIZE) { + if (dev->props.options & SPS_BAM_HOLD_MEM) + memset(pipe->sys.desc_cache, 0, + pipe->desc_size + size); + else + kfree(pipe->sys.desc_cache); + } else { vfree(pipe->sys.desc_cache); + } pipe->sys.desc_cache = NULL; } dev->pipes[pipe_index] = BAM_PIPE_UNASSIGNED; @@ -1086,30 +1091,43 @@ int sps_bam_pipe_set_params(struct sps_bam *dev, u32 pipe_index, u32 options) /* Allocate both descriptor cache and user pointer array */ size = pipe->num_descs * sizeof(void *); - if (pipe->desc_size + size <= PAGE_SIZE) - pipe->sys.desc_cache = - kzalloc(pipe->desc_size + size, GFP_KERNEL); - else { + if (pipe->desc_size + size <= PAGE_SIZE) { + if ((dev->props.options & + SPS_BAM_HOLD_MEM)) { + if (dev->desc_cache_pointers[pipe_index]) { + pipe->sys.desc_cache = + dev->desc_cache_pointers + [pipe_index]; + } else { + pipe->sys.desc_cache = + kzalloc(pipe->desc_size + size, + GFP_KERNEL); + dev->desc_cache_pointers[pipe_index] = + pipe->sys.desc_cache; + } + } else { + pipe->sys.desc_cache = + kzalloc(pipe->desc_size + size, + GFP_KERNEL); + } + if (pipe->sys.desc_cache == NULL) { + SPS_ERR("sps:No memory for pipe%d of BAM 0x%x\n", + pipe_index, BAM_ID(dev)); + return -ENOMEM; + } + } else { pipe->sys.desc_cache = vmalloc(pipe->desc_size + size); if (pipe->sys.desc_cache == NULL) { - SPS_ERR( - "sps:No memory for pipe %d of BAM 0x%x\n", - pipe_index, BAM_ID(dev)); + SPS_ERR("sps:No memory for pipe %d of BAM 0x%x\n", + pipe_index, BAM_ID(dev)); return -ENOMEM; } memset(pipe->sys.desc_cache, 0, pipe->desc_size + size); } - if (pipe->sys.desc_cache == NULL) { - /*** MUST BE LAST POINT OF FAILURE (see below) *****/ - SPS_ERR("sps:Desc cache error: BAM 0x%x pipe %d: %d\n", - BAM_ID(dev), pipe_index, - pipe->desc_size + size); - return SPS_ERROR; - } pipe->sys.user_ptrs = (void **)(pipe->sys.desc_cache + pipe->desc_size); pipe->sys.cache_offset = pipe->sys.acked_offset; diff --git a/drivers/platform/msm/sps/sps_bam.h b/drivers/platform/msm/sps/sps_bam.h index a20156b4f93..ddca05834de 100644 --- a/drivers/platform/msm/sps/sps_bam.h +++ b/drivers/platform/msm/sps/sps_bam.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2014, The Linux Foundation. All rights reserved. +/* Copyright (c) 2011-2014, 2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -214,6 +214,9 @@ struct sps_bam { u32 irq_from_disabled_pipe; u32 event_trigger_failures; + /* Desc cache pointers */ + u8 *desc_cache_pointers[BAM_MAX_PIPES]; + }; /** From 84aba3b15df72c73fa3758e082946d717c100b6d Mon Sep 17 00:00:00 2001 From: Rama Krishna Phani A Date: Wed, 6 May 2015 20:58:27 +0530 Subject: [PATCH 101/168] soc: qcom: bam_dmux: Use SPS hold mem flag Bam_dmux sometimes calls sps_connect() during time sensitive operations. Sps_connect() internally has some memory allocations which may stall during low memory conditions and can cause a timeout in bam_dmux. SPS supports a configuration flag which changes these memory allocations to hold memory. Use this flag to prevent allocations from stalling and leading to timeouts. Change-Id: I506d5664e07baf56f4caa10d1868b1795a4439d9 Signed-off-by: Rama Krishna Phani A Signed-off-by: Srinivasarao P --- arch/arm/mach-msm/bam_dmux.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm/mach-msm/bam_dmux.c b/arch/arm/mach-msm/bam_dmux.c index 084f377017b..4caaf814161 100644 --- a/arch/arm/mach-msm/bam_dmux.c +++ b/arch/arm/mach-msm/bam_dmux.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2014, The Linux Foundation. All rights reserved. +/* Copyright (c) 2011-2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -2084,7 +2084,7 @@ static int bam_init(void) a2_props.virt_addr = a2_virt_addr; a2_props.virt_size = a2_phys_size; a2_props.irq = a2_bam_irq; - a2_props.options = SPS_BAM_OPT_IRQ_WAKEUP; + a2_props.options = SPS_BAM_OPT_IRQ_WAKEUP | SPS_BAM_HOLD_MEM; a2_props.num_pipes = A2_NUM_PIPES; a2_props.summing_threshold = A2_SUMMING_THRESHOLD; a2_props.constrained_logging = true; From 9dbe0bba79ee192b1882b2fec739e451446c640c Mon Sep 17 00:00:00 2001 From: Ravinder Konka Date: Tue, 15 Mar 2016 12:25:39 +0530 Subject: [PATCH 102/168] defconfig: msm8926: Enable CONFIG_NETFILTER_XT_TARGET_IDLETIMER Enabling the config option to support IDLETIMER Change-Id: If7d225b1af76565af27c75402d3f7e3efa98389d CRs-Fixed: 988781 Signed-off-by: Ravinder Konka --- arch/arm/configs/msm8226-perf_defconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm/configs/msm8226-perf_defconfig b/arch/arm/configs/msm8226-perf_defconfig index 67f44ed9705..f0b79499604 100644 --- a/arch/arm/configs/msm8226-perf_defconfig +++ b/arch/arm/configs/msm8226-perf_defconfig @@ -155,6 +155,7 @@ CONFIG_NETFILTER_XT_TARGET_NFLOG=y CONFIG_NETFILTER_XT_TARGET_NFQUEUE=y CONFIG_NETFILTER_XT_TARGET_NOTRACK=y CONFIG_NETFILTER_XT_TARGET_SECMARK=y +CONFIG_NETFILTER_XT_TARGET_IDLETIMER=y CONFIG_NETFILTER_XT_MATCH_COMMENT=y CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=y CONFIG_NETFILTER_XT_MATCH_CONNMARK=y From d666b748f37cee280abd22f5d47122270e018fce Mon Sep 17 00:00:00 2001 From: Deva Ramasubramanian Date: Thu, 21 Jan 2016 15:36:49 -0800 Subject: [PATCH 103/168] msm: vidc: Make buffer validity checks stronger Check for the exact number of planes that we advertised to the client rather than the worst-case checks. Change-Id: Ibaf705367db98beb0e01bb2c3087126cf2ba73e8 Signed-off-by: Deva Ramasubramanian --- drivers/media/platform/msm/vidc/msm_vidc.c | 30 +++++++++++----------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/drivers/media/platform/msm/vidc/msm_vidc.c b/drivers/media/platform/msm/vidc/msm_vidc.c index 0b279cc02fc..97621123ef8 100644 --- a/drivers/media/platform/msm/vidc/msm_vidc.c +++ b/drivers/media/platform/msm/vidc/msm_vidc.c @@ -698,11 +698,23 @@ int output_buffer_cache_invalidate(struct msm_vidc_inst *inst, return 0; } +static bool valid_v4l2_buffer(struct v4l2_buffer *b, + struct msm_vidc_inst *inst) { + enum vidc_ports port = + !V4L2_TYPE_IS_MULTIPLANAR(b->type) ? MAX_PORT_NUM : + b->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE ? CAPTURE_PORT : + b->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE ? OUTPUT_PORT : + MAX_PORT_NUM; + + return port != MAX_PORT_NUM && + inst->fmts[port]->num_planes == b->length; +} + int msm_vidc_prepare_buf(void *instance, struct v4l2_buffer *b) { struct msm_vidc_inst *inst = instance; - if (!inst || !b) + if (!inst || !b || !valid_v4l2_buffer(b, inst)) return -EINVAL; if (is_dynamic_output_buffer_mode(b, inst)) { @@ -852,15 +864,9 @@ int msm_vidc_qbuf(void *instance, struct v4l2_buffer *b) int rc = 0; int i; - if (!inst || !b) + if (!inst || !b || !valid_v4l2_buffer(b, inst)) return -EINVAL; - if (b->length > VIDEO_MAX_PLANES) { - dprintk(VIDC_ERR, "num planes exceeds max: %d\n", - b->length); - return -EINVAL; - } - if (is_dynamic_output_buffer_mode(b, inst)) { if (b->m.planes[0].reserved[0]) inst->map_output_buffer = true; @@ -934,15 +940,9 @@ int msm_vidc_dqbuf(void *instance, struct v4l2_buffer *b) struct buffer_info *buffer_info = NULL; int i = 0, rc = 0; - if (!inst || !b) + if (!inst || !b || !valid_v4l2_buffer(b, inst)) return -EINVAL; - if (b->length > VIDEO_MAX_PLANES) { - dprintk(VIDC_ERR, "num planes exceed maximum: %d\n", - b->length); - return -EINVAL; - } - if (inst->session_type == MSM_VIDC_DECODER) rc = msm_vdec_dqbuf(instance, b); if (inst->session_type == MSM_VIDC_ENCODER) From b405b92f3be6f5da745b0fe2444766e34cd19a87 Mon Sep 17 00:00:00 2001 From: Vikram Mulukutla Date: Thu, 14 Aug 2014 12:37:06 -0700 Subject: [PATCH 104/168] arm: mach-msm: pil-msa: Set the strongly-ordered attribute on DMA allocations To completely eliminate the possibility of speculative accesses on DMA memory, set the strongly-ordered attribute on all such allocations. Change-Id: I4bad68eeae33aaebb4b1a30ab7a0ba133c6a0343 Signed-off-by: Vikram Mulukutla Signed-off-by: Srinivasarao P --- arch/arm/mach-msm/pil-msa.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/arch/arm/mach-msm/pil-msa.c b/arch/arm/mach-msm/pil-msa.c index 76afe6c0da5..6fb859e9db4 100644 --- a/arch/arm/mach-msm/pil-msa.c +++ b/arch/arm/mach-msm/pil-msa.c @@ -299,10 +299,12 @@ static int pil_msa_mba_init_image(struct pil_desc *pil, dma_addr_t mdata_phys; s32 status; int ret; + DEFINE_DMA_ATTRS(attrs); + dma_set_attr(DMA_ATTR_STRONGLY_ORDERED, &attrs); /* Make metadata physically contiguous and 4K aligned. */ - mdata_virt = dma_alloc_coherent(pil->dev, size, &mdata_phys, - GFP_KERNEL); + mdata_virt = dma_alloc_attrs(pil->dev, size, &mdata_phys, + GFP_KERNEL, &attrs); if (!mdata_virt) { dev_err(pil->dev, "MBA metadata buffer allocation failed\n"); return -ENOMEM; @@ -328,7 +330,7 @@ static int pil_msa_mba_init_image(struct pil_desc *pil, ret = -EINVAL; } - dma_free_coherent(pil->dev, size, mdata_virt, mdata_phys); + dma_free_attrs(pil->dev, size, mdata_virt, mdata_phys, &attrs); return ret; } From 585696a3d831f91aafc990066ea9e907e9732b21 Mon Sep 17 00:00:00 2001 From: Vikram Mulukutla Date: Thu, 14 Aug 2014 12:37:06 -0700 Subject: [PATCH 105/168] arm: mach-msm: pil-msa: Set the strongly-ordered attribute on DMA allocations To completely eliminate the possibility of speculative accesses on DMA memory, set the strongly-ordered attribute on all such allocations. Change-Id: I4bad68eeae33aaebb4b1a30ab7a0ba133c6a0343 Signed-off-by: Vikram Mulukutla Signed-off-by: Srinivasarao P --- arch/arm/mach-msm/pil-msa.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/arch/arm/mach-msm/pil-msa.c b/arch/arm/mach-msm/pil-msa.c index 76afe6c0da5..6fb859e9db4 100644 --- a/arch/arm/mach-msm/pil-msa.c +++ b/arch/arm/mach-msm/pil-msa.c @@ -299,10 +299,12 @@ static int pil_msa_mba_init_image(struct pil_desc *pil, dma_addr_t mdata_phys; s32 status; int ret; + DEFINE_DMA_ATTRS(attrs); + dma_set_attr(DMA_ATTR_STRONGLY_ORDERED, &attrs); /* Make metadata physically contiguous and 4K aligned. */ - mdata_virt = dma_alloc_coherent(pil->dev, size, &mdata_phys, - GFP_KERNEL); + mdata_virt = dma_alloc_attrs(pil->dev, size, &mdata_phys, + GFP_KERNEL, &attrs); if (!mdata_virt) { dev_err(pil->dev, "MBA metadata buffer allocation failed\n"); return -ENOMEM; @@ -328,7 +330,7 @@ static int pil_msa_mba_init_image(struct pil_desc *pil, ret = -EINVAL; } - dma_free_coherent(pil->dev, size, mdata_virt, mdata_phys); + dma_free_attrs(pil->dev, size, mdata_virt, mdata_phys, &attrs); return ret; } From c546ccdd03187dc98c4be8e0e5ff3d47b682f03f Mon Sep 17 00:00:00 2001 From: liu chuansheng Date: Fri, 6 Jul 2012 09:50:08 -0700 Subject: [PATCH 106/168] printk: replacing the raw_spin_lock/unlock with raw_spin_lock/unlock_irq In function devkmsg_read/writev/llseek/poll/open()..., the function raw_spin_lock/unlock is used, there is potential deadlock case happening. CPU1: thread1 doing the cat /dev/kmsg: raw_spin_lock(&logbuf_lock); while (user->seq == log_next_seq) { when thread1 run here, at this time one interrupt is coming on CPU1 and running based on this thread,if the interrupt handle called the printk which need the logbuf_lock spin also, it will cause deadlock. So we should use raw_spin_lock/unlock_irq here. Change-Id: I62320806e0836cd61e341775e62043c96499cbbe Acked-by: Kay Sievers Signed-off-by: liu chuansheng Signed-off-by: Greg Kroah-Hartman Git-commit: 5c53d819c71c63fdc91f30a59164583f68e2d63a Git-repo: https://android.googlesource.com/kernel/common/ [spathi@codeaurora.org: resolved trivial merge conflicts] Signed-off-by: Srinivasarao P --- kernel/printk.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/kernel/printk.c b/kernel/printk.c index 8f94984a136..91fc1db1baf 100644 --- a/kernel/printk.c +++ b/kernel/printk.c @@ -417,20 +417,20 @@ static ssize_t devkmsg_read(struct file *file, char __user *buf, return -EBADF; mutex_lock(&user->lock); - raw_spin_lock(&logbuf_lock); + raw_spin_lock_irq(&logbuf_lock); while (user->seq == log_next_seq) { if (file->f_flags & O_NONBLOCK) { ret = -EAGAIN; - raw_spin_unlock(&logbuf_lock); + raw_spin_unlock_irq(&logbuf_lock); goto out; } - raw_spin_unlock(&logbuf_lock); + raw_spin_unlock_irq(&logbuf_lock); ret = wait_event_interruptible(log_wait, user->seq != log_next_seq); if (ret) goto out; - raw_spin_lock(&logbuf_lock); + raw_spin_lock_irq(&logbuf_lock); } if (user->seq < log_first_seq) { @@ -438,7 +438,7 @@ static ssize_t devkmsg_read(struct file *file, char __user *buf, user->idx = log_first_idx; user->seq = log_first_seq; ret = -EPIPE; - raw_spin_unlock(&logbuf_lock); + raw_spin_unlock_irq(&logbuf_lock); goto out; } @@ -487,7 +487,7 @@ static ssize_t devkmsg_read(struct file *file, char __user *buf, user->idx = log_next(user->idx); user->seq++; - raw_spin_unlock(&logbuf_lock); + raw_spin_unlock_irq(&logbuf_lock); if (len > count) { ret = -EINVAL; @@ -514,7 +514,7 @@ static loff_t devkmsg_llseek(struct file *file, loff_t offset, int whence) if (offset) return -ESPIPE; - raw_spin_lock(&logbuf_lock); + raw_spin_lock_irq(&logbuf_lock); switch (whence) { case SEEK_SET: /* the first record */ @@ -538,7 +538,7 @@ static loff_t devkmsg_llseek(struct file *file, loff_t offset, int whence) default: ret = -EINVAL; } - raw_spin_unlock(&logbuf_lock); + raw_spin_unlock_irq(&logbuf_lock); return ret; } @@ -552,14 +552,14 @@ static unsigned int devkmsg_poll(struct file *file, poll_table *wait) poll_wait(file, &log_wait, wait); - raw_spin_lock(&logbuf_lock); + raw_spin_lock_irq(&logbuf_lock); if (user->seq < log_next_seq) { /* return error when data has vanished underneath us */ if (user->seq < log_first_seq) ret = POLLIN|POLLRDNORM|POLLERR|POLLPRI; ret = POLLIN|POLLRDNORM; } - raw_spin_unlock(&logbuf_lock); + raw_spin_unlock_irq(&logbuf_lock); return ret; } @@ -583,10 +583,10 @@ static int devkmsg_open(struct inode *inode, struct file *file) mutex_init(&user->lock); - raw_spin_lock(&logbuf_lock); + raw_spin_lock_irq(&logbuf_lock); user->idx = log_first_idx; user->seq = log_first_seq; - raw_spin_unlock(&logbuf_lock); + raw_spin_unlock_irq(&logbuf_lock); file->private_data = user; return 0; From f2e58c0fe1616edb9fbf12f583a96e6cdf0e3e0e Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Fri, 31 Oct 2014 09:54:53 +0530 Subject: [PATCH 107/168] mm: fix prctl_set_vma_anon_name prctl_set_vma_anon_name could attempt to set the name across two vmas at the same time due to a typo, which might corrupt the vma list. Fix it to use tmp instead of end to limit the name setting to a single vma at a time. Change-Id:I8dc2353f32b5f8510986d01c5f27d450b645902a Reported-by: Jed Davis Signed-off-by: Colin Cross Git-commit: 9bc0c15675840178cee1486c2a7f25faead1518e Git-Repo: https://android.googlesource.com/kernel/common.git Signed-off-by: Vinayak Menon --- kernel/sys.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sys.c b/kernel/sys.c index 7d01c443b1a..6fbf7944212 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -1909,7 +1909,7 @@ static int prctl_set_vma_anon_name(unsigned long start, unsigned long end, tmp = end; /* Here vma->vm_start <= start < tmp <= (end|vma->vm_end). */ - error = prctl_update_vma_anon_name(vma, &prev, start, end, + error = prctl_update_vma_anon_name(vma, &prev, start, tmp, (const char __user *)arg); if (error) return error; From 4c810f58ebbb0bee3335b21977190cf0106c1a66 Mon Sep 17 00:00:00 2001 From: Veera Sundaram Sankaran Date: Tue, 15 Mar 2016 18:42:27 -0700 Subject: [PATCH 108/168] msm: mdss: fix possible out-of-bounds and overflow issue in mdp debugfs There are few cases where the count argument passed by the user space is not validated, which can potentially lead to out of bounds or overflow issues. In some cases, kernel might copy more data than what is requested. Add necessary checks to avoid such cases. Change-Id: Ifa42fbd475665a0ca581c907ce5432584ea0e7ed [veeras@codeaurora.org: Resolved conflicts in mdss_debug.c] Signed-off-by: Veera Sundaram Sankaran Signed-off-by: Raghavendra Ambadas --- drivers/video/msm/mdss/mdss_debug.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/video/msm/mdss/mdss_debug.c b/drivers/video/msm/mdss/mdss_debug.c index aeb3f866daf..f49299502d4 100644 --- a/drivers/video/msm/mdss/mdss_debug.c +++ b/drivers/video/msm/mdss/mdss_debug.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2009-2013, The Linux Foundation. All rights reserved. +/* Copyright (c) 2009-2013, 2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -90,7 +90,7 @@ static ssize_t mdss_debug_base_offset_read(struct file *file, { struct mdss_debug_base *dbg = file->private_data; int len = 0; - char buf[24]; + char buf[24] = {'\0'}; if (!dbg) return -ENODEV; @@ -99,10 +99,10 @@ static ssize_t mdss_debug_base_offset_read(struct file *file, return 0; /* the end */ len = snprintf(buf, sizeof(buf), "0x%08x %x\n", dbg->off, dbg->cnt); - if (len < 0) + if (len < 0 || len >= sizeof(buf)) return 0; - if (copy_to_user(buff, buf, len)) + if ((count < sizeof(buf)) || copy_to_user(buff, buf, len)) return -EFAULT; *ppos += len; /* increase offset */ From f68c1544f9d4837a8d492ed3782c24f6adadf3c4 Mon Sep 17 00:00:00 2001 From: Sunil Khatri Date: Sun, 22 May 2016 16:14:31 +0530 Subject: [PATCH 109/168] msm: kgsl: Defer adding the mem entry to a process If we add the mem entry pointer in the process idr and rb tree too early, other threads can do operations on the entry by guessing the ID or GPU address before the object gets returned by the creating operation. Allocate an ID for the object but don't assign the pointer until right before the creating function returns ensuring that another operation can't access it until it is ready. CRs-Fixed: 1002974 Change-Id: Ic0dedbadc0dd2125bd2a7bcc152972c0555e07f8 Signed-off-by: Jordan Crouse Signed-off-by: Sunil Khatri --- drivers/gpu/msm/kgsl.c | 102 ++++++++++++++++++++++++++--------------- 1 file changed, 64 insertions(+), 38 deletions(-) diff --git a/drivers/gpu/msm/kgsl.c b/drivers/gpu/msm/kgsl.c index b93b1e7fef1..71f907227de 100644 --- a/drivers/gpu/msm/kgsl.c +++ b/drivers/gpu/msm/kgsl.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2008-2014, The Linux Foundation. All rights reserved. +/* Copyright (c) 2008-2014,2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -311,18 +311,13 @@ kgsl_mem_entry_destroy(struct kref *kref) EXPORT_SYMBOL(kgsl_mem_entry_destroy); /** - * kgsl_mem_entry_track_gpuaddr - Insert a mem_entry in the address tree and - * assign it with a gpu address space before insertion + * kgsl_mem_entry_track_gpuaddr - Get the entry gpu address space before + * insertion to the process * @process: the process that owns the memory * @entry: the memory entry * - * @returns - 0 on succcess else error code + * @returns - 0 on success else error code * - * Insert the kgsl_mem_entry in to the rb_tree for searching by GPU address. - * The assignment of gpu address and insertion into list needs to - * happen with the memory lock held to avoid race conditions between - * gpu address being selected and some other thread looking through the - * rb list in search of memory based on gpuaddr * This function should be called with processes memory spinlock held */ static int @@ -330,8 +325,6 @@ kgsl_mem_entry_track_gpuaddr(struct kgsl_process_private *process, struct kgsl_mem_entry *entry) { int ret = 0; - struct rb_node **node; - struct rb_node *parent = NULL; assert_spin_locked(&process->mem_lock); /* @@ -339,36 +332,17 @@ kgsl_mem_entry_track_gpuaddr(struct kgsl_process_private *process, * gpu address */ if (kgsl_memdesc_use_cpu_map(&entry->memdesc)) { - if (!entry->memdesc.gpuaddr) + /* cpu map flag is enabled. do nothing */ + } else { + if (entry->memdesc.gpuaddr) { + WARN_ONCE(1, "gpuaddr assigned w/o holding memory lock\n"); + ret = -EINVAL; goto done; - } else if (entry->memdesc.gpuaddr) { - WARN_ONCE(1, "gpuaddr assigned w/o holding memory lock\n"); - ret = -EINVAL; - goto done; - } - if (!kgsl_memdesc_use_cpu_map(&entry->memdesc)) { + } + ret = kgsl_mmu_get_gpuaddr(process->pagetable, &entry->memdesc); - if (ret) - goto done; } - node = &process->mem_rb.rb_node; - - while (*node) { - struct kgsl_mem_entry *cur; - - parent = *node; - cur = rb_entry(parent, struct kgsl_mem_entry, node); - - if (entry->memdesc.gpuaddr < cur->memdesc.gpuaddr) - node = &parent->rb_left; - else - node = &parent->rb_right; - } - - rb_link_node(&entry->node, parent, node); - rb_insert_color(&entry->node, &process->mem_rb); - done: return ret; } @@ -392,6 +366,47 @@ kgsl_mem_entry_untrack_gpuaddr(struct kgsl_process_private *process, } } +static void kgsl_mem_entry_commit_mem_list(struct kgsl_process_private *process, + struct kgsl_mem_entry *entry) +{ + struct rb_node **node; + struct rb_node *parent = NULL; + + if (!entry->memdesc.gpuaddr) + return; + + /* Insert mem entry in mem_rb tree */ + node = &process->mem_rb.rb_node; + while (*node) { + struct kgsl_mem_entry *cur; + + parent = *node; + cur = rb_entry(parent, struct kgsl_mem_entry, node); + + if (entry->memdesc.gpuaddr < cur->memdesc.gpuaddr) + node = &parent->rb_left; + else + node = &parent->rb_right; + } + + rb_link_node(&entry->node, parent, node); + rb_insert_color(&entry->node, &process->mem_rb); +} + +static void kgsl_mem_entry_commit_process(struct kgsl_process_private *process, + struct kgsl_mem_entry *entry) +{ + if (!entry) + return; + + spin_lock(&entry->priv->mem_lock); + /* Insert mem entry in mem_rb tree */ + kgsl_mem_entry_commit_mem_list(process, entry); + /* Replace mem entry in mem_idr using id */ + idr_replace(&entry->priv->mem_idr, entry, entry->id); + spin_unlock(&entry->priv->mem_lock); +} + /** * kgsl_mem_entry_attach_process - Attach a mem_entry to its owner process * @entry: the memory entry @@ -422,7 +437,8 @@ kgsl_mem_entry_attach_process(struct kgsl_mem_entry *entry, } spin_lock(&process->mem_lock); - ret = idr_get_new_above(&process->mem_idr, entry, 1, + /* Allocate the ID but don't attach the pointer just yet */ + ret = idr_get_new_above(&process->mem_idr, NULL, 1, &entry->id); spin_unlock(&process->mem_lock); @@ -3158,6 +3174,7 @@ static long kgsl_ioctl_map_user_mem(struct kgsl_device_private *dev_priv, trace_kgsl_mem_map(entry, param->fd); + kgsl_mem_entry_commit_process(private, entry); return result; error_attach: @@ -3447,6 +3464,8 @@ kgsl_ioctl_gpumem_alloc(struct kgsl_device_private *dev_priv, param->gpuaddr = entry->memdesc.gpuaddr; param->size = entry->memdesc.size; param->flags = entry->memdesc.flags; + + kgsl_mem_entry_commit_process(private, entry); return result; err: kgsl_sharedmem_free(&entry->memdesc); @@ -3482,6 +3501,8 @@ kgsl_ioctl_gpumem_alloc_id(struct kgsl_device_private *dev_priv, param->size = entry->memdesc.size; param->mmapsize = kgsl_memdesc_mmapsize(&entry->memdesc); param->gpuaddr = entry->memdesc.gpuaddr; + + kgsl_mem_entry_commit_process(private, entry); return result; err: if (entry) @@ -4049,6 +4070,11 @@ kgsl_get_unmapped_area(struct file *file, unsigned long addr, kgsl_mem_entry_untrack_gpuaddr(private, entry); spin_unlock(&private->mem_lock); ret = ret_val; + } else { + /* Insert mem entry in mem_rb tree */ + spin_lock(&private->mem_lock); + kgsl_mem_entry_commit_mem_list(private, entry); + spin_unlock(&private->mem_lock); } break; } From 159b90f0decbb7ae1356a7f459c43af2a19dd8cd Mon Sep 17 00:00:00 2001 From: Raghavendra Ambadas Date: Wed, 24 Feb 2016 17:39:11 +0530 Subject: [PATCH 110/168] msm: display: Validate MDP, MDDI and HDMI debug reg offset Validate the input arguments of MDP, MDDI and HDMI offset values in the respective write functions. Change-Id: Ida1b7ec292c365701fe17446019f625fdaff38d2 Signed-off-by: Raghavendra Ambadas --- drivers/video/msm/mdp_debugfs.c | 112 +++++++++++++++++++++++++------- 1 file changed, 87 insertions(+), 25 deletions(-) diff --git a/drivers/video/msm/mdp_debugfs.c b/drivers/video/msm/mdp_debugfs.c index d3e0c8ddd63..2b31e4a35c6 100644 --- a/drivers/video/msm/mdp_debugfs.c +++ b/drivers/video/msm/mdp_debugfs.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2009-2012, The Linux Foundation. All rights reserved. +/* Copyright (c) 2009-2012, 2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -39,6 +39,9 @@ #endif #define MDP_DEBUG_BUF 2048 +#define MDP_MAX_OFFSET 0xF05FC +#define MDDI_MAX_OFFSET 0xC +#define HDMI_MAX_OFFSET 0x59C static uint32 mdp_offset; static uint32 mdp_count; @@ -78,11 +81,18 @@ static ssize_t mdp_offset_write( debug_buf[count] = 0; /* end of string */ - sscanf(debug_buf, "%x %d", &off, &cnt); + if (sscanf(debug_buf, "%x %d", &off, &cnt) != 2) + return -EFAULT; if (cnt <= 0) cnt = 1; + if ((off > MDP_MAX_OFFSET) || (cnt > (MDP_MAX_OFFSET - off))) { + printk(KERN_INFO "%s: Invalid offset%x+cnt%d > %x\n", __func__, + off, cnt, MDP_MAX_OFFSET); + return -EFAULT; + } + mdp_offset = off; mdp_count = cnt; @@ -154,6 +164,14 @@ static ssize_t mdp_reg_write( debug_buf[count] = 0; /* end of string */ cnt = sscanf(debug_buf, "%x %x", &off, &data); + if (cnt != 2) + return -EFAULT; + + if (off > MDP_MAX_OFFSET) { + printk(KERN_INFO "%s: Invalid offset%x > %x\n", __func__, + off, MDP_MAX_OFFSET); + return -EFAULT; + } mdp_pipe_ctrl(MDP_CMD_BLOCK, MDP_BLOCK_POWER_ON, FALSE); outpdw(MDP_BASE + off, data); @@ -620,6 +638,11 @@ static void mddi_reg_write(int ndx, uint32 off, uint32 data) else base = (char *)msm_pmdh_base; + if (off > MDDI_MAX_OFFSET) { + printk(KERN_INFO "%s: Invalid offset=%x > %x\n", __func__, + off, MDDI_MAX_OFFSET); + return; + } mdp_pipe_ctrl(MDP_CMD_BLOCK, MDP_BLOCK_POWER_ON, FALSE); writel(data, base + off); mdp_pipe_ctrl(MDP_CMD_BLOCK, MDP_BLOCK_POWER_OFF, FALSE); @@ -682,6 +705,14 @@ static ssize_t pmdh_reg_write( debug_buf[count] = 0; /* end of string */ cnt = sscanf(debug_buf, "%x %x", &off, &data); + if (cnt != 2) + return -EFAULT; + + if (off > MDDI_MAX_OFFSET) { + printk(KERN_INFO "%s: Invalid offset=%x > %x\n", __func__, + off, MDDI_MAX_OFFSET); + return -EFAULT; + } mddi_reg_write(0, off, data); @@ -737,6 +768,14 @@ static ssize_t emdh_reg_write( debug_buf[count] = 0; /* end of string */ cnt = sscanf(debug_buf, "%x %x", &off, &data); + if (cnt != 2) + return -EFAULT; + + if (off > MDDI_MAX_OFFSET) { + printk(KERN_INFO "%s: Invalid offset=%x > %x\n", __func__, + off, MDDI_MAX_OFFSET); + return -EFAULT; + } mddi_reg_write(1, off, data); @@ -884,15 +923,18 @@ static ssize_t dbg_offset_write( cnt = sscanf(debug_buf, "%x %d %x", &off, &num, &base); - if (cnt < 0) - cnt = 0; + if (cnt != 3) + return -EFAULT; - if (cnt >= 1) - dbg_offset = off; - if (cnt >= 2) - dbg_count = num; - if (cnt >= 3) - dbg_base = (char *)base; + if ((off > MDP_MAX_OFFSET) || (num > (MDP_MAX_OFFSET - off))) { + printk(KERN_INFO "%s: Invalid offset%x+num%d > %x\n", __func__, + off, num, MDP_MAX_OFFSET); + return -EFAULT; + } + + dbg_offset = off; + dbg_count = num; + dbg_base = (char *)base; printk(KERN_INFO "%s: offset=%x cnt=%d base=%x\n", __func__, dbg_offset, dbg_count, (int)dbg_base); @@ -951,6 +993,14 @@ static ssize_t dbg_reg_write( debug_buf[count] = 0; /* end of string */ cnt = sscanf(debug_buf, "%x %x", &off, &data); + if (cnt != 2) + return -EFAULT; + + if (off > MDP_MAX_OFFSET) { + printk(KERN_INFO "%s: Invalid offset%x > %x\n", __func__, + off, MDP_MAX_OFFSET); + return -EFAULT; + } writel(data, dbg_base + off); @@ -1191,14 +1241,17 @@ static ssize_t hdmi_offset_write( debug_buf[count] = 0; /* end of string */ cnt = sscanf(debug_buf, "%x %d", &off, &num); + if (cnt != 2) + return -EFAULT; - if (cnt < 0) - cnt = 0; + if ((off > HDMI_MAX_OFFSET) || (num > (HDMI_MAX_OFFSET - off))) { + printk(KERN_INFO "%s: Invalid offset%x+num%d > %x\n", __func__, + off, num, HDMI_MAX_OFFSET); + return -EFAULT; + } - if (cnt >= 1) - hdmi_offset = off; - if (cnt >= 2) - hdmi_count = num; + hdmi_offset = off; + hdmi_count = num; printk(KERN_INFO "%s: offset=%x cnt=%d\n", __func__, hdmi_offset, hdmi_count); @@ -1262,6 +1315,15 @@ static ssize_t hdmi_reg_write( cnt = sscanf(debug_buf, "%x %x", &off, &data); + if (cnt != 2) + return -EFAULT; + + if (off > HDMI_MAX_OFFSET) { + printk(KERN_INFO "%s: Invalid offset%x > %x\n", __func__, + off, HDMI_MAX_OFFSET); + return -EFAULT; + } + writel(data, base + off); printk(KERN_INFO "%s: addr=%x data=%x\n", @@ -1355,14 +1417,14 @@ int mdp_debugfs_init(void) return -1; } - if (debugfs_create_file("off", 0644, dent, 0, &mdp_off_fops) + if (debugfs_create_file("off", 0600, dent, 0, &mdp_off_fops) == NULL) { printk(KERN_ERR "%s(%d): debugfs_create_file: index fail\n", __FILE__, __LINE__); return -1; } - if (debugfs_create_file("reg", 0644, dent, 0, &mdp_reg_fops) + if (debugfs_create_file("reg", 0600, dent, 0, &mdp_reg_fops) == NULL) { printk(KERN_ERR "%s(%d): debugfs_create_file: debug fail\n", __FILE__, __LINE__); @@ -1402,7 +1464,7 @@ int mdp_debugfs_init(void) return -1; } - if (debugfs_create_file("reg", 0644, dent, 0, &pmdh_fops) + if (debugfs_create_file("reg", 0600, dent, 0, &pmdh_fops) == NULL) { printk(KERN_ERR "%s(%d): debugfs_create_file: debug fail\n", __FILE__, __LINE__); @@ -1417,7 +1479,7 @@ int mdp_debugfs_init(void) return -1; } - if (debugfs_create_file("reg", 0644, dent, 0, &emdh_fops) + if (debugfs_create_file("reg", 0600, dent, 0, &emdh_fops) == NULL) { printk(KERN_ERR "%s(%d): debugfs_create_file: debug fail\n", __FILE__, __LINE__); @@ -1432,21 +1494,21 @@ int mdp_debugfs_init(void) return -1; } - if (debugfs_create_file("base", 0644, dent, 0, &dbg_base_fops) + if (debugfs_create_file("base", 0600, dent, 0, &dbg_base_fops) == NULL) { printk(KERN_ERR "%s(%d): debugfs_create_file: index fail\n", __FILE__, __LINE__); return -1; } - if (debugfs_create_file("off", 0644, dent, 0, &dbg_off_fops) + if (debugfs_create_file("off", 0600, dent, 0, &dbg_off_fops) == NULL) { printk(KERN_ERR "%s(%d): debugfs_create_file: index fail\n", __FILE__, __LINE__); return -1; } - if (debugfs_create_file("reg", 0644, dent, 0, &dbg_reg_fops) + if (debugfs_create_file("reg", 0600, dent, 0, &dbg_reg_fops) == NULL) { printk(KERN_ERR "%s(%d): debugfs_create_file: debug fail\n", __FILE__, __LINE__); @@ -1462,14 +1524,14 @@ int mdp_debugfs_init(void) return PTR_ERR(dent); } - if (debugfs_create_file("off", 0644, dent, 0, &hdmi_off_fops) + if (debugfs_create_file("off", 0600, dent, 0, &hdmi_off_fops) == NULL) { printk(KERN_ERR "%s(%d): debugfs_create_file: 'off' fail\n", __FILE__, __LINE__); return -ENOENT; } - if (debugfs_create_file("reg", 0644, dent, 0, &hdmi_reg_fops) + if (debugfs_create_file("reg", 0600, dent, 0, &hdmi_reg_fops) == NULL) { printk(KERN_ERR "%s(%d): debugfs_create_file: 'reg' fail\n", __FILE__, __LINE__); From a7b04014b93fecef5ae3f36a95465c46f4503ce4 Mon Sep 17 00:00:00 2001 From: Rajesh Kemisetti Date: Tue, 28 Jun 2016 10:48:55 +0530 Subject: [PATCH 111/168] msm: kgsl: Add missing check for alloc size In _kgsl_sharedmem_page_alloc(): - Make len of type size_t to be in line with size. - Check for boundary limits of requested alloc size before honoring. Change-Id: I8b9e225e515a0f31593df6f4cad253236475d0ae Signed-off-by: Rajesh Kemisetti --- drivers/gpu/msm/kgsl_sharedmem.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/msm/kgsl_sharedmem.c b/drivers/gpu/msm/kgsl_sharedmem.c index c04c4d04bd7..e53a9499c80 100644 --- a/drivers/gpu/msm/kgsl_sharedmem.c +++ b/drivers/gpu/msm/kgsl_sharedmem.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2002,2007-2014, The Linux Foundation. All rights reserved. +/* Copyright (c) 2002,2007-2014,2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -617,13 +617,18 @@ _kgsl_sharedmem_page_alloc(struct kgsl_memdesc *memdesc, size_t size) { int pcount = 0, ret = 0; - int j, len, page_size, sglen_alloc, sglen = 0; + int j, page_size, sglen_alloc, sglen = 0; + size_t len; struct page **pages = NULL; pgprot_t page_prot = pgprot_writecombine(PAGE_KERNEL); void *ptr; unsigned int align; int step = ((VMALLOC_END - VMALLOC_START)/8) >> PAGE_SHIFT; + size = PAGE_ALIGN(size); + if (size == 0 || size > UINT_MAX) + return -EINVAL; + align = (memdesc->flags & KGSL_MEMALIGN_MASK) >> KGSL_MEMALIGN_SHIFT; page_size = (align >= ilog2(SZ_64K) && size >= SZ_64K) From f693ac940d93048c655ad472492d2aaa02f6e6a1 Mon Sep 17 00:00:00 2001 From: Arun Menon Date: Tue, 21 Jun 2016 12:42:40 +0530 Subject: [PATCH 112/168] msm: vidc: Clean up video instance locking logic Clean up unnecessary locks in video driver code. Change-Id: Ib00527531863a06fa0bad28ed3cc71fbdf167e2d Signed-off-by: Arun Menon Signed-off-by: Vasantha Balla --- drivers/media/platform/msm/vidc/msm_vdec.c | 3 - drivers/media/platform/msm/vidc/msm_venc.c | 2 - drivers/media/platform/msm/vidc/msm_vidc.c | 145 ++++------ .../media/platform/msm/vidc/msm_vidc_common.c | 267 ++++++++---------- .../media/platform/msm/vidc/msm_vidc_debug.c | 12 +- .../platform/msm/vidc/msm_vidc_internal.h | 13 +- 6 files changed, 196 insertions(+), 246 deletions(-) diff --git a/drivers/media/platform/msm/vidc/msm_vdec.c b/drivers/media/platform/msm/vidc/msm_vdec.c index 2cf59c6626f..da004ba29eb 100644 --- a/drivers/media/platform/msm/vidc/msm_vdec.c +++ b/drivers/media/platform/msm/vidc/msm_vdec.c @@ -1210,7 +1210,6 @@ static int msm_vdec_queue_setup(struct vb2_queue *q, "Failed to get buffer requirements: %d\n", rc); break; } - mutex_lock(&inst->lock); bufreq = get_buff_req_buffer(inst, msm_comm_get_hal_output_buffer(inst)); if (!bufreq) { @@ -1218,7 +1217,6 @@ static int msm_vdec_queue_setup(struct vb2_queue *q, "No buffer requirement for buffer type %x\n", HAL_BUFFER_OUTPUT); rc = -EINVAL; - mutex_unlock(&inst->lock); break; } *num_buffers = max(*num_buffers, bufreq->buffer_count_min); @@ -1230,7 +1228,6 @@ static int msm_vdec_queue_setup(struct vb2_queue *q, rc = call_hfi_op(hdev, session_set_property, inst->session, property_id, &new_buf_count); } - mutex_unlock(&inst->lock); dprintk(VIDC_DBG, "count = %d, size = %d, alignment = %d\n", inst->buff_req.buffer[1].buffer_count_actual, inst->buff_req.buffer[1].buffer_size, diff --git a/drivers/media/platform/msm/vidc/msm_venc.c b/drivers/media/platform/msm/vidc/msm_venc.c index c6fc3a69cc7..d534f9678f2 100644 --- a/drivers/media/platform/msm/vidc/msm_venc.c +++ b/drivers/media/platform/msm/vidc/msm_venc.c @@ -996,11 +996,9 @@ static int msm_venc_queue_setup(struct vb2_queue *q, case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE: *num_planes = 1; - mutex_lock(&inst->lock); *num_buffers = inst->buff_req.buffer[0].buffer_count_actual = max(*num_buffers, inst->buff_req.buffer[0]. buffer_count_actual); - mutex_unlock(&inst->lock); property_id = HAL_PARAM_BUFFER_COUNT_ACTUAL; new_buf_count.buffer_type = HAL_BUFFER_INPUT; diff --git a/drivers/media/platform/msm/vidc/msm_vidc.c b/drivers/media/platform/msm/vidc/msm_vidc.c index 97621123ef8..de7f561c9d6 100644 --- a/drivers/media/platform/msm/vidc/msm_vidc.c +++ b/drivers/media/platform/msm/vidc/msm_vidc.c @@ -223,20 +223,19 @@ struct buffer_info *get_registered_buf(struct msm_vidc_inst *inst, struct buffer_info *temp; struct buffer_info *ret = NULL; int i; - struct list_head *list = &inst->registered_bufs; int fd = b->m.planes[idx].reserved[0]; u32 buff_off = b->m.planes[idx].reserved[1]; u32 size = b->m.planes[idx].length; u32 device_addr = b->m.planes[idx].m.userptr; - if (!list || fd < 0 || !plane) { + if (fd < 0 || !plane) { dprintk(VIDC_ERR, "Invalid input\n"); goto err_invalid_input; } *plane = 0; - mutex_lock(&inst->lock); - list_for_each_entry(temp, list, list) { + mutex_lock(&inst->registeredbufs.lock); + list_for_each_entry(temp, &inst->registeredbufs.list, list) { for (i = 0; (i < temp->num_planes) && (i < VIDEO_MAX_PLANES); i++) { if (temp && @@ -259,26 +258,28 @@ struct buffer_info *get_registered_buf(struct msm_vidc_inst *inst, if (ret) break; } - mutex_unlock(&inst->lock); + mutex_unlock(&inst->registeredbufs.lock); err_invalid_input: return ret; } -struct msm_smem *get_same_fd_buffer(struct msm_vidc_inst *inst, - struct list_head *list, int fd) +static struct msm_smem *get_same_fd_buffer(struct msm_vidc_list *buf_list, + int fd) { struct buffer_info *temp; struct msm_smem *same_fd_handle = NULL; int i; + if (fd == 0) return NULL; - if (!list || fd < 0) { + + if (!buf_list || fd < 0) { dprintk(VIDC_ERR, "Invalid input\n"); goto err_invalid_input; } - mutex_lock(&inst->lock); - list_for_each_entry(temp, list, list) { + mutex_lock(&buf_list->lock); + list_for_each_entry(temp, &buf_list->list, list) { for (i = 0; (i < temp->num_planes) && (i < VIDEO_MAX_PLANES); i++) { if (temp && (temp->fd[i] == fd) && @@ -293,26 +294,26 @@ struct msm_smem *get_same_fd_buffer(struct msm_vidc_inst *inst, if (same_fd_handle) break; } - mutex_unlock(&inst->lock); + mutex_unlock(&buf_list->lock); err_invalid_input: return same_fd_handle; } -struct buffer_info *device_to_uvaddr(struct msm_vidc_inst *inst, - struct list_head *list, u32 device_addr) +struct buffer_info *device_to_uvaddr(struct msm_vidc_list *buf_list, + u32 device_addr) { struct buffer_info *temp = NULL; - struct buffer_info *dummy = NULL; int found = 0; int i; - if (!list || !device_addr || !inst) { + + if (!buf_list || !device_addr) { dprintk(VIDC_ERR, - "Invalid input- list: %p device_addr: %u inst: %p\n", - list, device_addr, inst); + "Invalid input- device_addr: %u buf_list: %p\n", + device_addr, buf_list); goto err_invalid_input; } - mutex_lock(&inst->lock); - list_for_each_entry_safe(temp, dummy, list, list) { + mutex_lock(&buf_list->lock); + list_for_each_entry(temp, &buf_list->list, list) { for (i = 0; (i < temp->num_planes) && (i < VIDEO_MAX_PLANES); i++) { if (temp && !temp->inactive && @@ -326,7 +327,7 @@ struct buffer_info *device_to_uvaddr(struct msm_vidc_inst *inst, if (found) break; } - mutex_unlock(&inst->lock); + mutex_unlock(&buf_list->lock); err_invalid_input: return temp; } @@ -499,9 +500,8 @@ int map_and_register_buf(struct msm_vidc_inst *inst, struct v4l2_buffer *b) if (rc < 0) goto exit; - same_fd_handle = get_same_fd_buffer(inst, - &inst->registered_bufs, - b->m.planes[i].reserved[0]); + same_fd_handle = get_same_fd_buffer(&inst->registeredbufs, + b->m.planes[i].reserved[0]); populate_buf_info(binfo, b, i); if (same_fd_handle) { @@ -542,9 +542,9 @@ int map_and_register_buf(struct msm_vidc_inst *inst, struct v4l2_buffer *b) binfo->device_addr[i], binfo->fd[i], binfo->buff_off[i], binfo->mapped[i]); } - mutex_lock(&inst->lock); - list_add_tail(&binfo->list, &inst->registered_bufs); - mutex_unlock(&inst->lock); + mutex_lock(&inst->registeredbufs.lock); + list_add_tail(&binfo->list, &inst->registeredbufs.list); + mutex_unlock(&inst->registeredbufs.lock); return 0; exit: kfree(binfo); @@ -555,8 +555,6 @@ int unmap_and_deregister_buf(struct msm_vidc_inst *inst, { int i = 0; struct buffer_info *temp = NULL; - struct buffer_info *dummy = NULL; - struct list_head *list; bool found = false, keep_node = false; if (!inst || !binfo) { @@ -565,13 +563,13 @@ int unmap_and_deregister_buf(struct msm_vidc_inst *inst, return -EINVAL; } - mutex_lock(&inst->lock); - list = &inst->registered_bufs; + mutex_lock(&inst->registeredbufs.lock); + /* * Make sure the buffer to be unmapped and deleted * from the registered list is present in the list. */ - list_for_each_entry_safe(temp, dummy, list, list) { + list_for_each_entry(temp, &inst->registeredbufs.list, list) { if (temp == binfo) { found = true; break; @@ -629,7 +627,7 @@ int unmap_and_deregister_buf(struct msm_vidc_inst *inst, dprintk(VIDC_DBG, "[UNMAP] NOT-FREED binfo: %p\n", temp); } exit: - mutex_unlock(&inst->lock); + mutex_unlock(&inst->registeredbufs.lock); return 0; } @@ -739,9 +737,8 @@ int msm_vidc_prepare_buf(void *instance, struct v4l2_buffer *b) int msm_vidc_release_buffers(void *instance, int buffer_type) { - struct list_head *ptr, *next; struct msm_vidc_inst *inst = instance; - struct buffer_info *bi; + struct buffer_info *bi, *dummy; struct v4l2_buffer buffer_info; struct v4l2_plane plane[VIDEO_MAX_PLANES]; int i, rc = 0; @@ -769,10 +766,9 @@ int msm_vidc_release_buffers(void *instance, int buffer_type) goto free_and_unmap; } - list_for_each_safe(ptr, next, &inst->registered_bufs) { + mutex_lock(&inst->registeredbufs.lock); + list_for_each_entry(bi, &inst->registeredbufs.list, list) { bool release_buf = false; - mutex_lock(&inst->lock); - bi = list_entry(ptr, struct buffer_info, list); if (bi->type == buffer_type) { buffer_info.type = bi->type; for (i = 0; (i < bi->num_planes) @@ -791,7 +787,6 @@ int msm_vidc_release_buffers(void *instance, int buffer_type) buffer_info.length = bi->num_planes; release_buf = true; } - mutex_unlock(&inst->lock); if (!release_buf) continue; if (inst->session_type == MSM_VIDC_DECODER) @@ -807,11 +802,10 @@ int msm_vidc_release_buffers(void *instance, int buffer_type) buffer_info.m.planes[0].reserved[1], buffer_info.m.planes[0].length); } - + mutex_unlock(&inst->registeredbufs.lock); free_and_unmap: - mutex_lock(&inst->lock); - list_for_each_safe(ptr, next, &inst->registered_bufs) { - bi = list_entry(ptr, struct buffer_info, list); + mutex_lock(&inst->registeredbufs.lock); + list_for_each_entry_safe(bi, dummy, &inst->registeredbufs.list, list) { if (bi->type == buffer_type) { list_del(&bi->list); for (i = 0; i < bi->num_planes; i++) { @@ -828,7 +822,7 @@ free_and_unmap: kfree(bi); } } - mutex_unlock(&inst->lock); + mutex_unlock(&inst->registeredbufs.lock); return rc; } @@ -959,9 +953,8 @@ int msm_vidc_dqbuf(void *instance, struct v4l2_buffer *b) !b->m.planes[i].m.userptr) { continue; } - buffer_info = device_to_uvaddr(inst, - &inst->registered_bufs, - b->m.planes[i].m.userptr); + buffer_info = device_to_uvaddr(&inst->registeredbufs, + b->m.planes[i].m.userptr); if (!buffer_info) { dprintk(VIDC_ERR, @@ -987,9 +980,7 @@ int msm_vidc_dqbuf(void *instance, struct v4l2_buffer *b) } if (is_dynamic_output_buffer_mode(b, inst)) { - mutex_lock(&inst->lock); buffer_info->dequeued = true; - mutex_unlock(&inst->lock); dprintk(VIDC_DBG, "[DEQUEUED]: fd[0] = %d\n", buffer_info->fd[0]); rc = unmap_and_deregister_buf(inst, buffer_info); @@ -1224,10 +1215,11 @@ void *msm_vidc_open(int core_id, int session_type) mutex_init(&inst->lock); inst->session_type = session_type; INIT_MSM_VIDC_LIST(&inst->pendingq); - INIT_LIST_HEAD(&inst->internalbufs); - INIT_LIST_HEAD(&inst->persistbufs); - INIT_LIST_HEAD(&inst->registered_bufs); - INIT_LIST_HEAD(&inst->outputbufs); + INIT_MSM_VIDC_LIST(&inst->internalbufs); + INIT_MSM_VIDC_LIST(&inst->persistbufs); + INIT_MSM_VIDC_LIST(&inst->outputbufs); + INIT_MSM_VIDC_LIST(&inst->registeredbufs); + init_waitqueue_head(&inst->kernel_event_queue); inst->state = MSM_VIDC_CORE_UNINIT_DONE; inst->core = core; @@ -1315,37 +1307,23 @@ static void cleanup_instance(struct msm_vidc_inst *inst) kfree(entry); } mutex_unlock(&inst->pendingq.lock); - mutex_lock(&inst->lock); - if (!list_empty(&inst->internalbufs)) { - mutex_unlock(&inst->lock); - if (msm_comm_release_scratch_buffers(inst)) - dprintk(VIDC_ERR, - "Failed to release scratch buffers\n"); - - mutex_lock(&inst->lock); + if (msm_comm_release_scratch_buffers(inst)) { + dprintk(VIDC_ERR, + "Failed to release scratch buffers\n"); } - if (!list_empty(&inst->persistbufs)) { - mutex_unlock(&inst->lock); - if (msm_comm_release_persist_buffers(inst)) - dprintk(VIDC_ERR, - "Failed to release persist buffers\n"); - mutex_lock(&inst->lock); + if (msm_comm_release_persist_buffers(inst)) { + dprintk(VIDC_ERR, + "Failed to release persist buffers\n"); } - if (!list_empty(&inst->outputbufs)) { - mutex_unlock(&inst->lock); - if (msm_comm_release_output_buffers(inst)) - dprintk(VIDC_ERR, - "Failed to release output buffers\n"); - mutex_lock(&inst->lock); + if (msm_comm_release_output_buffers(inst)) { + dprintk(VIDC_ERR, + "Failed to release output buffers\n"); } - if (inst->extradata_handle) { - mutex_unlock(&inst->lock); + if (inst->extradata_handle) msm_comm_smem_free(inst, inst->extradata_handle); - mutex_lock(&inst->lock); - } - mutex_unlock(&inst->lock); + debugfs_remove_recursive(inst->debugfs_root); } } @@ -1353,10 +1331,9 @@ static void cleanup_instance(struct msm_vidc_inst *inst) int msm_vidc_close(void *instance) { struct msm_vidc_inst *inst = instance; - struct msm_vidc_inst *temp; + struct msm_vidc_inst *temp,*inst_dummy; struct msm_vidc_core *core; - struct list_head *ptr, *next; - struct buffer_info *bi; + struct buffer_info *bi,*dummy; int rc = 0; int i; @@ -1364,8 +1341,8 @@ int msm_vidc_close(void *instance) return -EINVAL; v4l2_fh_del(&inst->event_handler); - list_for_each_safe(ptr, next, &inst->registered_bufs) { - bi = list_entry(ptr, struct buffer_info, list); + mutex_lock(&inst->registeredbufs.lock); + list_for_each_entry_safe(bi, dummy, &inst->registeredbufs.list, list) { if (bi->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) { list_del(&bi->list); for (i = 0; (i < bi->num_planes) @@ -1376,12 +1353,12 @@ int msm_vidc_close(void *instance) kfree(bi); } } + mutex_unlock(&inst->registeredbufs.lock); core = inst->core; mutex_lock(&core->lock); - list_for_each_safe(ptr, next, &core->instances) { - temp = list_entry(ptr, struct msm_vidc_inst, list); + list_for_each_entry_safe(temp, inst_dummy, &core->instances, list) { if (temp == inst) list_del(&inst->list); } diff --git a/drivers/media/platform/msm/vidc/msm_vidc_common.c b/drivers/media/platform/msm/vidc/msm_vidc_common.c index 8558f3b3b71..83829609778 100644 --- a/drivers/media/platform/msm/vidc/msm_vidc_common.c +++ b/drivers/media/platform/msm/vidc/msm_vidc_common.c @@ -165,9 +165,7 @@ static int msm_comm_get_load(struct msm_vidc_core *core, if (inst->session_type != type) continue; - mutex_lock(&inst->lock); num_mbs_per_sec += msm_comm_get_inst_load(inst, quirks); - mutex_unlock(&inst->lock); } mutex_unlock(&core->lock); return num_mbs_per_sec; @@ -349,7 +347,8 @@ static void handle_session_release_buf_done(enum command_response cmd, buffer = (struct hal_buffer_info *) response->data; address = (u32) buffer->buffer_addr; - list_for_each_safe(ptr, next, &inst->internalbufs) { + mutex_lock(&inst->internalbufs.lock); + list_for_each_safe(ptr, next, &inst->internalbufs.list) { buf = list_entry(ptr, struct internal_buf, list); if (address == buf->handle->device_addr) { dprintk(VIDC_DBG, "releasing scratch: 0x%x", @@ -357,8 +356,10 @@ static void handle_session_release_buf_done(enum command_response cmd, buf_found = true; } } + mutex_unlock(&inst->internalbufs.lock); - list_for_each_safe(ptr, next, &inst->persistbufs) { + mutex_lock(&inst->persistbufs.lock); + list_for_each_safe(ptr, next, &inst->persistbufs.list) { buf = list_entry(ptr, struct internal_buf, list); if (address == (u32) buf->handle->device_addr) { dprintk(VIDC_DBG, "releasing persist: 0x%x", @@ -366,6 +367,7 @@ static void handle_session_release_buf_done(enum command_response cmd, buf_found = true; } } + mutex_unlock(&inst->persistbufs.lock); if (!buf_found) dprintk(VIDC_ERR, "invalid buffer received from firmware"); @@ -583,9 +585,8 @@ static void handle_event_change(enum command_response cmd, void *data) * Get the buffer_info entry for the * device address. */ - binfo = device_to_uvaddr(inst, - &inst->registered_bufs, - (u32)event_notify->packet_buffer); + binfo = device_to_uvaddr(&inst->registeredbufs, + (u32)event_notify->packet_buffer); if (!binfo) { dprintk(VIDC_ERR, "%s buffer not found in registered list\n", @@ -763,11 +764,8 @@ void validate_output_buffers(struct msm_vidc_inst *inst) HAL_BUFFER_OUTPUT); return; } - list_for_each_entry(binfo, &inst->outputbufs, list) { - if (!binfo) { - dprintk(VIDC_ERR, "Invalid parameter\n"); - return; - } + mutex_lock(&inst->outputbufs.lock); + list_for_each_entry(binfo, &inst->outputbufs.list, list) { if (binfo->buffer_ownership != DRIVER) { dprintk(VIDC_DBG, "This buffer is with FW 0x%lx\n", @@ -776,6 +774,7 @@ void validate_output_buffers(struct msm_vidc_inst *inst) } buffers_owned_by_driver++; } + mutex_unlock(&inst->outputbufs.lock); if (buffers_owned_by_driver != output_buf->buffer_count_actual) dprintk(VIDC_ERR, "OUTPUT Buffer count mismatch %d of %d", @@ -811,7 +810,8 @@ int msm_comm_queue_output_buffers(struct msm_vidc_inst *inst) output_buf->buffer_count_actual, output_buf->buffer_size); - list_for_each_entry(binfo, &inst->outputbufs, list) { + mutex_lock(&inst->outputbufs.lock); + list_for_each_entry(binfo, &inst->outputbufs.list, list) { if (binfo->buffer_ownership != DRIVER) continue; handle = binfo->handle; @@ -827,6 +827,7 @@ int msm_comm_queue_output_buffers(struct msm_vidc_inst *inst) (void *) inst->session, &frame_data); binfo->buffer_ownership = FIRMWARE; } + mutex_unlock(&inst->outputbufs.lock); return 0; } @@ -839,7 +840,6 @@ static void handle_session_flush(enum command_response cmd, void *data) inst = (struct msm_vidc_inst *)response->session_id; if (msm_comm_get_stream_output_mode(inst) == HAL_VIDEO_DECODER_SECONDARY) { - mutex_lock(&inst->lock); validate_output_buffers(inst); if (!inst->in_reconfig) { rc = msm_comm_queue_output_buffers(inst); @@ -849,7 +849,6 @@ static void handle_session_flush(enum command_response cmd, void *data) rc); } } - mutex_unlock(&inst->lock); } msm_vidc_queue_v4l2_event(inst, V4L2_EVENT_MSM_VIDC_FLUSH_DONE); } else { @@ -1106,7 +1105,6 @@ int buf_ref_get(struct msm_vidc_inst *inst, struct buffer_info *binfo) if (!inst || !binfo) return -EINVAL; - mutex_lock(&inst->lock); atomic_inc(&binfo->ref_count); cnt = atomic_read(&binfo->ref_count); if (cnt > 2) { @@ -1114,7 +1112,6 @@ int buf_ref_get(struct msm_vidc_inst *inst, struct buffer_info *binfo) cnt = -EINVAL; } dprintk(VIDC_DBG, "REF_GET[%d] fd[0] = %d\n", cnt, binfo->fd[0]); - mutex_unlock(&inst->lock); return cnt; } @@ -1128,7 +1125,6 @@ int buf_ref_put(struct msm_vidc_inst *inst, struct buffer_info *binfo) if (!inst || !binfo) return -EINVAL; - mutex_lock(&inst->lock); atomic_dec(&binfo->ref_count); cnt = atomic_read(&binfo->ref_count); dprintk(VIDC_DBG, "REF_PUT[%d] fd[0] = %d\n", cnt, binfo->fd[0]); @@ -1140,7 +1136,6 @@ int buf_ref_put(struct msm_vidc_inst *inst, struct buffer_info *binfo) dprintk(VIDC_DBG, "%s: invalid ref_cnt: %d\n", __func__, cnt); cnt = -EINVAL; } - mutex_unlock(&inst->lock); if (cnt < 0) return cnt; @@ -1179,8 +1174,7 @@ static void handle_dynamic_buffer(struct msm_vidc_inst *inst, * only when firmware is not holding a reference. */ if (inst->buffer_mode_set[CAPTURE_PORT] == HAL_BUFFER_MODE_DYNAMIC) { - binfo = device_to_uvaddr(inst, &inst->registered_bufs, - device_addr); + binfo = device_to_uvaddr(&inst->registeredbufs, device_addr); if (!binfo) { dprintk(VIDC_ERR, "%s buffer not found in registered list\n", @@ -1206,12 +1200,9 @@ static int handle_multi_stream_buffers(struct msm_vidc_inst *inst, struct internal_buf *binfo; struct msm_smem *handle; bool found = false; - mutex_lock(&inst->lock); - list_for_each_entry(binfo, &inst->outputbufs, list) { - if (!binfo) { - dprintk(VIDC_ERR, "Invalid parameter\n"); - break; - } + + mutex_lock(&inst->outputbufs.lock); + list_for_each_entry(binfo, &inst->outputbufs.list, list) { handle = binfo->handle; if (handle && dev_addr == handle->device_addr) { if (binfo->buffer_ownership == DRIVER) { @@ -1225,11 +1216,12 @@ static int handle_multi_stream_buffers(struct msm_vidc_inst *inst, break; } } + mutex_unlock(&inst->outputbufs.lock); + if (!found) dprintk(VIDC_ERR, "Failed to find output buffer in queued list: 0x%x\n", dev_addr); - mutex_unlock(&inst->lock); return 0; } @@ -1826,11 +1818,11 @@ static int msm_comm_session_init(int flipped_state, } init_completion( &inst->completions[SESSION_MSG_INDEX(SESSION_INIT_DONE)]); - mutex_lock(&inst->lock); + inst->session = call_hfi_op(hdev, session_init, hdev->hfi_device_data, (u32) inst, get_hal_domain(inst->session_type), get_hal_codec_type(fourcc)); - mutex_unlock(&inst->lock); + if (!inst->session) { dprintk(VIDC_ERR, "Failed to call session init for: %d, %d, %d, %d\n", @@ -1852,7 +1844,6 @@ static void msm_vidc_print_running_insts(struct msm_vidc_core *core) mutex_lock(&core->lock); list_for_each_entry(temp, &core->instances, list) { - mutex_lock(&temp->lock); if (temp->state >= MSM_VIDC_OPEN_DONE && temp->state < MSM_VIDC_STOP_DONE) { dprintk(VIDC_ERR, "%4d|%4d|%4d|%4d\n", @@ -1861,7 +1852,6 @@ static void msm_vidc_print_running_insts(struct msm_vidc_core *core) temp->prop.height[CAPTURE_PORT], temp->prop.fps); } - mutex_unlock(&temp->lock); } mutex_unlock(&core->lock); } @@ -2180,7 +2170,6 @@ static int set_output_buffers(struct msm_vidc_inst *inst, rc = -ENOMEM; goto fail_kzalloc; } - mutex_lock(&inst->lock); binfo->handle = handle; buffer_info.buffer_size = output_buf->buffer_size; buffer_info.buffer_type = buffer_type; @@ -2200,16 +2189,15 @@ static int set_output_buffers(struct msm_vidc_inst *inst, buffer_info.extradata_addr); rc = call_hfi_op(hdev, session_set_buffers, (void *) inst->session, &buffer_info); - mutex_unlock(&inst->lock); if (rc) { dprintk(VIDC_ERR, "%s : session_set_buffers failed", __func__); goto fail_set_buffers; } - mutex_lock(&inst->lock); - list_add_tail(&binfo->list, &inst->outputbufs); - mutex_unlock(&inst->lock); + mutex_lock(&inst->outputbufs.lock); + list_add_tail(&binfo->list, &inst->outputbufs.list); + mutex_unlock(&inst->outputbufs.lock); } } return rc; @@ -2289,9 +2277,9 @@ static int set_scratch_buffers(struct msm_vidc_inst *inst, "vidc_hal_session_set_buffers failed"); goto fail_set_buffers; } - mutex_lock(&inst->lock); - list_add_tail(&binfo->list, &inst->internalbufs); - mutex_unlock(&inst->lock); + mutex_lock(&inst->internalbufs.lock); + list_add_tail(&binfo->list, &inst->internalbufs.list); + mutex_unlock(&inst->internalbufs.lock); } } return rc; @@ -2329,11 +2317,14 @@ static int set_persist_buffers(struct msm_vidc_inst *inst, "persist: num = %d, size = %d\n", persist_buf->buffer_count_actual, persist_buf->buffer_size); - if (!list_empty(&inst->persistbufs)) { + mutex_lock(&inst->persistbufs.lock); + if (!list_empty(&inst->persistbufs.list)) { dprintk(VIDC_ERR, "Persist buffers already allocated\n"); + mutex_unlock(&inst->persistbufs.lock); return rc; } + mutex_unlock(&inst->persistbufs.lock); if (inst->flags & VIDC_SECURE) smem_flags |= SMEM_SECURE; @@ -2376,9 +2367,9 @@ static int set_persist_buffers(struct msm_vidc_inst *inst, "vidc_hal_session_set_buffers failed"); goto fail_set_buffers; } - mutex_lock(&inst->lock); - list_add_tail(&binfo->list, &inst->persistbufs); - mutex_unlock(&inst->lock); + mutex_lock(&inst->persistbufs.lock); + list_add_tail(&binfo->list, &inst->persistbufs.list); + mutex_unlock(&inst->persistbufs.lock); } } return rc; @@ -2715,8 +2706,7 @@ exit: int msm_comm_release_output_buffers(struct msm_vidc_inst *inst) { struct msm_smem *handle; - struct list_head *ptr, *next; - struct internal_buf *buf; + struct internal_buf *buf,*dummy; struct vidc_buffer_addr_info buffer_info; int rc = 0; struct msm_vidc_core *core; @@ -2737,43 +2727,45 @@ int msm_comm_release_output_buffers(struct msm_vidc_inst *inst) dprintk(VIDC_ERR, "Invalid device pointer = %p\n", hdev); return -EINVAL; } - mutex_lock(&inst->lock); - if (!list_empty(&inst->outputbufs)) { - list_for_each_safe(ptr, next, &inst->outputbufs) { - buf = list_entry(ptr, struct internal_buf, - list); - handle = buf->handle; - buffer_info.buffer_size = handle->size; - buffer_info.buffer_type = buf->buffer_type; - buffer_info.num_buffers = 1; - buffer_info.align_device_addr = handle->device_addr; - if (inst->state != MSM_VIDC_CORE_INVALID && - core->state != VIDC_CORE_INVALID) { - buffer_info.response_required = false; - rc = call_hfi_op(hdev, session_release_buffers, - (void *)inst->session, &buffer_info); - if (rc) - dprintk(VIDC_WARN, - "Rel output buf fail:0x%x, %d", - buffer_info.align_device_addr, - buffer_info.buffer_size); - } - list_del(&buf->list); - mutex_unlock(&inst->lock); - msm_comm_smem_free(inst, buf->handle); - kfree(buf); - mutex_lock(&inst->lock); + + mutex_lock(&inst->outputbufs.lock); + list_for_each_entry_safe(buf, dummy, &inst->outputbufs.list, list) { + handle = buf->handle; + if (!handle) { + dprintk(VIDC_ERR, "%s - invalid handle\n", __func__); + goto exit; } + + buffer_info.buffer_size = handle->size; + buffer_info.buffer_type = buf->buffer_type; + buffer_info.num_buffers = 1; + buffer_info.align_device_addr = handle->device_addr; + if (inst->state != MSM_VIDC_CORE_INVALID && + core->state != VIDC_CORE_INVALID) { + buffer_info.response_required = false; + rc = call_hfi_op(hdev, session_release_buffers, + (void *)inst->session, &buffer_info); + if (rc) { + dprintk(VIDC_WARN, + "Rel output buf fail:0x%pa, %d\n", + &buffer_info.align_device_addr, + buffer_info.buffer_size); + } + } + + list_del(&buf->list); + msm_comm_smem_free(inst, buf->handle); + kfree(buf); } - mutex_unlock(&inst->lock); +exit: + mutex_unlock(&inst->outputbufs.lock); return rc; } int msm_comm_release_scratch_buffers(struct msm_vidc_inst *inst) { struct msm_smem *handle; - struct list_head *ptr, *next; - struct internal_buf *buf; + struct internal_buf *buf,*dummy; struct vidc_buffer_addr_info buffer_info; int rc = 0; struct msm_vidc_core *core; @@ -2794,11 +2786,9 @@ int msm_comm_release_scratch_buffers(struct msm_vidc_inst *inst) dprintk(VIDC_ERR, "Invalid device pointer = %p\n", hdev); return -EINVAL; } - mutex_lock(&inst->lock); - if (!list_empty(&inst->internalbufs)) { - list_for_each_safe(ptr, next, &inst->internalbufs) { - buf = list_entry(ptr, struct internal_buf, - list); + + mutex_lock(&inst->internalbufs.lock); + list_for_each_entry_safe(buf, dummy, &inst->internalbufs.list, list) { handle = buf->handle; buffer_info.buffer_size = handle->size; buffer_info.buffer_type = buf->buffer_type; @@ -2817,7 +2807,7 @@ int msm_comm_release_scratch_buffers(struct msm_vidc_inst *inst) "Rel scrtch buf fail:0x%x, %d", buffer_info.align_device_addr, buffer_info.buffer_size); - mutex_unlock(&inst->lock); + mutex_unlock(&inst->internalbufs.lock); rc = wait_for_sess_signal_receipt(inst, SESSION_RELEASE_BUFFER_DONE); if (rc) { @@ -2825,16 +2815,13 @@ int msm_comm_release_scratch_buffers(struct msm_vidc_inst *inst) MSM_VIDC_CORE_INVALID); msm_comm_kill_session(inst); } - mutex_lock(&inst->lock); + mutex_lock(&inst->internalbufs.lock); } list_del(&buf->list); - mutex_unlock(&inst->lock); msm_comm_smem_free(inst, buf->handle); kfree(buf); - mutex_lock(&inst->lock); } - } - mutex_unlock(&inst->lock); + mutex_unlock(&inst->internalbufs.lock); return rc; } @@ -2863,47 +2850,42 @@ int msm_comm_release_persist_buffers(struct msm_vidc_inst *inst) dprintk(VIDC_ERR, "Invalid device pointer = %p\n", hdev); return -EINVAL; } - mutex_lock(&inst->lock); - if (!list_empty(&inst->persistbufs)) { - list_for_each_safe(ptr, next, &inst->persistbufs) { - buf = list_entry(ptr, struct internal_buf, - list); - handle = buf->handle; - buffer_info.buffer_size = handle->size; - buffer_info.buffer_type = buf->buffer_type; - buffer_info.num_buffers = 1; - buffer_info.align_device_addr = handle->device_addr; - if (inst->state != MSM_VIDC_CORE_INVALID && - core->state != VIDC_CORE_INVALID) { - buffer_info.response_required = true; - init_completion( - &inst->completions[SESSION_MSG_INDEX - (SESSION_RELEASE_BUFFER_DONE)]); - rc = call_hfi_op(hdev, session_release_buffers, - (void *)inst->session, &buffer_info); - if (rc) - dprintk(VIDC_WARN, - "Rel prst buf fail:0x%x, %d", - buffer_info.align_device_addr, - buffer_info.buffer_size); - mutex_unlock(&inst->lock); - rc = wait_for_sess_signal_receipt(inst, - SESSION_RELEASE_BUFFER_DONE); - if (rc) { - change_inst_state(inst, - MSM_VIDC_CORE_INVALID); - msm_comm_kill_session(inst); - } - mutex_lock(&inst->lock); + mutex_lock(&inst->persistbufs.lock); + list_for_each_safe(ptr, next, &inst->persistbufs.list) { + buf = list_entry(ptr, struct internal_buf, list); + handle = buf->handle; + buffer_info.buffer_size = handle->size; + buffer_info.buffer_type = buf->buffer_type; + buffer_info.num_buffers = 1; + buffer_info.align_device_addr = handle->device_addr; + if (inst->state != MSM_VIDC_CORE_INVALID && + core->state != VIDC_CORE_INVALID) { + buffer_info.response_required = true; + init_completion( + &inst->completions[SESSION_MSG_INDEX + (SESSION_RELEASE_BUFFER_DONE)]); + rc = call_hfi_op(hdev, session_release_buffers, + (void *)inst->session, &buffer_info); + if (rc) { + dprintk(VIDC_WARN, + "Rel prst buf fail:0x%pa, %d\n", + &buffer_info.align_device_addr, + buffer_info.buffer_size); } - list_del(&buf->list); - mutex_unlock(&inst->lock); - msm_comm_smem_free(inst, buf->handle); - kfree(buf); - mutex_lock(&inst->lock); + mutex_unlock(&inst->persistbufs.lock); + rc = wait_for_sess_signal_receipt(inst, + SESSION_RELEASE_BUFFER_DONE); + if (rc) { + change_inst_state(inst, MSM_VIDC_CORE_INVALID); + msm_comm_kill_session(inst); + } + mutex_lock(&inst->persistbufs.lock); } + list_del(&buf->list); + msm_comm_smem_free(inst, buf->handle); + kfree(buf); } - mutex_unlock(&inst->lock); + mutex_unlock(&inst->persistbufs.lock); return rc; } @@ -3053,8 +3035,7 @@ static void msm_comm_flush_in_invalid_state(struct msm_vidc_inst *inst) void msm_comm_flush_dynamic_buffers(struct msm_vidc_inst *inst) { - struct buffer_info *binfo = NULL, *dummy = NULL; - struct list_head *list = NULL; + struct buffer_info *binfo = NULL; if (inst->buffer_mode_set[CAPTURE_PORT] != HAL_BUFFER_MODE_DYNAMIC) return; @@ -3079,15 +3060,16 @@ void msm_comm_flush_dynamic_buffers(struct msm_vidc_inst *inst) * should not queue back the buffer to f/w. * Flush all buffers with ref_count 2. */ - mutex_lock(&inst->lock); - if (!list_empty(&inst->registered_bufs)) { + mutex_lock(&inst->registeredbufs.lock); + if (!list_empty(&inst->registeredbufs.list)) { struct v4l2_event buf_event = {0}; u32 *ptr = NULL; - list = &inst->registered_bufs; - list_for_each_entry_safe(binfo, dummy, list, list) { - if (binfo && - (binfo->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) && - (atomic_read(&binfo->ref_count) == 2)) { + + list_for_each_entry(binfo, &inst->registeredbufs.list, list) { + if ((binfo->type == + V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) && + (atomic_read(&binfo->ref_count) == 2)) { + atomic_dec(&binfo->ref_count); buf_event.type = V4L2_EVENT_MSM_VIDC_RELEASE_UNQUEUED_BUFFER; @@ -3099,8 +3081,8 @@ void msm_comm_flush_dynamic_buffers(struct msm_vidc_inst *inst) ptr[4] = (u32) binfo->timestamp.tv_usec; ptr[5] = binfo->v4l2_index; dprintk(VIDC_DBG, - "released buffer held in driver before issuing flush: 0x%x fd[0]: %d\n", - binfo->device_addr[0], binfo->fd[0]); + "released buffer held in driver before issuing flush: 0x%pa fd[0]: %d\n", + &binfo->device_addr[0], binfo->fd[0]); /*send event to client*/ v4l2_event_queue_fh(&inst->event_handler, &buf_event); @@ -3108,13 +3090,12 @@ void msm_comm_flush_dynamic_buffers(struct msm_vidc_inst *inst) } } } - mutex_unlock(&inst->lock); + mutex_unlock(&inst->registeredbufs.lock); } void msm_comm_flush_pending_dynamic_buffers(struct msm_vidc_inst *inst) { struct buffer_info *binfo = NULL; - struct list_head *list = NULL; if (!inst) return; @@ -3123,10 +3104,9 @@ void msm_comm_flush_pending_dynamic_buffers(struct msm_vidc_inst *inst) return; if (list_empty(&inst->pendingq.list) || - list_empty(&inst->registered_bufs)) + list_empty(&inst->registeredbufs.list)) return; - list = &inst->registered_bufs; /* * Dynamic Buffer mode - Since pendingq is not empty @@ -3134,8 +3114,8 @@ void msm_comm_flush_pending_dynamic_buffers(struct msm_vidc_inst *inst) * Hence remove reference to all pendingq o/p buffers * before flushing them. */ - - list_for_each_entry(binfo, list, list) { + mutex_lock(&inst->registeredbufs.lock); + list_for_each_entry(binfo, &inst->registeredbufs.list, list) { if (binfo && binfo->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) { dprintk(VIDC_DBG, @@ -3144,6 +3124,7 @@ void msm_comm_flush_pending_dynamic_buffers(struct msm_vidc_inst *inst) buf_ref_put(inst, binfo); } } + mutex_unlock(&inst->registeredbufs.lock); } int msm_comm_flush(struct msm_vidc_inst *inst, u32 flags) @@ -3180,9 +3161,9 @@ int msm_comm_flush(struct msm_vidc_inst *inst, u32 flags) dprintk(VIDC_INFO, "Input only flush not supported\n"); return 0; } - mutex_lock(&inst->sync_lock); + msm_comm_flush_dynamic_buffers(inst); - mutex_unlock(&inst->sync_lock); + if (inst->state == MSM_VIDC_CORE_INVALID || core->state == VIDC_CORE_UNINIT || core->state == VIDC_CORE_INVALID) { diff --git a/drivers/media/platform/msm/vidc/msm_vidc_debug.c b/drivers/media/platform/msm/vidc/msm_vidc_debug.c index 8c9e7466749..1fb68b2b978 100644 --- a/drivers/media/platform/msm/vidc/msm_vidc_debug.c +++ b/drivers/media/platform/msm/vidc/msm_vidc_debug.c @@ -216,20 +216,16 @@ static int inst_info_open(struct inode *inode, struct file *file) static int publish_unreleased_reference(struct msm_vidc_inst *inst) { struct buffer_info *temp = NULL; - struct buffer_info *dummy = NULL; - struct list_head *list = NULL; if (!inst) { dprintk(VIDC_ERR, "%s: invalid param\n", __func__); return -EINVAL; } - list = &inst->registered_bufs; - mutex_lock(&inst->lock); if (inst->buffer_mode_set[CAPTURE_PORT] == HAL_BUFFER_MODE_DYNAMIC) { - list_for_each_entry_safe(temp, dummy, list, list) { - if (temp && temp->type == - V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE && + mutex_lock(&inst->registeredbufs.lock); + list_for_each_entry(temp, &inst->registeredbufs.list, list) { + if (temp->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE && !temp->inactive && atomic_read(&temp->ref_count)) { write_str(&dbg_buf, "\tpending buffer: 0x%x fd[0] = %d ref_count = %d held by: %s\n", @@ -239,8 +235,8 @@ static int publish_unreleased_reference(struct msm_vidc_inst *inst) DYNAMIC_BUF_OWNER(temp)); } } + mutex_unlock(&inst->registeredbufs.lock); } - mutex_unlock(&inst->lock); return 0; } diff --git a/drivers/media/platform/msm/vidc/msm_vidc_internal.h b/drivers/media/platform/msm/vidc/msm_vidc_internal.h index 9968e77b302..aacfb1e81db 100644 --- a/drivers/media/platform/msm/vidc/msm_vidc_internal.h +++ b/drivers/media/platform/msm/vidc/msm_vidc_internal.h @@ -239,9 +239,10 @@ struct msm_vidc_inst { struct msm_vidc_format *fmts[MAX_PORT_NUM]; struct buf_queue bufq[MAX_PORT_NUM]; struct msm_vidc_list pendingq; - struct list_head internalbufs; - struct list_head persistbufs; - struct list_head outputbufs; + struct msm_vidc_list internalbufs; + struct msm_vidc_list persistbufs; + struct msm_vidc_list outputbufs; + struct msm_vidc_list registeredbufs; struct buffer_requirements buff_req; void *mem_client; struct v4l2_ctrl_handler ctrl_handler; @@ -263,7 +264,7 @@ struct msm_vidc_inst { u32 multi_stream_mode; struct msm_vidc_core_capability capability; enum buffer_mode_type buffer_mode_set[MAX_PORT_NUM]; - struct list_head registered_bufs; + bool map_output_buffer; struct v4l2_ctrl **ctrls; }; @@ -314,8 +315,8 @@ struct buffer_info { struct timeval timestamp; }; -struct buffer_info *device_to_uvaddr(struct msm_vidc_inst *inst, - struct list_head *list, u32 device_addr); +struct buffer_info *device_to_uvaddr(struct msm_vidc_list *buf_list, + u32 device_addr); int buf_ref_get(struct msm_vidc_inst *inst, struct buffer_info *binfo); int buf_ref_put(struct msm_vidc_inst *inst, struct buffer_info *binfo); int output_buffer_cache_invalidate(struct msm_vidc_inst *inst, From 7e1db7d2cf05dadd242d15d0b118ae43d7203a42 Mon Sep 17 00:00:00 2001 From: Praneeth Paladugu Date: Tue, 21 Jun 2016 15:48:13 +0530 Subject: [PATCH 113/168] msm: vidc: Use registeredbuf.lock for buffer synchronization Right now dynamic buffers are managed with both sync_lock and registeredbufs.lock. sync_lock is meant only for state transition. Hence remove sync_lock and achieve the same purpose with registeredbufs.lock. Change-Id: I3b0eb62019e7e992f63c159bf65d8c21cf2f52c8 Signed-off-by: Praneeth Paladugu Signed-off-by: Vikash Garodia Signed-off-by: Vasantha Balla --- drivers/media/platform/msm/vidc/msm_vidc.c | 18 +++++++++++------- .../media/platform/msm/vidc/msm_vidc_common.c | 4 ++-- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/drivers/media/platform/msm/vidc/msm_vidc.c b/drivers/media/platform/msm/vidc/msm_vidc.c index de7f561c9d6..9eb7c337bf1 100644 --- a/drivers/media/platform/msm/vidc/msm_vidc.c +++ b/drivers/media/platform/msm/vidc/msm_vidc.c @@ -233,8 +233,10 @@ struct buffer_info *get_registered_buf(struct msm_vidc_inst *inst, goto err_invalid_input; } + WARN(!mutex_is_locked(&inst->registeredbufs.lock), + "Registered buf lock is not acquired for %s", __func__); + *plane = 0; - mutex_lock(&inst->registeredbufs.lock); list_for_each_entry(temp, &inst->registeredbufs.list, list) { for (i = 0; (i < temp->num_planes) && (i < VIDEO_MAX_PLANES); i++) { @@ -258,7 +260,6 @@ struct buffer_info *get_registered_buf(struct msm_vidc_inst *inst, if (ret) break; } - mutex_unlock(&inst->registeredbufs.lock); err_invalid_input: return ret; } @@ -471,7 +472,7 @@ int map_and_register_buf(struct msm_vidc_inst *inst, struct v4l2_buffer *b) !b->m.planes[i].length) { continue; } - mutex_lock(&inst->sync_lock); + mutex_lock(&inst->registeredbufs.lock); temp = get_registered_buf(inst, b, i, &plane); if (temp && !is_dynamic_output_buffer_mode(b, inst)) { dprintk(VIDC_DBG, @@ -496,7 +497,7 @@ int map_and_register_buf(struct msm_vidc_inst *inst, struct v4l2_buffer *b) rc = -EEXIST; } } - mutex_unlock(&inst->sync_lock); + mutex_unlock(&inst->registeredbufs.lock); if (rc < 0) goto exit; @@ -563,7 +564,8 @@ int unmap_and_deregister_buf(struct msm_vidc_inst *inst, return -EINVAL; } - mutex_lock(&inst->registeredbufs.lock); + WARN(!mutex_is_locked(&inst->registeredbufs.lock), + "Registered buf lock is not acquired for %s", __func__); /* * Make sure the buffer to be unmapped and deleted @@ -627,7 +629,6 @@ int unmap_and_deregister_buf(struct msm_vidc_inst *inst, dprintk(VIDC_DBG, "[UNMAP] NOT-FREED binfo: %p\n", temp); } exit: - mutex_unlock(&inst->registeredbufs.lock); return 0; } @@ -881,8 +882,9 @@ int msm_vidc_qbuf(void *instance, struct v4l2_buffer *b) b->m.planes[i].m.userptr = 0; continue; } - + mutex_lock(&inst->registeredbufs.lock); binfo = get_registered_buf(inst, b, i, &plane); + mutex_unlock(&inst->registeredbufs.lock); if (!binfo) { dprintk(VIDC_ERR, "This buffer is not registered: %d, %d, %d\n", @@ -983,7 +985,9 @@ int msm_vidc_dqbuf(void *instance, struct v4l2_buffer *b) buffer_info->dequeued = true; dprintk(VIDC_DBG, "[DEQUEUED]: fd[0] = %d\n", buffer_info->fd[0]); + mutex_lock(&inst->registeredbufs.lock); rc = unmap_and_deregister_buf(inst, buffer_info); + mutex_unlock(&inst->registeredbufs.lock); } else rc = output_buffer_cache_invalidate(inst, buffer_info); diff --git a/drivers/media/platform/msm/vidc/msm_vidc_common.c b/drivers/media/platform/msm/vidc/msm_vidc_common.c index 83829609778..29dd2637f2c 100644 --- a/drivers/media/platform/msm/vidc/msm_vidc_common.c +++ b/drivers/media/platform/msm/vidc/msm_vidc_common.c @@ -605,7 +605,7 @@ static void handle_event_change(enum command_response cmd, void *data) "RELEASE REFERENCE EVENT FROM F/W - fd = %d offset = %d\n", ptr[0], ptr[1]); - mutex_lock(&inst->sync_lock); + mutex_lock(&inst->registeredbufs.lock); /* Decrement buffer reference count*/ buf_ref_put(inst, binfo); @@ -616,7 +616,7 @@ static void handle_event_change(enum command_response cmd, void *data) if (unmap_and_deregister_buf(inst, binfo)) dprintk(VIDC_ERR, "%s: buffer unmap failed\n", __func__); - mutex_unlock(&inst->sync_lock); + mutex_unlock(&inst->registeredbufs.lock); /*send event to client*/ v4l2_event_queue_fh(&inst->event_handler, From b31a31a84ca76d1ac710ecb56064881800e60c02 Mon Sep 17 00:00:00 2001 From: Trishansh Bhardwaj Date: Wed, 29 Jun 2016 14:34:31 +0530 Subject: [PATCH 114/168] msm: camera: Fix memory read security flaw Adds bound check on reg_cfg_cmd->u.dmi_info.hi_tbl_offset. IOCTL VIDIOC_MSM_VFE_REG_CFG uses usersupplied value without performing bounds check for following cmd_type. VFE_READ_DMI_16BIT VFE_READ_DMI_32BIT VFE_READ_DMI_64BIT Change-Id: I554c45ef3a172f5b5891b67a7e8e7a1f3f3882ed Signed-off-by: Trishansh Bhardwaj Signed-off-by: VijayaKumar T M --- drivers/media/platform/msm/camera_v2/isp/msm_isp_util.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/media/platform/msm/camera_v2/isp/msm_isp_util.c b/drivers/media/platform/msm/camera_v2/isp/msm_isp_util.c index dbc27ad8d89..cda566de6ed 100644 --- a/drivers/media/platform/msm/camera_v2/isp/msm_isp_util.c +++ b/drivers/media/platform/msm/camera_v2/isp/msm_isp_util.c @@ -521,7 +521,8 @@ static int msm_isp_send_hw_cmd(struct vfe_device *vfe_dev, case VFE_READ_DMI_16BIT: case VFE_READ_DMI_32BIT: case VFE_READ_DMI_64BIT: { - if (reg_cfg_cmd->cmd_type == VFE_WRITE_DMI_64BIT) { + if (reg_cfg_cmd->cmd_type == VFE_WRITE_DMI_64BIT || + reg_cfg_cmd->cmd_type == VFE_READ_DMI_64BIT) { if ((reg_cfg_cmd->u.dmi_info.hi_tbl_offset <= reg_cfg_cmd->u.dmi_info.lo_tbl_offset) || (reg_cfg_cmd->u.dmi_info.hi_tbl_offset - From 9e9c3c34d0e9006f85410707a4ce7058bee1974e Mon Sep 17 00:00:00 2001 From: Vijaya Kumar T M Date: Mon, 11 Jul 2016 12:59:56 +0530 Subject: [PATCH 115/168] msm: camera: ispif: Validate VFE num input during reset Userspace supplies the actual number of used VFEs in session to ISPIF. Validate the userspace input value and if found to be invalid, return error. CRs-Fixed: 898074 Change-Id: I3288ddb6404e817a705a92281b4c54666f372c56 Signed-off-by: Venu Yeshala Signed-off-by: Vijaya Kumar T M --- .../media/platform/msm/camera_v2/ispif/msm_ispif.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/drivers/media/platform/msm/camera_v2/ispif/msm_ispif.c b/drivers/media/platform/msm/camera_v2/ispif/msm_ispif.c index 3f8f844d454..034fdaaf5f0 100755 --- a/drivers/media/platform/msm/camera_v2/ispif/msm_ispif.c +++ b/drivers/media/platform/msm/camera_v2/ispif/msm_ispif.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2014, The Linux Foundation. All rights reserved. +/* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -946,9 +946,15 @@ static irqreturn_t msm_io_ispif_irq(int irq_num, void *data) static int msm_ispif_set_vfe_info(struct ispif_device *ispif, struct msm_ispif_vfe_info *vfe_info) { - memcpy(&ispif->vfe_info, vfe_info, sizeof(struct msm_ispif_vfe_info)); - if (ispif->vfe_info.num_vfe > ispif->hw_num_isps) + if (!vfe_info || (vfe_info->num_vfe <= 0) || + ((uint32_t)(vfe_info->num_vfe) > ispif->hw_num_isps)) { + pr_err("Invalid VFE info: %p %d\n", vfe_info, + (vfe_info ? vfe_info->num_vfe:0)); return -EINVAL; + } + + memcpy(&ispif->vfe_info, vfe_info, sizeof(struct msm_ispif_vfe_info)); + return 0; } From 91c9804b2941f5ad77823207da77493da08e4dd4 Mon Sep 17 00:00:00 2001 From: Vijaya Kumar T M Date: Mon, 11 Jul 2016 13:08:32 +0530 Subject: [PATCH 116/168] msm: camera: isp: Fix warning and errors based on static analysis This change fixes the warning/errors from static analysis Conflicts: drivers/media/platform/msm/camera_v2/isp/msm_isp_axi_util.c drivers/media/platform/msm/camera_v2/isp/msm_isp_util.c drivers/media/platform/msm/camera_v2/ispif/msm_ispif.c CRs-fixed: 992942 Change-Id: Iaf90ab4c1d17f903d03458d76cab1b4c0a5c8836 Signed-off-by: Jing Zhou Signed-off-by: Vijaya Kumar T M --- .../msm/camera_v2/isp/msm_isp_axi_util.c | 16 ++++++---------- .../platform/msm/camera_v2/isp/msm_isp_util.c | 4 +--- .../platform/msm/camera_v2/ispif/msm_ispif.c | 14 +++++++------- 3 files changed, 14 insertions(+), 20 deletions(-) diff --git a/drivers/media/platform/msm/camera_v2/isp/msm_isp_axi_util.c b/drivers/media/platform/msm/camera_v2/isp/msm_isp_axi_util.c index 22572aaf69e..6c4bcbe8288 100644 --- a/drivers/media/platform/msm/camera_v2/isp/msm_isp_axi_util.c +++ b/drivers/media/platform/msm/camera_v2/isp/msm_isp_axi_util.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2014, The Linux Foundation. All rights reserved. +/* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -28,15 +28,11 @@ int msm_isp_axi_create_stream( struct msm_vfe_axi_shared_data *axi_data, struct msm_vfe_axi_stream_request_cmd *stream_cfg_cmd) { - int i, rc = -1; - for (i = 0; i < MAX_NUM_STREAM; i++) { - if (axi_data->stream_info[i].state == AVALIABLE) - break; - } - - if (i == MAX_NUM_STREAM) { - pr_err("%s: No free stream\n", __func__); - return rc; + uint32_t i = stream_cfg_cmd->stream_src; + if (i >= VFE_AXI_SRC_MAX) { + pr_err("%s:%d invalid stream_src %d\n", __func__, __LINE__, + stream_cfg_cmd->stream_src); + return -EINVAL; } if ((axi_data->stream_handle_cnt << 8) == 0) diff --git a/drivers/media/platform/msm/camera_v2/isp/msm_isp_util.c b/drivers/media/platform/msm/camera_v2/isp/msm_isp_util.c index cda566de6ed..03fd016c7a8 100644 --- a/drivers/media/platform/msm/camera_v2/isp/msm_isp_util.c +++ b/drivers/media/platform/msm/camera_v2/isp/msm_isp_util.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2014, The Linux Foundation. All rights reserved. +/* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -1020,8 +1020,6 @@ void msm_isp_update_error_frame_count(struct vfe_device *vfe_dev) { struct msm_vfe_error_info *error_info = &vfe_dev->error_info; error_info->info_dump_frame_count++; - if (error_info->info_dump_frame_count == 0) - error_info->info_dump_frame_count++; } void msm_isp_process_error_info(struct vfe_device *vfe_dev) diff --git a/drivers/media/platform/msm/camera_v2/ispif/msm_ispif.c b/drivers/media/platform/msm/camera_v2/ispif/msm_ispif.c index 034fdaaf5f0..c73127b4a51 100755 --- a/drivers/media/platform/msm/camera_v2/ispif/msm_ispif.c +++ b/drivers/media/platform/msm/camera_v2/ispif/msm_ispif.c @@ -304,23 +304,23 @@ static void msm_ispif_sel_csid_core(struct ispif_device *ispif, switch (intftype) { case PIX0: data &= ~(BIT(1) | BIT(0)); - data |= csid; + data |= (uint32_t)csid; break; case RDI0: data &= ~(BIT(5) | BIT(4)); - data |= (csid << 4); + data |= (uint32_t)(csid << 4); break; case PIX1: data &= ~(BIT(9) | BIT(8)); - data |= (csid << 8); + data |= (uint32_t)(csid << 8); break; case RDI1: data &= ~(BIT(13) | BIT(12)); - data |= (csid << 12); + data |= (uint32_t)(csid << 12); break; case RDI2: data &= ~(BIT(21) | BIT(20)); - data |= (csid << 20); + data |= (uint32_t)(csid << 20); break; } @@ -396,9 +396,9 @@ static void msm_ispif_enable_intf_cids(struct ispif_device *ispif, data = msm_camera_io_r(ispif->base + intf_addr); if (enable) - data |= cid_mask; + data |= (uint32_t)cid_mask; else - data &= ~cid_mask; + data &= ~((uint32_t)cid_mask); msm_camera_io_w_mb(data, ispif->base + intf_addr); } From a98d6928b2af58bf2cdefa874f1fa725d43b7ec4 Mon Sep 17 00:00:00 2001 From: Sunil Khatri Date: Tue, 8 Mar 2016 12:52:17 +0530 Subject: [PATCH 117/168] msm: kgsl: Cast perfcounter query count to an unsigned int In adreno_perfcounter_query_group() make sure to cast the user passed count to an unsigned int before comparing it to the group count. Otherwise the user count could be interpeted as a signed int and hilarity ensues. Change-Id: Ic0dedbad825f5b3fd4434f9b9f6d4d308206c0d9 Signed-off-by: Jordan Crouse Signed-off-by: Sunil Khatri --- drivers/gpu/msm/adreno.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/msm/adreno.c b/drivers/gpu/msm/adreno.c index 64600e1caea..78b0338dda1 100644 --- a/drivers/gpu/msm/adreno.c +++ b/drivers/gpu/msm/adreno.c @@ -629,7 +629,7 @@ int adreno_perfcounter_query_group(struct adreno_device *adreno_dev, return 0; } - t = min_t(int, group->reg_count, count); + t = min_t(unsigned int, group->reg_count, count); buf = kmalloc(t * sizeof(unsigned int), GFP_KERNEL); if (buf == NULL) { From ceb77b452c71b6cc0afadf02547f3494594631ad Mon Sep 17 00:00:00 2001 From: Arun KS Date: Wed, 11 May 2016 10:11:36 +0530 Subject: [PATCH 118/168] msm: perf: Do not allocate new hw_event if event is duplicate. During a perf_event_enable, kernel/events/core.c calls pmu->add() which is platform implementation(arch/arm/kernel/perf_event.c). Due to the duplicate constraints, arch/arm/mach-msm/perf_event_msm_krait_l2.c drivers marks the event as OFF but returns TRUE to perf_event.c which goes ahead and allocates the hw_event and enables it. Since event is marked OFF, kernel events core will try to enable this event again during next perf_event_enable. Which results in same event enabled on multiple hw_events. But during the perf_release, event struct is freed and only one hw_event is released. This results in dereferencing the invalid pointer and hence the crash. Fix this by returning error in case of constraint event duplicate. Hence avoiding the same event programmed on multiple hw event counters. Change-Id: Ia3360be027dfe87ac753191ffe7e0bc947e72455 Signed-off-by: Arun KS Signed-off-by: Chetan Ravindranath --- arch/arm/kernel/perf_event.c | 1 + arch/arm/mach-msm/perf_event_msm_krait_l2.c | 1 + 2 files changed, 2 insertions(+) diff --git a/arch/arm/kernel/perf_event.c b/arch/arm/kernel/perf_event.c index 1a09188f12a..5989418ca04 100644 --- a/arch/arm/kernel/perf_event.c +++ b/arch/arm/kernel/perf_event.c @@ -311,6 +311,7 @@ armpmu_add(struct perf_event *event, int flags) pr_err("Event: %llx failed constraint check.\n", event->attr.config); event->state = PERF_EVENT_STATE_OFF; + err = -EPERM; goto out; } diff --git a/arch/arm/mach-msm/perf_event_msm_krait_l2.c b/arch/arm/mach-msm/perf_event_msm_krait_l2.c index 562e90e4737..8deb3cceaa9 100644 --- a/arch/arm/mach-msm/perf_event_msm_krait_l2.c +++ b/arch/arm/mach-msm/perf_event_msm_krait_l2.c @@ -466,6 +466,7 @@ static int msm_l2_test_set_ev_constraint(struct perf_event *event) if (!(event->cpu < 0)) { event->state = PERF_EVENT_STATE_OFF; event->attr.constraint_duplicate = 1; + err = -EPERM; } } out: From c6869db5a6e66b2ee2408fb98b7b2e4858b90ea9 Mon Sep 17 00:00:00 2001 From: Rajesh Bondugula Date: Wed, 13 Apr 2016 14:31:58 -0700 Subject: [PATCH 119/168] msm: camera: sensor: Validate step_boundary step_boundary can take values upto the total_steps Validate the step_boundary before consuming it. Convert the type of step_index and region_index to uint16_t step_index and region_index cannot be negative. CRs-Fixed: 1001092 Change-Id: I1f23fd6f28bb897824a1ef99a8873b9f986eee70 Signed-off-by: Rajesh Bondugula --- .../msm/camera_v2/sensor/actuator/msm_actuator.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/drivers/media/platform/msm/camera_v2/sensor/actuator/msm_actuator.c b/drivers/media/platform/msm/camera_v2/sensor/actuator/msm_actuator.c index 35997eda8d3..14b6acf2758 100644 --- a/drivers/media/platform/msm/camera_v2/sensor/actuator/msm_actuator.c +++ b/drivers/media/platform/msm/camera_v2/sensor/actuator/msm_actuator.c @@ -389,7 +389,7 @@ static int32_t msm_actuator_init_step_table(struct msm_actuator_ctrl_t *a_ctrl, { int16_t code_per_step = 0; int16_t cur_code = 0; - int16_t step_index = 0, region_index = 0; + uint16_t step_index = 0, region_index = 0; uint16_t step_boundary = 0; uint32_t max_code_size = 1; uint16_t data_size = set_info->actuator_params.data_size; @@ -428,6 +428,15 @@ static int32_t msm_actuator_init_step_table(struct msm_actuator_ctrl_t *a_ctrl, step_boundary = a_ctrl->region_params[region_index]. step_bound[MOVE_NEAR]; + if (step_boundary > + set_info->af_tuning_params.total_steps) { + pr_err("invalid step_boundary = %d, max_val = %d", + step_boundary, + set_info->af_tuning_params.total_steps); + kfree(a_ctrl->step_position_table); + a_ctrl->step_position_table = NULL; + return -EINVAL; + } for (; step_index <= step_boundary; step_index++) { cur_code += code_per_step; From b56005e1ac86bd2bfecfbee0dafda03f88c0e929 Mon Sep 17 00:00:00 2001 From: Takashi Iwai Date: Thu, 14 Jan 2016 16:30:58 +0100 Subject: [PATCH 120/168] ALSA: timer: Harden slave timer list handling A slave timer instance might be still accessible in a racy way while operating the master instance as it lacks of locking. Since the master operation is mostly protected with timer->lock, we should cope with it while changing the slave instance, too. Also, some linked lists (active_list and ack_list) of slave instances aren't unlinked immediately at stopping or closing, and this may lead to unexpected accesses. This patch tries to address these issues. It adds spin lock of timer->lock (either from master or slave, which is equivalent) in a few places. For avoiding a deadlock, we ensure that the global slave_active_lock is always locked at first before each timer lock. Also, ack and active_list of slave instances are properly unlinked at snd_timer_stop() and snd_timer_close(). Last but not least, remove the superfluous call of _snd_timer_stop() at removing slave links. This is a noop, and calling it may confuse readers wrt locking. Further cleanup will follow in a later patch. Actually we've got reports of use-after-free by syzkaller fuzzer, and this hopefully fixes these issues. Reported-by: Dmitry Vyukov Cc: Signed-off-by: Takashi Iwai (cherry picked from commit b5a663aa426f4884c71cd8580adae73f33570f0d) Change-Id: I7e7e4e1ab476f93131111d60d8f4e6a1add43193 Signed-off-by: Dennis Cagle --- sound/core/timer.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/sound/core/timer.c b/sound/core/timer.c index 6ddcf06f52f..38a137d6b04 100644 --- a/sound/core/timer.c +++ b/sound/core/timer.c @@ -215,11 +215,13 @@ static void snd_timer_check_master(struct snd_timer_instance *master) slave->slave_id == master->slave_id) { list_move_tail(&slave->open_list, &master->slave_list_head); spin_lock_irq(&slave_active_lock); + spin_lock(&master->timer->lock); slave->master = master; slave->timer = master->timer; if (slave->flags & SNDRV_TIMER_IFLG_RUNNING) list_add_tail(&slave->active_list, &master->slave_active_head); + spin_unlock(&master->timer->lock); spin_unlock_irq(&slave_active_lock); } } @@ -345,15 +347,18 @@ int snd_timer_close(struct snd_timer_instance *timeri) timer->hw.close) timer->hw.close(timer); /* remove slave links */ + spin_lock_irq(&slave_active_lock); + spin_lock(&timer->lock); list_for_each_entry_safe(slave, tmp, &timeri->slave_list_head, open_list) { - spin_lock_irq(&slave_active_lock); - _snd_timer_stop(slave, 1, SNDRV_TIMER_EVENT_RESOLUTION); list_move_tail(&slave->open_list, &snd_timer_slave_list); slave->master = NULL; slave->timer = NULL; - spin_unlock_irq(&slave_active_lock); + list_del_init(&slave->ack_list); + list_del_init(&slave->active_list); } + spin_unlock(&timer->lock); + spin_unlock_irq(&slave_active_lock); mutex_unlock(®ister_mutex); } out: @@ -440,9 +445,12 @@ static int snd_timer_start_slave(struct snd_timer_instance *timeri) spin_lock_irqsave(&slave_active_lock, flags); timeri->flags |= SNDRV_TIMER_IFLG_RUNNING; - if (timeri->master) + if (timeri->master && timeri->timer) { + spin_lock(&timeri->timer->lock); list_add_tail(&timeri->active_list, &timeri->master->slave_active_head); + spin_unlock(&timeri->timer->lock); + } spin_unlock_irqrestore(&slave_active_lock, flags); return 1; /* delayed start */ } @@ -488,6 +496,8 @@ static int _snd_timer_stop(struct snd_timer_instance * timeri, if (!keep_flag) { spin_lock_irqsave(&slave_active_lock, flags); timeri->flags &= ~SNDRV_TIMER_IFLG_RUNNING; + list_del_init(&timeri->ack_list); + list_del_init(&timeri->active_list); spin_unlock_irqrestore(&slave_active_lock, flags); } goto __end; From 3f82d3527ce3a56f6666fe68664e20ce122ee93c Mon Sep 17 00:00:00 2001 From: Srinivasarao P Date: Tue, 1 Mar 2016 12:16:03 +0530 Subject: [PATCH 121/168] perf: duplicate deletion of perf event a malicious app can open a perf event with constraint_duplicate bit set, disable the event, and close the fd. On closing the fd, the perf_release() modification causes the kernel to clean up the event as if it still were enabled, leading to the event being removed from a list twice. CRs-Fixed: 977563 Change-Id: I5fbec3722407d2f3d0ff0d9f7097c5889e31fd62 Signed-off-by: Srinivasarao P --- kernel/events/core.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kernel/events/core.c b/kernel/events/core.c index 7dd822b5e39..868300dba34 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -6243,6 +6243,9 @@ SYSCALL_DEFINE5(perf_event_open, if (err) return err; + if (attr.constraint_duplicate || attr.__reserved_1) + return -EINVAL; + if (!attr.exclude_kernel) { if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN)) return -EACCES; From 620581c21cd94af0252f630e999e28d6a27477f7 Mon Sep 17 00:00:00 2001 From: Arun Kumar Neelakantam Date: Wed, 10 Aug 2016 17:47:36 +0530 Subject: [PATCH 122/168] msm: ipc: Bind only a client port as control port IPC Router binds any port as a control port and moves it from the client port list to control port list. Misbehaving clients can exploit this incorrect behavior. IPC Router to check if the port is a client port before binding it as a control port. CRs-Fixed: 974577 Change-Id: I9f189b76967d5f85750218a7cb6537d187a69663 Signed-off-by: Karthikeyan Ramasubramanian Signed-off-by: Arun Kumar Neelakantam --- arch/arm/mach-msm/ipc_router.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm/mach-msm/ipc_router.c b/arch/arm/mach-msm/ipc_router.c index fa9546984c3..f66fc93283a 100644 --- a/arch/arm/mach-msm/ipc_router.c +++ b/arch/arm/mach-msm/ipc_router.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2014, The Linux Foundation. All rights reserved. +/* Copyright (c) 2011-2014, 2016 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -2853,7 +2853,7 @@ int msm_ipc_router_get_curr_pkt_size(struct msm_ipc_port *port_ptr) int msm_ipc_router_bind_control_port(struct msm_ipc_port *port_ptr) { - if (!port_ptr) + if (unlikely(!port_ptr || port_ptr->type != CLIENT_PORT)) return -EINVAL; down_write(&local_ports_lock_lha2); From 14054af4a935086bc7917dab8878eb8cd8c76da5 Mon Sep 17 00:00:00 2001 From: Arun Kumar Neelakantam Date: Wed, 27 Jan 2016 18:46:01 +0530 Subject: [PATCH 123/168] msm: ipc: fix leak of kernel memory to userspace The service info structure is allocated with uninitialized memory for the max number of services and returns the complete structure to the usersapce resulting in the information leak if lookup operation finds less number of services than the requested number. Check the minimum of requested and available services and copy the minimum information to the user-space. CRs-Fixed: 965934 Change-Id: Ic97f875855fdc6440c1db1d8d0338ee8b03a9d0a Signed-off-by: Arun Kumar Neelakantam --- arch/arm/mach-msm/ipc_socket.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/arch/arm/mach-msm/ipc_socket.c b/arch/arm/mach-msm/ipc_socket.c index 77943e3727d..751ebb0e78b 100644 --- a/arch/arm/mach-msm/ipc_socket.c +++ b/arch/arm/mach-msm/ipc_socket.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2014, The Linux Foundation. All rights reserved. +/* Copyright (c) 2011-2014, 2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -516,13 +516,18 @@ static int msm_ipc_router_ioctl(struct socket *sock, ret = copy_to_user((void *)arg, &server_arg, sizeof(server_arg)); - if (srv_info_sz) { + + n = min(server_arg.num_entries_found, + server_arg.num_entries_in_array); + + if (ret == 0 && n) { ret = copy_to_user((void *)(arg + sizeof(server_arg)), - srv_info, srv_info_sz); - if (ret) - ret = -EFAULT; - kfree(srv_info); + srv_info, n * sizeof(*srv_info)); } + + if (ret) + ret = -EFAULT; + kfree(srv_info); break; case IPC_ROUTER_IOCTL_BIND_CONTROL_PORT: From 0c70d3fa8030f452cd8354e440b5bd00f93348c3 Mon Sep 17 00:00:00 2001 From: Sunil Khatri Date: Wed, 22 Jun 2016 14:45:31 +0530 Subject: [PATCH 124/168] ashmem: Validate ashmem memory with fops pointer Validate the ashmem memory entry against f_op pointer rather then comparing its name with path of the dentry. This is to avoid any invalid access to ashmem area in cases where some one deliberately set the dentry name to /ashmem. Change-Id: I74e50cd244f68cb13009cf2355e528485f4de34b Signed-off-by: Sunil Khatri --- drivers/staging/android/ashmem.c | 38 +++++++++++++++----------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/drivers/staging/android/ashmem.c b/drivers/staging/android/ashmem.c index 41ebc1c4b34..de17b113578 100644 --- a/drivers/staging/android/ashmem.c +++ b/drivers/staging/android/ashmem.c @@ -805,11 +805,26 @@ static long ashmem_ioctl(struct file *file, unsigned int cmd, unsigned long arg) return ret; } +static const struct file_operations ashmem_fops = { + .owner = THIS_MODULE, + .open = ashmem_open, + .release = ashmem_release, + .read = ashmem_read, + .llseek = ashmem_llseek, + .mmap = ashmem_mmap, + .unlocked_ioctl = ashmem_ioctl, + .compat_ioctl = ashmem_ioctl, +}; + +static struct miscdevice ashmem_misc = { + .minor = MISC_DYNAMIC_MINOR, + .name = "ashmem", + .fops = &ashmem_fops, +}; + static int is_ashmem_file(struct file *file) { - char fname[256], *name; - name = dentry_path(file->f_dentry, fname, 256); - return strcmp(name, "/ashmem") ? 0 : 1; + return file->f_op == &ashmem_fops; } int get_ashmem_file(int fd, struct file **filp, struct file **vm_file, @@ -858,23 +873,6 @@ void put_ashmem_file(struct file *file) } EXPORT_SYMBOL(put_ashmem_file); -static const struct file_operations ashmem_fops = { - .owner = THIS_MODULE, - .open = ashmem_open, - .release = ashmem_release, - .read = ashmem_read, - .llseek = ashmem_llseek, - .mmap = ashmem_mmap, - .unlocked_ioctl = ashmem_ioctl, - .compat_ioctl = ashmem_ioctl, -}; - -static struct miscdevice ashmem_misc = { - .minor = MISC_DYNAMIC_MINOR, - .name = "ashmem", - .fops = &ashmem_fops, -}; - static int __init ashmem_init(void) { int ret; From bf2ead33701433abcf614f9cccdcfaeef0a52d36 Mon Sep 17 00:00:00 2001 From: Weiyin Jiang Date: Tue, 26 Apr 2016 14:35:38 +0800 Subject: [PATCH 125/168] ASoC: msm: audio-effects: misc fixes in h/w accelerated effect Adding memory copy size check and integer overflow check in h/w accelerated effect driver. Change-Id: I17d4cc0a38770f0c5067fa8047cd63e7bf085e48 CRs-Fixed: 1006609 Signed-off-by: Weiyin Jiang --- sound/soc/msm/qdsp6v2/q6asm.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sound/soc/msm/qdsp6v2/q6asm.c b/sound/soc/msm/qdsp6v2/q6asm.c index 67963783265..b6d165dcfca 100644 --- a/sound/soc/msm/qdsp6v2/q6asm.c +++ b/sound/soc/msm/qdsp6v2/q6asm.c @@ -1051,6 +1051,12 @@ int q6asm_audio_client_buf_alloc_contiguous(unsigned int dir, ac->port[dir].buf = buf; + /* check for integer overflow */ + if ((bufcnt > 0) && ((INT_MAX / bufcnt) < bufsz)) { + pr_err("%s: integer overflow\n", __func__); + mutex_unlock(&ac->cmd_lock); + goto fail; + } bytes_to_alloc = bufsz * bufcnt; /* The size to allocate should be multiple of 4K bytes */ From b537e5db2660875997ea33ffe326c6e74fd8618c Mon Sep 17 00:00:00 2001 From: Vasily Kulikov Date: Wed, 9 Sep 2015 15:36:00 -0700 Subject: [PATCH 126/168] include/linux/poison.h: fix LIST_POISON{1,2} offset Poison pointer values should be small enough to find a room in non-mmap'able/hardly-mmap'able space. E.g. on x86 "poison pointer space" is located starting from 0x0. Given unprivileged users cannot mmap anything below mmap_min_addr, it should be safe to use poison pointers lower than mmap_min_addr. The current poison pointer values of LIST_POISON{1,2} might be too big for mmap_min_addr values equal or less than 1 MB (common case, e.g. Ubuntu uses only 0x10000). There is little point to use such a big value given the "poison pointer space" below 1 MB is not yet exhausted. Changing it to a smaller value solves the problem for small mmap_min_addr setups. The values are suggested by Solar Designer: http://www.openwall.com/lists/oss-security/2015/05/02/6 Change-Id: I4aff5bf048c42d1ca6c94f356a987e4ada563398 Signed-off-by: Vasily Kulikov Cc: Solar Designer Cc: Thomas Gleixner Cc: "Kirill A. Shutemov" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds Git-commit: 8a5e5e02fc83aaf67053ab53b359af08c6c49aaf Git-repo: https://us.codeaurora.org/cgit/quic/la/kernel/msm CRs-Fixed: 987330 Signed-off-by: Ravi Kumar Siddojigari --- include/linux/poison.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/poison.h b/include/linux/poison.h index 2110a81c5e2..253c9b4198e 100644 --- a/include/linux/poison.h +++ b/include/linux/poison.h @@ -19,8 +19,8 @@ * under normal circumstances, used to verify that nobody uses * non-initialized list entries. */ -#define LIST_POISON1 ((void *) 0x00100100 + POISON_POINTER_DELTA) -#define LIST_POISON2 ((void *) 0x00200200 + POISON_POINTER_DELTA) +#define LIST_POISON1 ((void *) 0x100 + POISON_POINTER_DELTA) +#define LIST_POISON2 ((void *) 0x200 + POISON_POINTER_DELTA) /********** include/linux/timer.h **********/ /* From 0b9762310abe5a24efcf7ecc8bb8671f890ea2c1 Mon Sep 17 00:00:00 2001 From: Vijayavardhan Vennapusa Date: Thu, 5 May 2016 14:37:08 +0530 Subject: [PATCH 127/168] USB: dwc3: debugfs: Add boundary check in dwc3_store_ep_num() User can pass arguments as part of write to requests and endpoint number will be calculated based on the arguments. There is a chance that driver can access ep structue that is not allocated due to invalid arguments passed by user. Hence fix the issue by having check and return error in case of invalid arguments. Change-Id: I060ea878b55ce0f9983b91c50e58718c8a2c2fa1 Signed-off-by: Vijayavardhan Vennapusa --- drivers/usb/dwc3/debugfs.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/usb/dwc3/debugfs.c b/drivers/usb/dwc3/debugfs.c index eb67c47295d..58d674b3993 100644 --- a/drivers/usb/dwc3/debugfs.c +++ b/drivers/usb/dwc3/debugfs.c @@ -661,7 +661,7 @@ static ssize_t dwc3_store_ep_num(struct file *file, const char __user *ubuf, struct seq_file *s = file->private_data; struct dwc3 *dwc = s->private; char kbuf[10]; - unsigned int num, dir; + unsigned int num, dir, temp; unsigned long flags; memset(kbuf, 0, 10); @@ -672,8 +672,15 @@ static ssize_t dwc3_store_ep_num(struct file *file, const char __user *ubuf, if (sscanf(kbuf, "%u %u", &num, &dir) != 2) return -EINVAL; + if (dir != 0 && dir != 1) + return -EINVAL; + + temp = (num << 1) + dir; + if (temp >= DWC3_ENDPOINTS_NUM) + return -EINVAL; + spin_lock_irqsave(&dwc->lock, flags); - ep_num = (num << 1) + dir; + ep_num = temp; spin_unlock_irqrestore(&dwc->lock, flags); return count; From bc29bd51028d8d13ce3570f16199b09fd42125c1 Mon Sep 17 00:00:00 2001 From: Josh Kirsch Date: Mon, 2 May 2016 14:55:04 -0700 Subject: [PATCH 128/168] drivers: soc: Add buffer overflow check for svc send request Add buffer overflow check in voice_svc_send_req. CRs-fixed: 1010081 Change-Id: I4ae703334b0cf04f327b392bc9cd6febd4ad32f2 Signed-off-by: Josh Kirsch --- arch/arm/mach-msm/qdsp6v2/voice_svc.c | 44 ++++++++++++++++----------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/arch/arm/mach-msm/qdsp6v2/voice_svc.c b/arch/arm/mach-msm/qdsp6v2/voice_svc.c index 9ea6a4c4b2b..6db47165e98 100755 --- a/arch/arm/mach-msm/qdsp6v2/voice_svc.c +++ b/arch/arm/mach-msm/qdsp6v2/voice_svc.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014, The Linux Foundation. All rights reserved. +/* Copyright (c) 2014, 2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -166,6 +166,7 @@ static int voice_svc_send_req(struct voice_svc_cmd_request *apr_request, void *apr_handle = NULL; struct apr_data *aprdata = NULL; uint32_t user_payload_size = 0; + uint32_t payload_size = 0; if (apr_request == NULL) { pr_err("%s: apr_request is NULL\n", __func__); @@ -175,17 +176,21 @@ static int voice_svc_send_req(struct voice_svc_cmd_request *apr_request, } user_payload_size = apr_request->payload_size; - - aprdata = kmalloc(sizeof(struct apr_data) + user_payload_size, - GFP_KERNEL); - - if (aprdata == NULL) { - pr_err("%s: aprdata kmalloc failed.", __func__); - - ret = -ENOMEM; + payload_size = sizeof(struct apr_data) + user_payload_size; + if (payload_size <= user_payload_size) { + pr_err("%s: invalid payload size ( 0x%x ).\n", + __func__, user_payload_size); + ret = -EINVAL; goto done; + } else { + aprdata = kmalloc(payload_size, GFP_KERNEL); + if (aprdata == NULL) { + ret = -ENOMEM; + goto done; + } } + voice_svc_update_hdr(apr_request, aprdata, prtd); if (!strncmp(apr_request->svc_name, VOICE_SVC_CVS_STR, @@ -327,6 +332,7 @@ static long voice_svc_ioctl(struct file *file, unsigned int cmd, void __user *arg = (void __user *)u_arg; uint32_t user_payload_size = 0; unsigned long spin_flags; + uint32_t payload_size = 0; pr_debug("%s: cmd: %u\n", __func__, cmd); @@ -357,17 +363,21 @@ static long voice_svc_ioctl(struct file *file, unsigned int cmd, user_payload_size = ((struct voice_svc_cmd_request*)arg)->payload_size; - - apr_request = kmalloc(sizeof(struct voice_svc_cmd_request) + - user_payload_size, GFP_KERNEL); - - if (apr_request == NULL) { - pr_err("%s: apr_request kmalloc failed.", __func__); - - ret = -ENOMEM; + payload_size = sizeof(struct voice_svc_cmd_request) + user_payload_size; + if (payload_size <= user_payload_size) { + pr_err("%s: invalid payload size ( 0x%x ).\n", + __func__, user_payload_size); + ret = -EINVAL; goto done; + } else { + apr_request = kmalloc(payload_size, GFP_KERNEL); + if (apr_request == NULL) { + ret = -ENOMEM; + goto done; + } } + if (copy_from_user(apr_request, arg, sizeof(struct voice_svc_cmd_request) + user_payload_size)) { From ad84ae0fcc8ed6a882634e10b42b6fb4081efccb Mon Sep 17 00:00:00 2001 From: Divya Ponnusamy Date: Wed, 24 Aug 2016 17:06:54 +0530 Subject: [PATCH 129/168] msm: kgsl: Change %p to %pK in debug messages The format specifier %p can leak kernel addresses while not valuing the kptr_restrict system settings. Use %pK instead of %p, which evaluates whether kptr_restrict is set. Change-Id: I0778e43e0a03852ca2944377256a7b401586a747 Signed-off-by: Divya Ponnusamy Signed-off-by: Sudeep Yedalapure --- drivers/gpu/msm/adreno_ringbuffer.c | 15 ++++++++------- drivers/gpu/msm/kgsl.c | 7 +++---- drivers/gpu/msm/kgsl_cffdump.c | 11 +++-------- drivers/gpu/msm/kgsl_gpummu.c | 7 ++++--- drivers/gpu/msm/kgsl_iommu.c | 22 +++++++++++----------- drivers/gpu/msm/kgsl_mmu.c | 5 +++-- drivers/gpu/msm/kgsl_pwrctrl.c | 4 ++-- drivers/gpu/msm/kgsl_sharedmem.c | 2 +- drivers/gpu/msm/kgsl_snapshot.c | 3 ++- 9 files changed, 37 insertions(+), 39 deletions(-) diff --git a/drivers/gpu/msm/adreno_ringbuffer.c b/drivers/gpu/msm/adreno_ringbuffer.c index 126de1dc9a5..0122dda93b6 100644 --- a/drivers/gpu/msm/adreno_ringbuffer.c +++ b/drivers/gpu/msm/adreno_ringbuffer.c @@ -1,4 +1,5 @@ -/* Copyright (c) 2002,2007-2014, The Linux Foundation. All rights reserved. +/* Copyright (c) 2002,2007-2014,2016, The Linux Foundation. All rights + * reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -982,7 +983,7 @@ static bool _parse_ibs(struct kgsl_device_private *dev_priv, level++; - KGSL_CMD_INFO(dev_priv->device, "ib: gpuaddr:0x%08x, wc:%d, hptr:%p\n", + KGSL_CMD_INFO(dev_priv->device, "ib: gpuaddr:0x%08x, wc:%d, hptr:%pK\n", gpuaddr, sizedwords, hostaddr); mb(); @@ -1003,8 +1004,8 @@ static bool _parse_ibs(struct kgsl_device_private *dev_priv, cur_ret = _handle_type3(dev_priv, hostaddr); break; default: - KGSL_CMD_ERR(dev_priv->device, "unexpected type: " - "type:%d, word:0x%08x @ 0x%p, gpu:0x%08x\n", + KGSL_CMD_ERR(dev_priv->device, + "unexpected type: type:%d, word:0x%08x @ 0x%pK, gpu:0x%08x\n", *hostaddr >> 30, *hostaddr, hostaddr, gpuaddr+4*(sizedwords-dwords_left)); cur_ret = false; @@ -1015,7 +1016,7 @@ static bool _parse_ibs(struct kgsl_device_private *dev_priv, if (!cur_ret) { KGSL_CMD_ERR(dev_priv->device, "bad sub-type: #:%d/%d, v:0x%08x" - " @ 0x%p[gb:0x%08x], level:%d\n", + " @ 0x%pK[gb:0x%08x], level:%d\n", sizedwords-dwords_left, sizedwords, *hostaddr, hostaddr, gpuaddr+4*(sizedwords-dwords_left), level); @@ -1035,7 +1036,7 @@ static bool _parse_ibs(struct kgsl_device_private *dev_priv, if (dwords_left < 0) { KGSL_CMD_ERR(dev_priv->device, "bad count: c:%d, #:%d/%d, " - "v:0x%08x @ 0x%p[gb:0x%08x], level:%d\n", + "v:0x%08x @ 0x%pK[gb:0x%08x], level:%d\n", count, sizedwords-(dwords_left+count), sizedwords, *(hostaddr-count), hostaddr-count, gpuaddr+4*(sizedwords-(dwords_left+count)), @@ -1055,7 +1056,7 @@ done: if (!ret) KGSL_DRV_ERR(dev_priv->device, "parsing failed: gpuaddr:0x%08x, " - "host:0x%p, wc:%d\n", gpuaddr, hoststart, sizedwords); + "host:0x%pK, wc:%d\n", gpuaddr, hoststart, sizedwords); level--; diff --git a/drivers/gpu/msm/kgsl.c b/drivers/gpu/msm/kgsl.c index 71f907227de..a062d8104a9 100644 --- a/drivers/gpu/msm/kgsl.c +++ b/drivers/gpu/msm/kgsl.c @@ -1613,7 +1613,7 @@ void kgsl_dump_syncpoints(struct kgsl_device *device, } case KGSL_CMD_SYNCPOINT_TYPE_FENCE: if (event->handle) - dev_err(device->dev, " fence: [%p] %s\n", + dev_err(device->dev, " fence: [%pK] %s\n", event->handle->fence, event->handle->name); else @@ -4384,9 +4384,8 @@ int kgsl_device_platform_probe(struct kgsl_device *device) disable_irq(device->pwrctrl.interrupt_num); KGSL_DRV_INFO(device, - "dev_id %d regs phys 0x%08lx size 0x%08x virt %p\n", - device->id, device->reg_phys, device->reg_len, - device->reg_virt); + "dev_id %d regs phys 0x%08lx size 0x%08x\n", + device->id, device->reg_phys, device->reg_len); rwlock_init(&device->context_lock); diff --git a/drivers/gpu/msm/kgsl_cffdump.c b/drivers/gpu/msm/kgsl_cffdump.c index 918e4a39749..ffa55101682 100644 --- a/drivers/gpu/msm/kgsl_cffdump.c +++ b/drivers/gpu/msm/kgsl_cffdump.c @@ -1,4 +1,5 @@ -/* Copyright (c) 2010-2012,2013, The Linux Foundation. All rights reserved. +/* Copyright (c) 2010-2012,2013,2016, The Linux Foundation. All rights + * reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -426,7 +427,7 @@ void kgsl_cffdump_syncmem(struct kgsl_device *device, src = (uint *)kgsl_gpuaddr_to_vaddr(memdesc, gpuaddr); if (memdesc->hostptr == NULL) { KGSL_CORE_ERR( - "no kernel map for gpuaddr: 0x%08x, m->host: 0x%p, phys: %pa\n", + "no kernel map for gpuaddr: 0x%08x, m->host: 0x%pK, phys: %pa\n", gpuaddr, memdesc->hostptr, &memdesc->physaddr); return; } @@ -514,9 +515,6 @@ EXPORT_SYMBOL(kgsl_cffdump_waitirq); static int subbuf_start_handler(struct rchan_buf *buf, void *subbuf, void *prev_subbuf, uint prev_padding) { - pr_debug("kgsl: cffdump: subbuf_start_handler(subbuf=%p, prev_subbuf" - "=%p, prev_padding=%08x)\n", subbuf, prev_subbuf, prev_padding); - if (relay_buf_full(buf)) { if (!suspended) { suspended = 1; @@ -573,9 +571,6 @@ static struct rchan *create_channel(unsigned subbuf_size, unsigned n_subbufs) { struct rchan *chan; - pr_info("kgsl: cffdump: relay: create_channel: subbuf_size %u, " - "n_subbufs %u, dir 0x%p\n", subbuf_size, n_subbufs, dir); - chan = relay_open("cpu", dir, subbuf_size, n_subbufs, &relay_callbacks, NULL); if (!chan) { diff --git a/drivers/gpu/msm/kgsl_gpummu.c b/drivers/gpu/msm/kgsl_gpummu.c index 2c3040f3623..d4864790ada 100644 --- a/drivers/gpu/msm/kgsl_gpummu.c +++ b/drivers/gpu/msm/kgsl_gpummu.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011,2013-2015, The Linux Foundation. All rights reserved. +/* Copyright (c) 2011,2013-2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -651,8 +651,9 @@ kgsl_gpummu_unmap(struct kgsl_pagetable *pt, #ifdef VERBOSE_DEBUG /* check if PTE exists */ if (!kgsl_pt_map_get(gpummu_pt, pte)) - KGSL_CORE_ERR("pt entry %x is already " - "unmapped for pagetable %p\n", pte, gpummu_pt); + KGSL_CORE_ERR( + "pt entry %x is already unmapped for pagetable %pK\n", + pte, gpummu_pt); #endif kgsl_pt_map_set(gpummu_pt, pte, GSL_PT_PAGE_DIRTY); superpte = pte - (pte & (GSL_PT_SUPER_PTE - 1)); diff --git a/drivers/gpu/msm/kgsl_iommu.c b/drivers/gpu/msm/kgsl_iommu.c index 6ccc3ff43d2..f755b226b07 100644 --- a/drivers/gpu/msm/kgsl_iommu.c +++ b/drivers/gpu/msm/kgsl_iommu.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2014, The Linux Foundation. All rights reserved. +/* Copyright (c) 2011-2014,2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -331,7 +331,7 @@ static int kgsl_iommu_fault_handler(struct iommu_domain *domain, iommu_dev = get_iommu_device(iommu_unit, dev); if (!iommu_dev) { - KGSL_CORE_ERR("Invalid IOMMU device %p\n", dev); + KGSL_CORE_ERR("Invalid IOMMU device %pK\n", dev); ret = -ENOSYS; goto done; } @@ -730,8 +730,8 @@ static void kgsl_detach_pagetable_iommu_domain(struct kgsl_mmu *mmu) iommu_detach_device(iommu_pt->domain, iommu_unit->dev[j].dev); iommu_unit->dev[j].attached = false; - KGSL_MEM_INFO(mmu->device, "iommu %p detached " - "from user dev of MMU: %p\n", + KGSL_MEM_INFO(mmu->device, + "iommu %pK detached from user dev of MMU: %pK\n", iommu_pt->domain, mmu); } } @@ -783,7 +783,7 @@ static int kgsl_attach_pagetable_iommu_domain(struct kgsl_mmu *mmu) } iommu_unit->dev[j].attached = true; KGSL_MEM_INFO(mmu->device, - "iommu pt %p attached to dev %p, ctx_id %d\n", + "iommu pt %pK attached to dev %pK, ctx_id %d\n", iommu_pt->domain, iommu_unit->dev[j].dev, iommu_unit->dev[j].ctx_id); } @@ -848,7 +848,7 @@ static int _get_iommu_ctxs(struct kgsl_mmu *mmu, iommu_unit->dev[iommu_unit->dev_count].kgsldev = mmu->device; KGSL_DRV_INFO(mmu->device, - "Obtained dev handle %p for iommu context %s\n", + "Obtained dev handle %pK for iommu context %s\n", iommu_unit->dev[iommu_unit->dev_count].dev, data->iommu_ctxs[i].iommu_ctx_name); @@ -1787,9 +1787,9 @@ kgsl_iommu_unmap(struct kgsl_pagetable *pt, ret = iommu_unmap_range(iommu_pt->domain, gpuaddr, range); if (ret) { - KGSL_CORE_ERR("iommu_unmap_range(%p, %x, %d) failed " - "with err: %d\n", iommu_pt->domain, gpuaddr, - range, ret); + KGSL_CORE_ERR( + "iommu_unmap_range(%pK, %x, %d) failed with err: %d\n", + iommu_pt->domain, gpuaddr, range, ret); return ret; } @@ -1816,7 +1816,7 @@ kgsl_iommu_map(struct kgsl_pagetable *pt, ret = iommu_map_range(iommu_pt->domain, iommu_virt_addr, memdesc->sg, size, protflags); if (ret) { - KGSL_CORE_ERR("iommu_map_range(%p, %x, %p, %d, %x) err: %d\n", + KGSL_CORE_ERR("iommu_map_range(%pK, %x, %pK, %d, %x) err: %d\n", iommu_pt->domain, iommu_virt_addr, memdesc->sg, size, protflags, ret); return ret; @@ -1826,7 +1826,7 @@ kgsl_iommu_map(struct kgsl_pagetable *pt, page_to_phys(kgsl_guard_page), PAGE_SIZE, protflags & ~IOMMU_WRITE); if (ret) { - KGSL_CORE_ERR("iommu_map(%p, %x, guard, %x) err: %d\n", + KGSL_CORE_ERR("iommu_map(%pK, %x, guard, %x) err: %d\n", iommu_pt->domain, iommu_virt_addr + size, protflags & ~IOMMU_WRITE, ret); diff --git a/drivers/gpu/msm/kgsl_mmu.c b/drivers/gpu/msm/kgsl_mmu.c index 27bd3e4eaf2..6663f431eb4 100755 --- a/drivers/gpu/msm/kgsl_mmu.c +++ b/drivers/gpu/msm/kgsl_mmu.c @@ -1,4 +1,5 @@ -/* Copyright (c) 2002,2007-2014, The Linux Foundation. All rights reserved. +/* Copyright (c) 2002,2007-2014,2016, The Linux Foundation. All rights + * reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -875,7 +876,7 @@ int kgsl_mmu_map_global(struct kgsl_pagetable *pagetable, /*global mappings must have the same gpu address in all pagetables*/ if (gpuaddr && gpuaddr != memdesc->gpuaddr) { - KGSL_CORE_ERR("pt %p addr mismatch phys %pa gpu 0x%0x 0x%08x", + KGSL_CORE_ERR("pt %pK addr mismatch phys %pa gpu 0x%0x 0x%08x", pagetable, &memdesc->physaddr, gpuaddr, memdesc->gpuaddr); goto error_unmap; } diff --git a/drivers/gpu/msm/kgsl_pwrctrl.c b/drivers/gpu/msm/kgsl_pwrctrl.c index e26c09c240e..aef82be96c8 100644 --- a/drivers/gpu/msm/kgsl_pwrctrl.c +++ b/drivers/gpu/msm/kgsl_pwrctrl.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2014, The Linux Foundation. All rights reserved. +/* Copyright (c) 2010-2014,2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -1136,7 +1136,7 @@ int kgsl_pwrctrl_init(struct kgsl_device *device) if (!pwr->pcl) { KGSL_PWR_ERR(device, "msm_bus_scale_register_client failed: " - "id %d table %p", device->id, + "id %d table %pK", device->id, pdata->bus_scale_table); result = -EINVAL; goto done; diff --git a/drivers/gpu/msm/kgsl_sharedmem.c b/drivers/gpu/msm/kgsl_sharedmem.c index e53a9499c80..452fe881da1 100644 --- a/drivers/gpu/msm/kgsl_sharedmem.c +++ b/drivers/gpu/msm/kgsl_sharedmem.c @@ -540,7 +540,7 @@ static int kgsl_ebimem_map_kernel(struct kgsl_memdesc *memdesc) if (!memdesc->hostptr) { memdesc->hostptr = ioremap(memdesc->physaddr, memdesc->size); if (!memdesc->hostptr) { - KGSL_CORE_ERR("ioremap failed, addr:0x%p, size:0x%x\n", + KGSL_CORE_ERR("ioremap failed, addr:0x%pK, size:0x%x\n", memdesc->hostptr, memdesc->size); ret = -ENOMEM; goto done; diff --git a/drivers/gpu/msm/kgsl_snapshot.c b/drivers/gpu/msm/kgsl_snapshot.c index 6dba0bba4d9..bbcfd1ec853 100644 --- a/drivers/gpu/msm/kgsl_snapshot.c +++ b/drivers/gpu/msm/kgsl_snapshot.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2014, The Linux Foundation. All rights reserved. +/* Copyright (c) 2012-2014,2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -880,3 +880,4 @@ void kgsl_device_snapshot_close(struct kgsl_device *device) device->snapshot_faultcount = 0; } EXPORT_SYMBOL(kgsl_device_snapshot_close); + From ce53882d35f96cdc5fd3dfd83846718119c09179 Mon Sep 17 00:00:00 2001 From: Chris Lew Date: Mon, 22 Aug 2016 18:42:11 -0700 Subject: [PATCH 130/168] msm: smp2p2: Fix kernel address leak Change format string to %pK instead of %p in the debug statements. This change fixes kernel address leaks from the usage of %p. CRs-Fixed: 1052825 Change-Id: Ic87320b47a97f265611e9fef602362cf118dff54 Signed-off-by: Chris Lew Signed-off-by: Dhoat Harpal --- arch/arm/mach-msm/smp2p.c | 6 +++--- arch/arm/mach-msm/smp2p_debug.c | 4 ++-- arch/arm/mach-msm/smp2p_gpio.c | 4 ++-- arch/arm/mach-msm/smp2p_test_common.h | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/arch/arm/mach-msm/smp2p.c b/arch/arm/mach-msm/smp2p.c index 22eb7991fb1..c32596cea0f 100644 --- a/arch/arm/mach-msm/smp2p.c +++ b/arch/arm/mach-msm/smp2p.c @@ -1,6 +1,6 @@ /* arch/arm/mach-msm/smp2p.c * - * Copyright (c) 2013-2014, The Linux Foundation. All rights reserved. + * Copyright (c) 2013,2016 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -504,8 +504,8 @@ static void smp2p_find_entry_v1(struct smp2p_smem __iomem *item, struct smp2p_entry_v1 *pos; if (!item || !name || !entry_ptr) { - SMP2P_ERR("%s: invalid arguments %p, %p, %p\n", - __func__, item, name, entry_ptr); + SMP2P_ERR("%s: invalid arguments %d, %d, %d\n", + __func__, !item, !name, !entry_ptr); return; } diff --git a/arch/arm/mach-msm/smp2p_debug.c b/arch/arm/mach-msm/smp2p_debug.c index f4ff42d970c..7bc8ff0cf6d 100644 --- a/arch/arm/mach-msm/smp2p_debug.c +++ b/arch/arm/mach-msm/smp2p_debug.c @@ -1,6 +1,6 @@ /* arch/arm/mach-msm/smp2p_debug.c * - * Copyright (c) 2013, The Linux Foundation. All rights reserved. + * Copyright (c) 2013,2016 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -41,7 +41,7 @@ static void smp2p_int_stats(struct seq_file *s) continue; seq_printf(s, - "| %5s (%d) | %11u | %10u | %10u | %p | %08x |\n", + "| %5s (%d) | %11u | %10u | %10u | %pK | %08x |\n", int_cfg[pid].name, pid, int_cfg[pid].in_int_id, int_cfg[pid].in_interrupt_count, diff --git a/arch/arm/mach-msm/smp2p_gpio.c b/arch/arm/mach-msm/smp2p_gpio.c index 5f70e0c38ee..e3949d200f0 100644 --- a/arch/arm/mach-msm/smp2p_gpio.c +++ b/arch/arm/mach-msm/smp2p_gpio.c @@ -1,6 +1,6 @@ /* arch/arm/mach-msm/smp2p_gpio.c * - * Copyright (c) 2013, The Linux Foundation. All rights reserved. + * Copyright (c) 2013,2016 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -337,7 +337,7 @@ static int smp2p_irq_map(struct irq_domain *domain_ptr, unsigned int virq, chip = domain_ptr->host_data; if (!chip) { - SMP2P_ERR("%s: invalid domain ptr %p\n", __func__, domain_ptr); + SMP2P_ERR("%s: invalid domain ptr %pK\n", __func__, domain_ptr); return -ENODEV; } diff --git a/arch/arm/mach-msm/smp2p_test_common.h b/arch/arm/mach-msm/smp2p_test_common.h index b9cddc45929..b7b9e24f7b2 100644 --- a/arch/arm/mach-msm/smp2p_test_common.h +++ b/arch/arm/mach-msm/smp2p_test_common.h @@ -1,6 +1,6 @@ /* arch/arm/mach-msm/smp2p_test_common.h * - * Copyright (c) 2013, The Linux Foundation. All rights reserved. + * Copyright (c) 2013,2016 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -51,7 +51,7 @@ void *b_tmp = (b); \ if (!((a_tmp)cmp(b_tmp))) { \ seq_printf(s, \ - "%s:%d Fail: " #a "(%p) " #cmp " " #b "(%p)\n", \ + "%s:%d Fail: " #a "(%pK) " #cmp " " #b "(%pK)\n", \ __func__, __LINE__, \ a_tmp, b_tmp); \ failed = 1; \ From 961c7634ae584a39b5f0661569d6ab06ec498b9b Mon Sep 17 00:00:00 2001 From: Zhen Kong Date: Tue, 16 Aug 2016 12:46:12 -0700 Subject: [PATCH 131/168] msm: crypto: Fix integer over flow check in qcrypto driver Integer overflow check is invalid when ULONG_MAX is used, as ULONG_MAX has typeof 'unsigned long', while req->assoclen, req->crytlen, and qreq.ivsize are 'unsigned int'. Make change to use UINT_MAX instead of ULONG_MAX. CRs-fixed: 1050970 Change-Id: I3782ea7ed2eaacdcad15b34e047a4699bf4f9e4f Signed-off-by: Zhen Kong --- drivers/crypto/msm/qcrypto.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/msm/qcrypto.c b/drivers/crypto/msm/qcrypto.c index 2a1322ad325..30e520daa9d 100644 --- a/drivers/crypto/msm/qcrypto.c +++ b/drivers/crypto/msm/qcrypto.c @@ -1771,12 +1771,12 @@ static int _qcrypto_process_aead(struct crypto_engine *pengine, * include assoicated data, ciphering data stream, * generated MAC, and CCM padding. */ - if ((MAX_ALIGN_SIZE * 2 > ULONG_MAX - req->assoclen) || + if ((MAX_ALIGN_SIZE * 2 > UINT_MAX - req->assoclen) || ((MAX_ALIGN_SIZE * 2 + req->assoclen) > - ULONG_MAX - qreq.ivsize) || + UINT_MAX - qreq.ivsize) || ((MAX_ALIGN_SIZE * 2 + req->assoclen + qreq.ivsize) - > ULONG_MAX - req->cryptlen)) { + > UINT_MAX - req->cryptlen)) { pr_err("Integer overflow on aead req length.\n"); return -EINVAL; } From 305ac7bc9904e09ec9618eedd4b2ecc1fc46041d Mon Sep 17 00:00:00 2001 From: Arnaldo Carvalho de Melo Date: Mon, 14 Mar 2016 09:56:35 -0300 Subject: [PATCH 132/168] net: Fix use after free in the recvmmsg exit path The syzkaller fuzzer hit the following use-after-free: Call Trace: [] __asan_report_load8_noabort+0x3e/0x40 mm/kasan/report.c:295 [] __sys_recvmmsg+0x6fa/0x7f0 net/socket.c:2261 [< inline >] SYSC_recvmmsg net/socket.c:2281 [] SyS_recvmmsg+0x16f/0x180 net/socket.c:2270 [] entry_SYSCALL_64_fastpath+0x16/0x7a arch/x86/entry/entry_64.S:185 And, as Dmitry rightly assessed, that is because we can drop the reference and then touch it when the underlying recvmsg calls return some packets and then hit an error, which will make recvmmsg to set sock->sk->sk_err, oops, fix it. Reported-and-Tested-by: Dmitry Vyukov Cc: Alexander Potapenko Cc: Eric Dumazet Cc: Kostya Serebryany Cc: Sasha Levin Fixes: a2e2725541fa ("net: Introduce recvmmsg socket syscall") http://lkml.kernel.org/r/20160122211644.GC2470@redhat.com Signed-off-by: Arnaldo Carvalho de Melo Signed-off-by: David S. Miller Change-Id: I447302392f46841f31c374bdb560fe5ee9c2d687 Git-repo: http://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git Git-commit: 34b88a68f26a75e4fded796f1a49c40f82234b7d Signed-off-by: Dennis Cagle --- net/socket.c | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/net/socket.c b/net/socket.c index 3548af38532..33b6d2e2672 100644 --- a/net/socket.c +++ b/net/socket.c @@ -2307,31 +2307,31 @@ int __sys_recvmmsg(int fd, struct mmsghdr __user *mmsg, unsigned int vlen, break; } + if (err == 0) + goto out_put; + + if (datagrams == 0) { + datagrams = err; + goto out_put; + } + + /* + * We may return less entries than requested (vlen) if the + * sock is non block and there aren't enough datagrams... + */ + if (err != -EAGAIN) { + /* + * ... or if recvmsg returns an error after we + * received some datagrams, where we record the + * error to return on the next call or if the + * app asks about it using getsockopt(SO_ERROR). + */ + sock->sk->sk_err = -err; + } out_put: fput_light(sock->file, fput_needed); - if (err == 0) - return datagrams; - - if (datagrams != 0) { - /* - * We may return less entries than requested (vlen) if the - * sock is non block and there aren't enough datagrams... - */ - if (err != -EAGAIN) { - /* - * ... or if recvmsg returns an error after we - * received some datagrams, where we record the - * error to return on the next call or if the - * app asks about it using getsockopt(SO_ERROR). - */ - sock->sk->sk_err = -err; - } - - return datagrams; - } - - return err; + return datagrams; } SYSCALL_DEFINE5(recvmmsg, int, fd, struct mmsghdr __user *, mmsg, From 2f8543b77cbda33269103329e004e262678c16c6 Mon Sep 17 00:00:00 2001 From: Sanjay Singh Date: Tue, 9 Aug 2016 17:39:35 +0530 Subject: [PATCH 133/168] msm: vidc: use %pK instead of %p which respects kptr_restrict sysctl Hide kernel pointers from unprivileged ussers by using %pK format- specifier instead of %p. This respects the kptr_restrict sysctl setting which is by default on. So by default %pK will print zeroes as address. echo 1 to kptr_restrict to print proper kernel addresses. CRs-Fixed: 987018 Change-Id: I4772257a557c6730ecc0624cbc8e5614e893e9fd Signed-off-by: Sanjay Singh --- .../platform/msm/vidc/hfi_packetization.c | 6 +- .../platform/msm/vidc/hfi_response_handler.c | 6 +- drivers/media/platform/msm/vidc/msm_smem.c | 20 ++-- .../media/platform/msm/vidc/msm_v4l2_vidc.c | 6 +- drivers/media/platform/msm/vidc/msm_vdec.c | 34 +++--- drivers/media/platform/msm/vidc/msm_venc.c | 30 ++--- drivers/media/platform/msm/vidc/msm_vidc.c | 34 +++--- .../media/platform/msm/vidc/msm_vidc_common.c | 112 +++++++++--------- .../media/platform/msm/vidc/msm_vidc_debug.c | 16 +-- drivers/media/platform/msm/vidc/q6_hfi.c | 20 ++-- drivers/media/platform/msm/vidc/venus_hfi.c | 68 +++++------ drivers/media/platform/msm/vidc/vidc_hfi.c | 4 +- 12 files changed, 178 insertions(+), 178 deletions(-) diff --git a/drivers/media/platform/msm/vidc/hfi_packetization.c b/drivers/media/platform/msm/vidc/hfi_packetization.c index a7ccd47e2ae..cc226fdfd01 100644 --- a/drivers/media/platform/msm/vidc/hfi_packetization.c +++ b/drivers/media/platform/msm/vidc/hfi_packetization.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2014, The Linux Foundation. All rights reserved. +/* Copyright (c) 2012-2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -1564,7 +1564,7 @@ int create_pkt_ssr_cmd(enum hal_ssr_trigger_type type, struct hfi_cmd_sys_test_ssr_packet *pkt) { if (!pkt) { - dprintk(VIDC_ERR, "Invalid params, device: %p\n", pkt); + dprintk(VIDC_ERR, "Invalid params, device: %pK\n", pkt); return -EINVAL; } pkt->size = sizeof(struct hfi_cmd_sys_test_ssr_packet); @@ -1577,7 +1577,7 @@ int create_pkt_cmd_sys_image_version( struct hfi_cmd_sys_get_property_packet *pkt) { if (!pkt) { - dprintk(VIDC_ERR, "%s invalid param :%p\n", __func__, pkt); + dprintk(VIDC_ERR, "%s invalid param :%pK\n", __func__, pkt); return -EINVAL; } pkt->size = sizeof(struct hfi_cmd_sys_get_property_packet); diff --git a/drivers/media/platform/msm/vidc/hfi_response_handler.c b/drivers/media/platform/msm/vidc/hfi_response_handler.c index bf69acf3614..9c165451298 100644 --- a/drivers/media/platform/msm/vidc/hfi_response_handler.c +++ b/drivers/media/platform/msm/vidc/hfi_response_handler.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2014, The Linux Foundation. All rights reserved. +/* Copyright (c) 2012-2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -659,7 +659,7 @@ static void hfi_process_sess_get_prop_buf_req( dprintk(VIDC_DBG, "Entered "); if (!prop) { dprintk(VIDC_ERR, - "hal_process_sess_get_prop_buf_req:bad_prop: %p", + "hal_process_sess_get_prop_buf_req:bad_prop: %pK", prop); return; } @@ -837,7 +837,7 @@ static void hfi_process_session_init_done( sess_close = (struct hal_session *)pkt->session_id; if (sess_close) { dprintk(VIDC_WARN, - "Sess init failed: 0x%x, 0x%p", + "Sess init failed: 0x%x, 0x%pK", sess_close->session_id, sess_close); } } diff --git a/drivers/media/platform/msm/vidc/msm_smem.c b/drivers/media/platform/msm/vidc/msm_smem.c index 9604a091bc6..f8214c464b9 100644 --- a/drivers/media/platform/msm/vidc/msm_smem.c +++ b/drivers/media/platform/msm/vidc/msm_smem.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2013, The Linux Foundation. All rights reserved. +/* Copyright (c) 2012-2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -54,7 +54,7 @@ static int get_device_address(struct smem_client *smem_client, struct ion_client *clnt = NULL; if (!iova || !buffer_size || !hndl || !smem_client) { - dprintk(VIDC_ERR, "Invalid params: %p, %p, %p, %p\n", + dprintk(VIDC_ERR, "Invalid params: %pK, %pK, %pK, %pK\n", smem_client, hndl, iova, buffer_size); return -EINVAL; } @@ -109,7 +109,7 @@ static void put_device_address(struct smem_client *smem_client, struct ion_client *clnt = NULL; if (!hndl || !smem_client) { - dprintk(VIDC_WARN, "Invalid params: %p, %p\n", + dprintk(VIDC_WARN, "Invalid params: %pK, %pK\n", smem_client, hndl); return; } @@ -143,7 +143,7 @@ static int ion_user_to_kernel(struct smem_client *client, int fd, u32 offset, hndl = ion_import_dma_buf(client->clnt, fd); if (IS_ERR_OR_NULL(hndl)) { - dprintk(VIDC_ERR, "Failed to get handle: %p, %d, %d, %p\n", + dprintk(VIDC_ERR, "Failed to get handle: %pK, %d, %d, %pK\n", client, fd, offset, hndl); rc = -ENOMEM; goto fail_import_fd; @@ -172,7 +172,7 @@ static int ion_user_to_kernel(struct smem_client *client, int fd, u32 offset, mem->device_addr = iova; mem->size = buffer_size; dprintk(VIDC_DBG, - "%s: ion_handle = 0x%p, fd = %d, device_addr = 0x%x, size = %d, kvaddr = 0x%p, buffer_type = %d\n", + "%s: ion_handle = 0x%pK, fd = %d, device_addr = 0x%x, size = %d, kvaddr = 0x%pK, buffer_type = %d\n", __func__, mem->smem_priv, fd, (u32)mem->device_addr, mem->size, mem->kvaddr, mem->buffer_type); return rc; @@ -215,7 +215,7 @@ static int alloc_ion_mem(struct smem_client *client, size_t size, u32 align, hndl = ion_alloc(client->clnt, size, align, heap_mask, flags); if (IS_ERR_OR_NULL(hndl)) { dprintk(VIDC_ERR, - "Failed to allocate shared memory = %p, %d, %d, 0x%x\n", + "Failed to allocate shared memory = %pK, %d, %d, 0x%x\n", client, size, align, flags); rc = -ENOMEM; goto fail_shared_mem_alloc; @@ -245,7 +245,7 @@ static int alloc_ion_mem(struct smem_client *client, size_t size, u32 align, mem->device_addr = iova; mem->size = size; dprintk(VIDC_DBG, - "%s: ion_handle = 0x%p, device_addr = 0x%x, size = %d, kvaddr = 0x%p, buffer_type = %d\n", + "%s: ion_handle = 0x%pK, device_addr = 0x%x, size = %d, kvaddr = 0x%pK, buffer_type = %d\n", __func__, mem->smem_priv, (u32)mem->device_addr, mem->size, mem->kvaddr, mem->buffer_type); return rc; @@ -262,7 +262,7 @@ static void free_ion_mem(struct smem_client *client, struct msm_smem *mem) int domain, partition, rc; dprintk(VIDC_DBG, - "%s: ion_handle = 0x%p, device_addr = 0x%x, size = %d, kvaddr = 0x%p, buffer_type = %d\n", + "%s: ion_handle = 0x%pK, device_addr = 0x%x, size = %d, kvaddr = 0x%pK, buffer_type = %d\n", __func__, mem->smem_priv, (u32)mem->device_addr, mem->size, mem->kvaddr, mem->buffer_type); rc = msm_smem_get_domain_partition((void *)client, mem->flags, @@ -334,7 +334,7 @@ static int ion_cache_operations(struct smem_client *client, int rc = 0; int msm_cache_ops = 0; if (!mem || !client) { - dprintk(VIDC_ERR, "Invalid params: %p, %p\n", + dprintk(VIDC_ERR, "Invalid params: %pK, %pK\n", mem, client); return -EINVAL; } @@ -381,7 +381,7 @@ int msm_smem_cache_operations(void *clt, struct msm_smem *mem, struct smem_client *client = clt; int rc = 0; if (!client) { - dprintk(VIDC_ERR, "Invalid params: %p\n", + dprintk(VIDC_ERR, "Invalid params: %pK\n", client); return -EINVAL; } diff --git a/drivers/media/platform/msm/vidc/msm_v4l2_vidc.c b/drivers/media/platform/msm/vidc/msm_v4l2_vidc.c index 58015186abb..e4d39a2b067 100644 --- a/drivers/media/platform/msm/vidc/msm_v4l2_vidc.c +++ b/drivers/media/platform/msm/vidc/msm_v4l2_vidc.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2014, The Linux Foundation. All rights reserved. +/* Copyright (c) 2012-2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -285,7 +285,7 @@ static int read_platform_resources(struct msm_vidc_core *core, struct platform_device *pdev) { if (!core || !pdev) { - dprintk(VIDC_ERR, "%s: Invalid params %p %p\n", + dprintk(VIDC_ERR, "%s: Invalid params %pK %pK\n", __func__, core, pdev); return -EINVAL; } @@ -516,7 +516,7 @@ static int __devexit msm_vidc_remove(struct platform_device *pdev) struct msm_vidc_core *core; if (!pdev) { - dprintk(VIDC_ERR, "%s invalid input %p", __func__, pdev); + dprintk(VIDC_ERR, "%s invalid input %pK", __func__, pdev); return -EINVAL; } core = pdev->dev.platform_data; diff --git a/drivers/media/platform/msm/vidc/msm_vdec.c b/drivers/media/platform/msm/vidc/msm_vdec.c index da004ba29eb..1014b9b7d2e 100644 --- a/drivers/media/platform/msm/vidc/msm_vdec.c +++ b/drivers/media/platform/msm/vidc/msm_vdec.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2015, The Linux Foundation. All rights reserved. +/* Copyright (c) 2012-2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -550,7 +550,7 @@ int msm_vdec_prepare_buf(struct msm_vidc_inst *inst, if (inst->state == MSM_VIDC_CORE_INVALID || inst->core->state == VIDC_CORE_INVALID) { dprintk(VIDC_ERR, - "Core %p in bad state, ignoring prepare buf\n", + "Core %pK in bad state, ignoring prepare buf\n", inst->core); goto exit; } @@ -628,7 +628,7 @@ int msm_vdec_release_buf(struct msm_vidc_inst *inst, if (inst->state == MSM_VIDC_CORE_INVALID || core->state == VIDC_CORE_INVALID) { dprintk(VIDC_ERR, - "Core %p in bad state, ignoring release output buf\n", + "Core %pK in bad state, ignoring release output buf\n", core); goto exit; } @@ -721,7 +721,7 @@ int msm_vdec_reqbufs(struct msm_vidc_inst *inst, struct v4l2_requestbuffers *b) int rc = 0; if (!inst || !b) { dprintk(VIDC_ERR, - "Invalid input, inst = %p, buffer = %p\n", inst, b); + "Invalid input, inst = %pK, buffer = %pK\n", inst, b); return -EINVAL; } q = msm_comm_get_vb2q(inst, b->type); @@ -752,7 +752,7 @@ int msm_vdec_g_fmt(struct msm_vidc_inst *inst, struct v4l2_format *f) if (!inst || !f || !inst->core || !inst->core->device) { dprintk(VIDC_ERR, - "Invalid input, inst = %p, format = %p\n", inst, f); + "Invalid input, inst = %pK, format = %pK\n", inst, f); return -EINVAL; } @@ -938,7 +938,7 @@ int msm_vdec_s_parm(struct msm_vidc_inst *inst, struct v4l2_streamparm *a) fps = fps - 1; if (inst->prop.fps != fps) { - dprintk(VIDC_PROF, "reported fps changed for %p: %d->%d\n", + dprintk(VIDC_PROF, "reported fps changed for %pK: %d->%d\n", inst, inst->prop.fps, fps); inst->prop.fps = fps; @@ -1102,7 +1102,7 @@ int msm_vdec_querycap(struct msm_vidc_inst *inst, struct v4l2_capability *cap) { if (!inst || !cap) { dprintk(VIDC_ERR, - "Invalid input, inst = %p, cap = %p\n", inst, cap); + "Invalid input, inst = %pK, cap = %pK\n", inst, cap); return -EINVAL; } strlcpy(cap->driver, MSM_VIDC_DRV_NAME, sizeof(cap->driver)); @@ -1122,7 +1122,7 @@ int msm_vdec_enum_fmt(struct msm_vidc_inst *inst, struct v4l2_fmtdesc *f) int rc = 0; if (!inst || !f) { dprintk(VIDC_ERR, - "Invalid input, inst = %p, f = %p\n", inst, f); + "Invalid input, inst = %pK, f = %pK\n", inst, f); return -EINVAL; } if (f->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) { @@ -1161,7 +1161,7 @@ static int msm_vdec_queue_setup(struct vb2_queue *q, enum hal_property property_id; if (!q || !num_buffers || !num_planes || !sizes || !q->drv_priv) { - dprintk(VIDC_ERR, "Invalid input, q = %p, %p, %p\n", + dprintk(VIDC_ERR, "Invalid input, q = %pK, %pK, %pK\n", q, num_buffers, num_planes); return -EINVAL; } @@ -1296,7 +1296,7 @@ static inline int start_streaming(struct msm_vidc_inst *inst) rc = msm_comm_try_state(inst, MSM_VIDC_START_DONE); if (rc) { dprintk(VIDC_ERR, - "Failed to move inst: %p to start done state\n", inst); + "Failed to move inst: %pK to start done state\n", inst); goto fail_start; } if (msm_comm_get_stream_output_mode(inst) == @@ -1333,7 +1333,7 @@ static inline int stop_streaming(struct msm_vidc_inst *inst) rc = msm_comm_try_state(inst, MSM_VIDC_RELEASE_RESOURCES_DONE); if (rc) dprintk(VIDC_ERR, - "Failed to move inst: %p to start done state\n", inst); + "Failed to move inst: %pK to start done state\n", inst); return rc; } @@ -1343,7 +1343,7 @@ static int msm_vdec_start_streaming(struct vb2_queue *q, unsigned int count) int rc = 0; struct hfi_device *hdev; if (!q || !q->drv_priv) { - dprintk(VIDC_ERR, "Invalid input, q = %p\n", q); + dprintk(VIDC_ERR, "Invalid input, q = %pK\n", q); return -EINVAL; } inst = q->drv_priv; @@ -1376,7 +1376,7 @@ static int msm_vdec_stop_streaming(struct vb2_queue *q) struct msm_vidc_inst *inst; int rc = 0; if (!q || !q->drv_priv) { - dprintk(VIDC_ERR, "Invalid input, q = %p\n", q); + dprintk(VIDC_ERR, "Invalid input, q = %pK\n", q); return -EINVAL; } inst = q->drv_priv; @@ -1401,7 +1401,7 @@ static int msm_vdec_stop_streaming(struct vb2_queue *q) if (rc) dprintk(VIDC_ERR, - "Failed to move inst: %p, cap = %d to state: %d\n", + "Failed to move inst: %pK, cap = %d to state: %d\n", inst, q->type, MSM_VIDC_RELEASE_RESOURCES_DONE); return rc; } @@ -1458,7 +1458,7 @@ int msm_vdec_cmd(struct msm_vidc_inst *inst, struct v4l2_decoder_cmd *dec) if (inst->state == MSM_VIDC_CORE_INVALID || core->state == VIDC_CORE_INVALID) { dprintk(VIDC_ERR, - "Core %p in bad state, Sending CLOSE event\n", + "Core %pK in bad state, Sending CLOSE event\n", core); msm_vidc_queue_v4l2_event(inst, V4L2_EVENT_MSM_VIDC_CLOSE_DONE); @@ -1500,7 +1500,7 @@ int msm_vdec_inst_init(struct msm_vidc_inst *inst) { int rc = 0; if (!inst) { - dprintk(VIDC_ERR, "Invalid input = %p\n", inst); + dprintk(VIDC_ERR, "Invalid input = %pK\n", inst); return -EINVAL; } inst->fmts[OUTPUT_PORT] = &vdec_formats[1]; @@ -1805,7 +1805,7 @@ static int msm_vdec_op_s_ctrl(struct v4l2_ctrl *ctrl) rc = msm_comm_try_state(inst, MSM_VIDC_OPEN_DONE); if (rc) { dprintk(VIDC_ERR, - "Failed to move inst: %p to start done state\n", inst); + "Failed to move inst: %pK to start done state\n", inst); goto failed_open_done; } diff --git a/drivers/media/platform/msm/vidc/msm_venc.c b/drivers/media/platform/msm/vidc/msm_venc.c index d534f9678f2..bf44b12b9c3 100644 --- a/drivers/media/platform/msm/vidc/msm_venc.c +++ b/drivers/media/platform/msm/vidc/msm_venc.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2015, The Linux Foundation. All rights reserved. +/* Copyright (c) 2012-2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -1098,7 +1098,7 @@ static inline int start_streaming(struct msm_vidc_inst *inst) rc = msm_comm_try_state(inst, MSM_VIDC_START_DONE); if (rc) { dprintk(VIDC_ERR, - "Failed to move inst: %p to start done state\n", inst); + "Failed to move inst: %pK to start done state\n", inst); goto fail_start; } @@ -1125,7 +1125,7 @@ static int msm_venc_start_streaming(struct vb2_queue *q, unsigned int count) struct msm_vidc_inst *inst; int rc = 0; if (!q || !q->drv_priv) { - dprintk(VIDC_ERR, "Invalid input, q = %p\n", q); + dprintk(VIDC_ERR, "Invalid input, q = %pK\n", q); return -EINVAL; } inst = q->drv_priv; @@ -1152,7 +1152,7 @@ static int msm_venc_stop_streaming(struct vb2_queue *q) struct msm_vidc_inst *inst; int rc = 0; if (!q || !q->drv_priv) { - dprintk(VIDC_ERR, "Invalid input, q = %p\n", q); + dprintk(VIDC_ERR, "Invalid input, q = %pK\n", q); return -EINVAL; } inst = q->drv_priv; @@ -1173,7 +1173,7 @@ static int msm_venc_stop_streaming(struct vb2_queue *q) if (rc) dprintk(VIDC_ERR, - "Failed to move inst: %p, cap = %d to state: %d\n", + "Failed to move inst: %pK, cap = %d to state: %d\n", inst, q->type, MSM_VIDC_CLOSE_DONE); return rc; } @@ -2355,7 +2355,7 @@ static int msm_venc_op_s_ctrl(struct v4l2_ctrl *ctrl) if (rc) { dprintk(VIDC_ERR, - "Failed to move inst: %p to start done state\n", inst); + "Failed to move inst: %pK to start done state\n", inst); goto failed_open_done; } @@ -2398,7 +2398,7 @@ int msm_venc_inst_init(struct msm_vidc_inst *inst) { int rc = 0; if (!inst) { - dprintk(VIDC_ERR, "Invalid input = %p\n", inst); + dprintk(VIDC_ERR, "Invalid input = %pK\n", inst); return -EINVAL; } inst->fmts[CAPTURE_PORT] = &venc_formats[1]; @@ -2480,7 +2480,7 @@ int msm_venc_querycap(struct msm_vidc_inst *inst, struct v4l2_capability *cap) { if (!inst || !cap) { dprintk(VIDC_ERR, - "Invalid input, inst = %p, cap = %p\n", inst, cap); + "Invalid input, inst = %pK, cap = %pK\n", inst, cap); return -EINVAL; } strlcpy(cap->driver, MSM_VIDC_DRV_NAME, sizeof(cap->driver)); @@ -2500,7 +2500,7 @@ int msm_venc_enum_fmt(struct msm_vidc_inst *inst, struct v4l2_fmtdesc *f) int rc = 0; if (!inst || !f) { dprintk(VIDC_ERR, - "Invalid input, inst = %p, f = %p\n", inst, f); + "Invalid input, inst = %pK, f = %pK\n", inst, f); return -EINVAL; } if (f->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) { @@ -2573,7 +2573,7 @@ int msm_venc_s_parm(struct msm_vidc_inst *inst, struct v4l2_streamparm *a) fps = fps - 1; if (inst->prop.fps != fps) { - dprintk(VIDC_PROF, "reported fps changed for %p: %d->%d\n", + dprintk(VIDC_PROF, "reported fps changed for %pK: %d->%d\n", inst, inst->prop.fps, fps); inst->prop.fps = fps; frame_rate.frame_rate = inst->prop.fps * (0x1<<16); @@ -2600,7 +2600,7 @@ int msm_venc_s_fmt(struct msm_vidc_inst *inst, struct v4l2_format *f) struct hfi_device *hdev; if (!inst || !f) { dprintk(VIDC_ERR, - "Invalid input, inst = %p, format = %p\n", inst, f); + "Invalid input, inst = %pK, format = %pK\n", inst, f); return -EINVAL; } @@ -2735,7 +2735,7 @@ int msm_venc_g_fmt(struct msm_vidc_inst *inst, struct v4l2_format *f) struct hal_buffer_requirements *buff_req_buffer; if (!inst || !f) { dprintk(VIDC_ERR, - "Invalid input, inst = %p, format = %p\n", inst, f); + "Invalid input, inst = %pK, format = %pK\n", inst, f); return -EINVAL; } @@ -2800,7 +2800,7 @@ int msm_venc_reqbufs(struct msm_vidc_inst *inst, struct v4l2_requestbuffers *b) int rc = 0; if (!inst || !b) { dprintk(VIDC_ERR, - "Invalid input, inst = %p, buffer = %p\n", inst, b); + "Invalid input, inst = %pK, buffer = %pK\n", inst, b); return -EINVAL; } q = msm_comm_get_vb2q(inst, b->type); @@ -2837,7 +2837,7 @@ int msm_venc_prepare_buf(struct msm_vidc_inst *inst, if (inst->state == MSM_VIDC_CORE_INVALID || inst->core->state == VIDC_CORE_INVALID) { dprintk(VIDC_ERR, - "Core %p in bad state, ignoring prepare buf\n", + "Core %pK in bad state, ignoring prepare buf\n", inst->core); goto exit; } @@ -2908,7 +2908,7 @@ int msm_venc_release_buf(struct msm_vidc_inst *inst, rc = msm_comm_try_state(inst, MSM_VIDC_RELEASE_RESOURCES_DONE); if (rc) { dprintk(VIDC_ERR, - "Failed to move inst: %p to release res done state\n", + "Failed to move inst: %pK to release res done state\n", inst); goto exit; } diff --git a/drivers/media/platform/msm/vidc/msm_vidc.c b/drivers/media/platform/msm/vidc/msm_vidc.c index 9eb7c337bf1..8e690ba569d 100644 --- a/drivers/media/platform/msm/vidc/msm_vidc.c +++ b/drivers/media/platform/msm/vidc/msm_vidc.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2014, The Linux Foundation. All rights reserved. +/* Copyright (c) 2012-2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -309,7 +309,7 @@ struct buffer_info *device_to_uvaddr(struct msm_vidc_list *buf_list, if (!buf_list || !device_addr) { dprintk(VIDC_ERR, - "Invalid input- device_addr: %u buf_list: %p\n", + "Invalid input- device_addr: %u buf_list: %pK\n", device_addr, buf_list); goto err_invalid_input; } @@ -463,7 +463,7 @@ int map_and_register_buf(struct msm_vidc_inst *inst, struct v4l2_buffer *b) goto exit; } - dprintk(VIDC_DBG, "[MAP] Create binfo = %p fd = %d type = %d\n", + dprintk(VIDC_DBG, "[MAP] Create binfo = %pK fd = %d type = %d\n", binfo, b->m.planes[0].reserved[0], b->type); for (i = 0; i < b->length; ++i) { @@ -538,7 +538,7 @@ int map_and_register_buf(struct msm_vidc_inst *inst, struct v4l2_buffer *b) goto exit; } dprintk(VIDC_DBG, - "%s: [MAP] binfo = %p, handle[%d] = %p, device_addr = 0x%x, fd = %d, offset = %d, mapped = %d\n", + "%s: [MAP] binfo = %pK, handle[%d] = %pK, device_addr = 0x%x, fd = %d, offset = %d, mapped = %d\n", __func__, binfo, i, binfo->handle[i], binfo->device_addr[i], binfo->fd[i], binfo->buff_off[i], binfo->mapped[i]); @@ -559,7 +559,7 @@ int unmap_and_deregister_buf(struct msm_vidc_inst *inst, bool found = false, keep_node = false; if (!inst || !binfo) { - dprintk(VIDC_ERR, "%s invalid param: %p %p\n", + dprintk(VIDC_ERR, "%s invalid param: %pK %pK\n", __func__, inst, binfo); return -EINVAL; } @@ -589,7 +589,7 @@ int unmap_and_deregister_buf(struct msm_vidc_inst *inst, for (i = 0; i < temp->num_planes; i++) { dprintk(VIDC_DBG, - "%s: [UNMAP] binfo = %p, handle[%d] = %p, device_addr = 0x%x, fd = %d, offset = %d, mapped = %d\n", + "%s: [UNMAP] binfo = %pK, handle[%d] = %pK, device_addr = 0x%x, fd = %d, offset = %d, mapped = %d\n", __func__, temp, i, temp->handle[i], temp->device_addr[i], temp->fd[i], temp->buff_off[i], temp->mapped[i]); @@ -621,12 +621,12 @@ int unmap_and_deregister_buf(struct msm_vidc_inst *inst, } } if (!keep_node) { - dprintk(VIDC_DBG, "[UNMAP] AND-FREED binfo: %p\n", temp); + dprintk(VIDC_DBG, "[UNMAP] AND-FREED binfo: %pK\n", temp); list_del(&temp->list); kfree(temp); } else { temp->inactive = true; - dprintk(VIDC_DBG, "[UNMAP] NOT-FREED binfo: %p\n", temp); + dprintk(VIDC_DBG, "[UNMAP] NOT-FREED binfo: %pK\n", temp); } exit: return 0; @@ -640,7 +640,7 @@ int qbuf_dynamic_buf(struct msm_vidc_inst *inst, struct v4l2_plane plane[VIDEO_MAX_PLANES] = { {0} }; if (!binfo) { - dprintk(VIDC_ERR, "%s invalid param: %p\n", __func__, binfo); + dprintk(VIDC_ERR, "%s invalid param: %pK\n", __func__, binfo); return -EINVAL; } dprintk(VIDC_DBG, "%s fd[0] = %d\n", __func__, binfo->fd[0]); @@ -663,7 +663,7 @@ int output_buffer_cache_invalidate(struct msm_vidc_inst *inst, int rc = 0; if (!inst) { - dprintk(VIDC_ERR, "%s: invalid inst: %p\n", __func__, inst); + dprintk(VIDC_ERR, "%s: invalid inst: %pK\n", __func__, inst); return -EINVAL; } @@ -671,7 +671,7 @@ int output_buffer_cache_invalidate(struct msm_vidc_inst *inst, return 0; if (!binfo) { - dprintk(VIDC_ERR, "%s: invalid buffer info: %p\n", + dprintk(VIDC_ERR, "%s: invalid buffer info: %pK\n", __func__, inst); return -EINVAL; } @@ -751,7 +751,7 @@ int msm_vidc_release_buffers(void *instance, int buffer_type) rc = msm_comm_try_state(inst, MSM_VIDC_RELEASE_RESOURCES_DONE); if (rc) { dprintk(VIDC_ERR, - "Failed to move inst: %p to release res done\n", + "Failed to move inst: %pK to release res done\n", inst); } } @@ -812,7 +812,7 @@ free_and_unmap: for (i = 0; i < bi->num_planes; i++) { if (bi->handle[i] && bi->mapped[i]) { dprintk(VIDC_DBG, - "%s: [UNMAP] binfo = %p, handle[%d] = %p, device_addr = 0x%x, fd = %d, offset = %d, mapped = %d\n", + "%s: [UNMAP] binfo = %pK, handle[%d] = %pK, device_addr = 0x%x, fd = %d, offset = %d, mapped = %d\n", __func__, bi, i, bi->handle[i], bi->device_addr[i], bi->fd[i], bi->buff_off[i], bi->mapped[i]); @@ -1029,7 +1029,7 @@ int msm_vidc_enum_framesizes(void *instance, struct v4l2_frmsizeenum *fsize) struct msm_vidc_core_capability *capability = NULL; if (!inst || !fsize) { - dprintk(VIDC_ERR, "%s: invalid parameter: %p %p\n", + dprintk(VIDC_ERR, "%s: invalid parameter: %pK %pK\n", __func__, inst, fsize); return -EINVAL; } @@ -1090,7 +1090,7 @@ void *msm_vidc_smem_get_client(void *instance) struct msm_vidc_inst *inst = instance; if (!inst || !inst->mem_client) { - dprintk(VIDC_ERR, "%s: invalid instance or client = %p %p\n", + dprintk(VIDC_ERR, "%s: invalid instance or client = %pK %pK\n", __func__, inst, inst->mem_client); return NULL; } @@ -1211,7 +1211,7 @@ void *msm_vidc_open(int core_id, int session_type) goto err_invalid_core; } - pr_info(VIDC_DBG_TAG "Opening video instance: %p, %d\n", + pr_info(VIDC_DBG_TAG "Opening video instance: %pK, %d\n", VIDC_INFO, inst, session_type); mutex_init(&inst->sync_lock); mutex_init(&inst->bufq[CAPTURE_PORT].lock); @@ -1389,7 +1389,7 @@ int msm_vidc_close(void *instance) msm_comm_session_clean(inst); msm_smem_delete_client(inst->mem_client); - pr_info(VIDC_DBG_TAG "Closed video instance: %p\n", VIDC_INFO, inst); + pr_info(VIDC_DBG_TAG "Closed video instance: %pK\n", VIDC_INFO, inst); kfree(inst); return 0; diff --git a/drivers/media/platform/msm/vidc/msm_vidc_common.c b/drivers/media/platform/msm/vidc/msm_vidc_common.c index 29dd2637f2c..2fcb6fbd343 100644 --- a/drivers/media/platform/msm/vidc/msm_vidc_common.c +++ b/drivers/media/platform/msm/vidc/msm_vidc_common.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2015, The Linux Foundation. All rights reserved. +/* Copyright (c) 2012-2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -142,7 +142,7 @@ static int msm_comm_get_inst_load(struct msm_vidc_inst *inst, if (is_non_realtime_session(inst) && (quirks & LOAD_CALC_IGNORE_NON_REALTIME_LOAD)) { if (!inst->prop.fps) { - dprintk(VIDC_INFO, "%s: instance:%p prop->fps is set 0\n", __func__, inst); + dprintk(VIDC_INFO, "%s: instance:%pK prop->fps is set 0\n", __func__, inst); load = 0; } else load = msm_comm_get_mbs_per_sec(inst) / inst->prop.fps; @@ -157,7 +157,7 @@ static int msm_comm_get_load(struct msm_vidc_core *core, struct msm_vidc_inst *inst = NULL; int num_mbs_per_sec = 0; if (!core) { - dprintk(VIDC_ERR, "Invalid args: %p\n", core); + dprintk(VIDC_ERR, "Invalid args: %pK\n", core); return -EINVAL; } mutex_lock(&core->lock); @@ -179,13 +179,13 @@ static int msm_comm_scale_bus(struct msm_vidc_core *core, struct hfi_device *hdev; if (!core || type >= MSM_VIDC_MAX_DEVICES) { - dprintk(VIDC_ERR, "Invalid args: %p, %d\n", core, type); + dprintk(VIDC_ERR, "Invalid args: %pK, %d\n", core, type); return -EINVAL; } hdev = core->device; if (!hdev) { - dprintk(VIDC_ERR, "Invalid device handle %p\n", hdev); + dprintk(VIDC_ERR, "Invalid device handle %pK\n", hdev); return -EINVAL; } @@ -253,7 +253,7 @@ const struct msm_vidc_format *msm_comm_get_pixel_fmt_index( { int i, k = 0; if (!fmt || index < 0) { - dprintk(VIDC_ERR, "Invalid inputs, fmt = %p, index = %d\n", + dprintk(VIDC_ERR, "Invalid inputs, fmt = %pK, index = %d\n", fmt, index); return NULL; } @@ -275,7 +275,7 @@ struct msm_vidc_format *msm_comm_get_pixel_fmt_fourcc( { int i; if (!fmt) { - dprintk(VIDC_ERR, "Invalid inputs, fmt = %p\n", fmt); + dprintk(VIDC_ERR, "Invalid inputs, fmt = %pK\n", fmt); return NULL; } for (i = 0; i < size; i++) { @@ -324,7 +324,7 @@ static void handle_sys_init_done(enum command_response cmd, void *data) core->dec_codec_supported = sys_init_msg->dec_codec_supported; dprintk(VIDC_DBG, "supported_codecs: enc = 0x%x, dec = 0x%x\n", core->enc_codec_supported, core->dec_codec_supported); - dprintk(VIDC_DBG, "ptr[%d] = %p\n", index, &(core->completions[index])); + dprintk(VIDC_DBG, "ptr[%d] = %pK\n", index, &(core->completions[index])); complete(&(core->completions[index])); } @@ -402,11 +402,11 @@ static void change_inst_state(struct msm_vidc_inst *inst, mutex_lock(&inst->lock); if (inst->state == MSM_VIDC_CORE_INVALID) { dprintk(VIDC_DBG, - "Inst: %p is in bad state can't change state to %d\n", + "Inst: %pK is in bad state can't change state to %d\n", inst, state); goto exit; } - dprintk(VIDC_DBG, "Moved inst: %p from state: %d to state: %d\n", + dprintk(VIDC_DBG, "Moved inst: %pK from state: %d to state: %d\n", inst, inst->state, state); inst->state = state; exit: @@ -417,7 +417,7 @@ static int signal_session_msg_receipt(enum command_response cmd, struct msm_vidc_inst *inst) { if (!inst) { - dprintk(VIDC_ERR, "Invalid(%p) instance id\n", inst); + dprintk(VIDC_ERR, "Invalid(%pK) instance id\n", inst); return -EINVAL; } complete(&inst->completions[SESSION_MSG_INDEX(cmd)]); @@ -450,7 +450,7 @@ static int wait_for_state(struct msm_vidc_inst *inst, { int rc = 0; if (IS_ALREADY_IN_STATE(flipped_state, desired_state)) { - dprintk(VIDC_INFO, "inst: %p is already in state: %d\n", + dprintk(VIDC_INFO, "inst: %pK is already in state: %d\n", inst, inst->state); goto err_same_state; } @@ -872,24 +872,24 @@ static void handle_session_error(enum command_response cmd, void *data) if (!inst || !inst->session || !inst->core->device) { dprintk(VIDC_ERR, - "Session (%p) not in a stable enough state to handle session error\n", + "Session (%pK) not in a stable enough state to handle session error\n", inst); return; } hdev = inst->core->device; - dprintk(VIDC_WARN, "Session error received for session %p\n", inst); + dprintk(VIDC_WARN, "Session error received for session %pK\n", inst); change_inst_state(inst, MSM_VIDC_CORE_INVALID); if (response->status == VIDC_ERR_MAX_CLIENTS) { dprintk(VIDC_WARN, - "send max clients reached error to client: %p\n", + "send max clients reached error to client: %pK\n", inst); msm_vidc_queue_v4l2_event(inst, V4L2_EVENT_MSM_VIDC_MAX_CLIENTS); } else { dprintk(VIDC_ERR, - "send session error to client: %p\n", + "send session error to client: %pK\n", inst); msm_vidc_queue_v4l2_event(inst, V4L2_EVENT_MSM_VIDC_SYS_ERROR); @@ -923,7 +923,7 @@ static void handle_sys_error(enum command_response cmd, void *data) return; } - dprintk(VIDC_WARN, "SYS_ERROR %d received for core %p\n", cmd, core); + dprintk(VIDC_WARN, "SYS_ERROR %d received for core %pK\n", cmd, core); mutex_lock(&core->lock); core->state = VIDC_CORE_INVALID; @@ -981,12 +981,12 @@ void msm_comm_session_clean(struct msm_vidc_inst *inst) hdev = inst->core->device; mutex_lock(&inst->lock); if (hdev && inst->session) { - dprintk(VIDC_DBG, "cleaning up instance: 0x%p\n", inst); + dprintk(VIDC_DBG, "cleaning up instance: 0x%pK\n", inst); rc = call_hfi_op(hdev, session_clean, (void *) inst->session); if (rc) { dprintk(VIDC_ERR, - "Session clean failed :%p\n", inst); + "Session clean failed :%pK\n", inst); } inst->session = NULL; } @@ -1516,13 +1516,13 @@ static int msm_comm_scale_clocks(struct msm_vidc_core *core) struct hfi_device *hdev; if (!core) { - dprintk(VIDC_ERR, "%s Invalid args: %p\n", __func__, core); + dprintk(VIDC_ERR, "%s Invalid args: %pK\n", __func__, core); return -EINVAL; } hdev = core->device; if (!hdev) { - dprintk(VIDC_ERR, "%s Invalid device handle: %p\n", + dprintk(VIDC_ERR, "%s Invalid device handle: %pK\n", __func__, hdev); return -EINVAL; } @@ -1804,7 +1804,7 @@ static int msm_comm_session_init(int flipped_state, hdev = inst->core->device; if (IS_ALREADY_IN_STATE(flipped_state, MSM_VIDC_OPEN)) { - dprintk(VIDC_INFO, "inst: %p is already in state: %d\n", + dprintk(VIDC_INFO, "inst: %pK is already in state: %d\n", inst, inst->state); goto exit; } @@ -1898,7 +1898,7 @@ static int msm_vidc_load_resources(int flipped_state, hdev = inst->core->device; if (IS_ALREADY_IN_STATE(flipped_state, MSM_VIDC_LOAD_RESOURCES)) { - dprintk(VIDC_INFO, "inst: %p is already in state: %d\n", + dprintk(VIDC_INFO, "inst: %pK is already in state: %d\n", inst, inst->state); goto exit; } @@ -1934,7 +1934,7 @@ static int msm_vidc_start(int flipped_state, struct msm_vidc_inst *inst) if (IS_ALREADY_IN_STATE(flipped_state, MSM_VIDC_START)) { dprintk(VIDC_INFO, - "inst: %p is already in state: %d\n", + "inst: %pK is already in state: %d\n", inst, inst->state); goto exit; } @@ -1964,7 +1964,7 @@ static int msm_vidc_stop(int flipped_state, struct msm_vidc_inst *inst) if (IS_ALREADY_IN_STATE(flipped_state, MSM_VIDC_STOP)) { dprintk(VIDC_INFO, - "inst: %p is already in state: %d\n", + "inst: %pK is already in state: %d\n", inst, inst->state); goto exit; } @@ -1994,7 +1994,7 @@ static int msm_vidc_release_res(int flipped_state, struct msm_vidc_inst *inst) if (IS_ALREADY_IN_STATE(flipped_state, MSM_VIDC_RELEASE_RESOURCES)) { dprintk(VIDC_INFO, - "inst: %p is already in state: %d\n", + "inst: %pK is already in state: %d\n", inst, inst->state); goto exit; } @@ -2026,7 +2026,7 @@ static int msm_comm_session_close(int flipped_state, hdev = inst->core->device; if (IS_ALREADY_IN_STATE(flipped_state, MSM_VIDC_CLOSE)) { dprintk(VIDC_INFO, - "inst: %p is already in state: %d\n", + "inst: %pK is already in state: %d\n", inst, inst->state); goto exit; } @@ -2388,16 +2388,16 @@ int msm_comm_try_state(struct msm_vidc_inst *inst, int state) struct msm_vidc_core *core; if (!inst) { dprintk(VIDC_ERR, - "Invalid instance pointer = %p\n", inst); + "Invalid instance pointer = %pK\n", inst); return -EINVAL; } dprintk(VIDC_DBG, - "Trying to move inst: %p from: 0x%x to 0x%x\n", + "Trying to move inst: %pK from: 0x%x to 0x%x\n", inst, inst->state, state); core = inst->core; if (!core) { dprintk(VIDC_ERR, - "Invalid core pointer = %p\n", inst); + "Invalid core pointer = %pK\n", inst); return -EINVAL; } mutex_lock(&inst->sync_lock); @@ -2516,18 +2516,18 @@ int msm_comm_qbuf(struct vb2_buffer *vb) q = vb->vb2_queue; inst = q->drv_priv; if (!inst || !vb) { - dprintk(VIDC_ERR, "Invalid input: %p, %p\n", inst, vb); + dprintk(VIDC_ERR, "Invalid input: %pK, %pK\n", inst, vb); return -EINVAL; } core = inst->core; if (!core) { dprintk(VIDC_ERR, - "Invalid input: %p, %p, %p\n", inst, core, vb); + "Invalid input: %pK, %pK, %pK\n", inst, core, vb); return -EINVAL; } hdev = core->device; if (!hdev) { - dprintk(VIDC_ERR, "Invalid input: %p", hdev); + dprintk(VIDC_ERR, "Invalid input: %pK", hdev); return -EINVAL; } @@ -2631,7 +2631,7 @@ int msm_comm_qbuf(struct vb2_buffer *vb) (void *) inst->session, &seq_hdr); if (!rc) { inst->vb2_seq_hdr = vb; - dprintk(VIDC_DBG, "Seq_hdr: %p\n", + dprintk(VIDC_DBG, "Seq_hdr: %pK\n", inst->vb2_seq_hdr); } } else { @@ -2713,18 +2713,18 @@ int msm_comm_release_output_buffers(struct msm_vidc_inst *inst) struct hfi_device *hdev; if (!inst) { dprintk(VIDC_ERR, - "Invalid instance pointer = %p\n", inst); + "Invalid instance pointer = %pK\n", inst); return -EINVAL; } core = inst->core; if (!core) { dprintk(VIDC_ERR, - "Invalid core pointer = %p\n", core); + "Invalid core pointer = %pK\n", core); return -EINVAL; } hdev = core->device; if (!hdev) { - dprintk(VIDC_ERR, "Invalid device pointer = %p\n", hdev); + dprintk(VIDC_ERR, "Invalid device pointer = %pK\n", hdev); return -EINVAL; } @@ -2772,18 +2772,18 @@ int msm_comm_release_scratch_buffers(struct msm_vidc_inst *inst) struct hfi_device *hdev; if (!inst) { dprintk(VIDC_ERR, - "Invalid instance pointer = %p\n", inst); + "Invalid instance pointer = %pK\n", inst); return -EINVAL; } core = inst->core; if (!core) { dprintk(VIDC_ERR, - "Invalid core pointer = %p\n", core); + "Invalid core pointer = %pK\n", core); return -EINVAL; } hdev = core->device; if (!hdev) { - dprintk(VIDC_ERR, "Invalid device pointer = %p\n", hdev); + dprintk(VIDC_ERR, "Invalid device pointer = %pK\n", hdev); return -EINVAL; } @@ -2836,18 +2836,18 @@ int msm_comm_release_persist_buffers(struct msm_vidc_inst *inst) struct hfi_device *hdev; if (!inst) { dprintk(VIDC_ERR, - "Invalid instance pointer = %p\n", inst); + "Invalid instance pointer = %pK\n", inst); return -EINVAL; } core = inst->core; if (!core) { dprintk(VIDC_ERR, - "Invalid core pointer = %p\n", core); + "Invalid core pointer = %pK\n", core); return -EINVAL; } hdev = core->device; if (!hdev) { - dprintk(VIDC_ERR, "Invalid device pointer = %p\n", hdev); + dprintk(VIDC_ERR, "Invalid device pointer = %pK\n", hdev); return -EINVAL; } mutex_lock(&inst->persistbufs.lock); @@ -2895,7 +2895,7 @@ int msm_comm_try_set_prop(struct msm_vidc_inst *inst, int rc = 0; struct hfi_device *hdev; if (!inst) { - dprintk(VIDC_ERR, "Invalid input: %p\n", inst); + dprintk(VIDC_ERR, "Invalid input: %pK\n", inst); return -EINVAL; } @@ -3119,7 +3119,7 @@ void msm_comm_flush_pending_dynamic_buffers(struct msm_vidc_inst *inst) if (binfo && binfo->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) { dprintk(VIDC_DBG, - "%s: binfo = %p device_addr = 0x%pa\n", + "%s: binfo = %pK device_addr = 0x%pa\n", __func__, binfo, &binfo->device_addr[0]); buf_ref_put(inst, binfo); } @@ -3139,18 +3139,18 @@ int msm_comm_flush(struct msm_vidc_inst *inst, u32 flags) struct hfi_device *hdev; if (!inst) { dprintk(VIDC_ERR, - "Invalid instance pointer = %p\n", inst); + "Invalid instance pointer = %pK\n", inst); return -EINVAL; } core = inst->core; if (!core) { dprintk(VIDC_ERR, - "Invalid core pointer = %p\n", core); + "Invalid core pointer = %pK\n", core); return -EINVAL; } hdev = core->device; if (!hdev) { - dprintk(VIDC_ERR, "Invalid device pointer = %p", hdev); + dprintk(VIDC_ERR, "Invalid device pointer = %pK", hdev); return -EINVAL; } @@ -3168,7 +3168,7 @@ int msm_comm_flush(struct msm_vidc_inst *inst, u32 flags) core->state == VIDC_CORE_UNINIT || core->state == VIDC_CORE_INVALID) { dprintk(VIDC_ERR, - "Core %p and inst %p are in bad state\n", + "Core %pK and inst %pK are in bad state\n", core, inst); msm_comm_flush_in_invalid_state(inst); return 0; @@ -3337,7 +3337,7 @@ int msm_vidc_trigger_ssr(struct msm_vidc_core *core, int rc = 0; struct hfi_device *hdev; if (!core || !core->device) { - dprintk(VIDC_WARN, "Invalid parameters: %p\n", core); + dprintk(VIDC_WARN, "Invalid parameters: %pK\n", core); return -EINVAL; } hdev = core->device; @@ -3564,7 +3564,7 @@ int msm_comm_kill_session(struct msm_vidc_inst *inst) } } else { dprintk(VIDC_WARN, - "Inactive session %p, triggering an internal session error\n", + "Inactive session %pK, triggering an internal session error\n", inst); msm_comm_generate_session_error(inst); @@ -3594,7 +3594,7 @@ struct msm_smem *msm_comm_smem_alloc(struct msm_vidc_inst *inst, enum hal_buffer buffer_type, int map_kernel) { if (!inst) { - dprintk(VIDC_ERR, "%s: invalid inst: %p\n", __func__, inst); + dprintk(VIDC_ERR, "%s: invalid inst: %pK\n", __func__, inst); return NULL; } if (power_on_for_smem(inst)) @@ -3608,7 +3608,7 @@ void msm_comm_smem_free(struct msm_vidc_inst *inst, struct msm_smem *mem) { if (!inst || !mem) { dprintk(VIDC_ERR, - "%s: invalid params: %p %p\n", __func__, inst, mem); + "%s: invalid params: %pK %pK\n", __func__, inst, mem); return; } if (inst->state != MSM_VIDC_CORE_INVALID) { @@ -3624,7 +3624,7 @@ int msm_comm_smem_cache_operations(struct msm_vidc_inst *inst, { if (!inst || !mem) { dprintk(VIDC_ERR, - "%s: invalid params: %p %p\n", __func__, inst, mem); + "%s: invalid params: %pK %pK\n", __func__, inst, mem); return -EINVAL; } return msm_smem_cache_operations(inst->mem_client, mem, cache_ops); @@ -3634,7 +3634,7 @@ struct msm_smem *msm_comm_smem_user_to_kernel(struct msm_vidc_inst *inst, int fd, u32 offset, enum hal_buffer buffer_type) { if (!inst) { - dprintk(VIDC_ERR, "%s: invalid inst: %p\n", __func__, inst); + dprintk(VIDC_ERR, "%s: invalid inst: %pK\n", __func__, inst); return NULL; } if (inst->state != MSM_VIDC_CORE_INVALID) { @@ -3651,7 +3651,7 @@ int msm_comm_smem_get_domain_partition(struct msm_vidc_inst *inst, int *domain_num, int *partition_num) { if (!inst || !domain_num || !partition_num) { - dprintk(VIDC_ERR, "%s: invalid params: %p %p %p\n", + dprintk(VIDC_ERR, "%s: invalid params: %pK %pK %pK\n", __func__, inst, domain_num, partition_num); return -EINVAL; } diff --git a/drivers/media/platform/msm/vidc/msm_vidc_debug.c b/drivers/media/platform/msm/vidc/msm_vidc_debug.c index 1fb68b2b978..2823c3d20bb 100644 --- a/drivers/media/platform/msm/vidc/msm_vidc_debug.c +++ b/drivers/media/platform/msm/vidc/msm_vidc_debug.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2015, The Linux Foundation. All rights reserved. +/* Copyright (c) 2012-2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -67,13 +67,13 @@ static ssize_t core_info_read(struct file *file, char __user *buf, int i = 0, rc = 0; if (!core || !core->device) { - dprintk(VIDC_ERR, "Invalid params, core: %p\n", core); + dprintk(VIDC_ERR, "Invalid params, core: %pK\n", core); return 0; } hdev = core->device; INIT_DBG_BUF(dbg_buf); write_str(&dbg_buf, "===============================\n"); - write_str(&dbg_buf, "CORE %d: 0x%p\n", core->id, core); + write_str(&dbg_buf, "CORE %d: 0x%pK\n", core->id, core); write_str(&dbg_buf, "===============================\n"); write_str(&dbg_buf, "Core state: %d\n", core->state); rc = call_hfi_op(hdev, get_fw_info, hdev->hfi_device_data, &fw_info); @@ -246,15 +246,15 @@ static ssize_t inst_info_read(struct file *file, char __user *buf, struct msm_vidc_inst *inst = file->private_data; int i, j; if (!inst) { - dprintk(VIDC_ERR, "Invalid params, core: %p\n", inst); + dprintk(VIDC_ERR, "Invalid params, core: %pK\n", inst); return 0; } INIT_DBG_BUF(dbg_buf); write_str(&dbg_buf, "===============================\n"); - write_str(&dbg_buf, "INSTANCE: 0x%p (%s)\n", inst, + write_str(&dbg_buf, "INSTANCE: 0x%pK (%s)\n", inst, inst->session_type == MSM_VIDC_ENCODER ? "Encoder" : "Decoder"); write_str(&dbg_buf, "===============================\n"); - write_str(&dbg_buf, "core: 0x%p\n", inst->core); + write_str(&dbg_buf, "core: 0x%pK\n", inst->core); write_str(&dbg_buf, "height: %d\n", inst->prop.height[CAPTURE_PORT]); write_str(&dbg_buf, "width: %d\n", inst->prop.width[CAPTURE_PORT]); write_str(&dbg_buf, "fps: %d\n", inst->prop.fps); @@ -312,10 +312,10 @@ struct dentry *msm_vidc_debugfs_init_inst(struct msm_vidc_inst *inst, struct dentry *dir = NULL; char debugfs_name[MAX_DEBUGFS_NAME]; if (!inst) { - dprintk(VIDC_ERR, "Invalid params, inst: %p\n", inst); + dprintk(VIDC_ERR, "Invalid params, inst: %pK\n", inst); goto failed_create_dir; } - snprintf(debugfs_name, MAX_DEBUGFS_NAME, "inst_%p", inst); + snprintf(debugfs_name, MAX_DEBUGFS_NAME, "inst_%pK", inst); dir = debugfs_create_dir(debugfs_name, parent); if (!dir) { dprintk(VIDC_ERR, "Failed to create debugfs for msm_vidc\n"); diff --git a/drivers/media/platform/msm/vidc/q6_hfi.c b/drivers/media/platform/msm/vidc/q6_hfi.c index 4bc534cdfae..5f7cd81cc41 100644 --- a/drivers/media/platform/msm/vidc/q6_hfi.c +++ b/drivers/media/platform/msm/vidc/q6_hfi.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2014, The Linux Foundation. All rights reserved. +/* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -200,7 +200,7 @@ static int q6_hfi_register_iommu_domains(struct q6_hfi_device *device) struct iommu_info *iommu_map; if (!device || !device->res) { - dprintk(VIDC_ERR, "Invalid parameter: %p", device); + dprintk(VIDC_ERR, "Invalid parameter: %pK", device); return -EINVAL; } @@ -218,7 +218,7 @@ static int q6_hfi_register_iommu_domains(struct q6_hfi_device *device) domain = iommu_group_get_iommudata(iommu_map->group); if (IS_ERR_OR_NULL(domain)) { dprintk(VIDC_ERR, - "Failed to get domain data for group %p", + "Failed to get domain data for group %pK", iommu_map->group); rc = -EINVAL; goto fail_group; @@ -226,7 +226,7 @@ static int q6_hfi_register_iommu_domains(struct q6_hfi_device *device) iommu_map->domain = msm_find_domain_no(domain); if (iommu_map->domain < 0) { dprintk(VIDC_ERR, - "Failed to get domain index for domain %p", + "Failed to get domain index for domain %pK", domain); rc = -EINVAL; goto fail_group; @@ -252,7 +252,7 @@ static void q6_hfi_deregister_iommu_domains(struct q6_hfi_device *device) int i = 0; if (!device || !device->res) { - dprintk(VIDC_ERR, "Invalid parameter: %p", device); + dprintk(VIDC_ERR, "Invalid parameter: %pK", device); return; } @@ -345,7 +345,7 @@ static void *q6_hfi_get_device(u32 device_id, int rc = 0; if (!callback) { - dprintk(VIDC_ERR, "%s Invalid params: %p\n", + dprintk(VIDC_ERR, "%s Invalid params: %pK\n", __func__, callback); return NULL; } @@ -1206,7 +1206,7 @@ static int q6_hfi_iommu_attach(struct q6_hfi_device *device) struct iommu_info *iommu_map; if (!device || !device->res) { - dprintk(VIDC_ERR, "Invalid parameter: %p", device); + dprintk(VIDC_ERR, "Invalid parameter: %pK", device); return -EINVAL; } @@ -1221,7 +1221,7 @@ static int q6_hfi_iommu_attach(struct q6_hfi_device *device) rc = IS_ERR(domain) ? PTR_ERR(domain) : -EINVAL; break; } - dprintk(VIDC_DBG, "Attaching domain(id:%d) %p to group %p", + dprintk(VIDC_DBG, "Attaching domain(id:%d) %pK to group %pK", iommu_map->domain, domain, group); rc = iommu_attach_group(domain, group); if (rc) { @@ -1252,7 +1252,7 @@ static void q6_hfi_iommu_detach(struct q6_hfi_device *device) int i; if (!device || !device->res) { - dprintk(VIDC_ERR, "Invalid parameter: %p", device); + dprintk(VIDC_ERR, "Invalid parameter: %pK", device); return; } @@ -1380,7 +1380,7 @@ int q6_hfi_initialize(struct hfi_device *hdev, u32 device_id, int rc = 0; if (!hdev || !res || !callback) { - dprintk(VIDC_ERR, "Invalid params: %p %p %p", + dprintk(VIDC_ERR, "Invalid params: %pK %pK %pK", hdev, res, callback); rc = -EINVAL; goto err_hfi_init; diff --git a/drivers/media/platform/msm/vidc/venus_hfi.c b/drivers/media/platform/msm/vidc/venus_hfi.c index b8656536b61..98f3f6b61f8 100644 --- a/drivers/media/platform/msm/vidc/venus_hfi.c +++ b/drivers/media/platform/msm/vidc/venus_hfi.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2014, The Linux Foundation. All rights reserved. +/* Copyright (c) 2012-2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -247,7 +247,7 @@ static int venus_hfi_write_queue(void *info, u8 *packet, u32 *rx_req_is_set) venus_hfi_sim_modify_cmd_packet(packet); if (msm_vidc_debug & VIDC_PKT) { - dprintk(VIDC_PKT, "%s: %p\n", __func__, qinfo); + dprintk(VIDC_PKT, "%s: %pK\n", __func__, qinfo); venus_hfi_dump_packet(packet); } @@ -431,7 +431,7 @@ static int venus_hfi_read_queue(void *info, u8 *packet, u32 *pb_tx_req_is_set) *pb_tx_req_is_set = (1 == queue->qhdr_tx_req) ? 1 : 0; venus_hfi_hal_sim_modify_msg_packet(packet); if (msm_vidc_debug & VIDC_PKT) { - dprintk(VIDC_PKT, "%s: %p\n", __func__, qinfo); + dprintk(VIDC_PKT, "%s: %pK\n", __func__, qinfo); venus_hfi_dump_packet(packet); } dprintk(VIDC_DBG, "Out : "); @@ -462,7 +462,7 @@ static int venus_hfi_alloc(struct venus_hfi_device *dev, void *mem, rc = -ENOMEM; goto fail_smem_alloc; } - dprintk(VIDC_DBG, "venus_hfi_alloc:ptr=%p,size=%d", + dprintk(VIDC_DBG, "venus_hfi_alloc:ptr=%pK,size=%d", alloc->kvaddr, size); rc = msm_smem_cache_operations(dev->hal_client, alloc, SMEM_CACHE_CLEAN); @@ -482,7 +482,7 @@ fail_smem_alloc: static void venus_hfi_free(struct venus_hfi_device *dev, struct msm_smem *mem) { if (!dev || !mem) { - dprintk(VIDC_ERR, "invalid param %p %p\n", dev, mem); + dprintk(VIDC_ERR, "invalid param %pK %pK\n", dev, mem); return; } venus_hfi_power_enable(dev); @@ -495,7 +495,7 @@ static void venus_hfi_write_register(struct venus_hfi_device *device, u32 reg, u32 hwiosymaddr = reg; u8 *base_addr; if (!device) { - dprintk(VIDC_ERR, "Invalid params: %p\n", device); + dprintk(VIDC_ERR, "Invalid params: %pK\n", device); return; } @@ -535,7 +535,7 @@ static int venus_hfi_read_register(struct venus_hfi_device *device, u32 reg) int rc ; u8 *base_addr; if (!device) { - dprintk(VIDC_ERR, "Invalid params: %p\n", device); + dprintk(VIDC_ERR, "Invalid params: %pK\n", device); return -EINVAL; } @@ -646,7 +646,7 @@ static void venus_hfi_iommu_detach(struct venus_hfi_device *device) int i; if (!device || !device->res) { - dprintk(VIDC_ERR, "Invalid paramter: %p\n", device); + dprintk(VIDC_ERR, "Invalid paramter: %pK\n", device); return; } @@ -920,7 +920,7 @@ static int __alloc_ocmem(struct venus_hfi_device *device) unsigned long size; if (!device || !device->res) { - dprintk(VIDC_ERR, "%s Invalid param, device: 0x%p\n", + dprintk(VIDC_ERR, "%s Invalid param, device: 0x%pK\n", __func__, device); return -EINVAL; } @@ -955,7 +955,7 @@ static int __free_ocmem(struct venus_hfi_device *device) int rc = 0; if (!device || !device->res) { - dprintk(VIDC_ERR, "%s Invalid param, device: 0x%p\n", + dprintk(VIDC_ERR, "%s Invalid param, device: 0x%pK\n", __func__, device); return -EINVAL; } @@ -979,14 +979,14 @@ static int __set_ocmem(struct venus_hfi_device *device, bool locked) struct on_chip_mem *ocmem; if (!device) { - dprintk(VIDC_ERR, "%s Invalid param, device: 0x%p\n", + dprintk(VIDC_ERR, "%s Invalid param, device: 0x%pK\n", __func__, device); return -EINVAL; } ocmem = &device->resources.ocmem; if (!ocmem->buf) { - dprintk(VIDC_ERR, "Invalid params, ocmem_buffer: 0x%p\n", + dprintk(VIDC_ERR, "Invalid params, ocmem_buffer: 0x%pK\n", ocmem->buf); return -EINVAL; } @@ -1015,7 +1015,7 @@ static int __unset_ocmem(struct venus_hfi_device *device) int rc = 0; if (!device) { - dprintk(VIDC_ERR, "%s Invalid param, device: 0x%p\n", + dprintk(VIDC_ERR, "%s Invalid param, device: 0x%pK\n", __func__, device); rc = -EINVAL; goto ocmem_unset_failed; @@ -1046,7 +1046,7 @@ static int __alloc_set_ocmem(struct venus_hfi_device *device, bool locked) int rc = 0; if (!device || !device->res) { - dprintk(VIDC_ERR, "%s Invalid param, device: 0x%p\n", + dprintk(VIDC_ERR, "%s Invalid param, device: 0x%pK\n", __func__, device); return -EINVAL; } @@ -1075,7 +1075,7 @@ static int __unset_free_ocmem(struct venus_hfi_device *device) { int rc = 0; if (!device || !device->res) { - dprintk(VIDC_ERR, "%s Invalid param, device: 0x%p\n", + dprintk(VIDC_ERR, "%s Invalid param, device: 0x%pK\n", __func__, device); return -EINVAL; } @@ -1250,7 +1250,7 @@ static int venus_hfi_halt_axi(struct venus_hfi_device *device) u32 reg; int rc = 0; if (!device) { - dprintk(VIDC_ERR, "Invalid input: %p\n", device); + dprintk(VIDC_ERR, "Invalid input: %pK\n", device); return -EINVAL; } mutex_lock(&device->clk_pwr_lock); @@ -1281,7 +1281,7 @@ static inline int venus_hfi_power_off(struct venus_hfi_device *device) { int rc = 0; if (!device) { - dprintk(VIDC_ERR, "Invalid params: %p\n", device); + dprintk(VIDC_ERR, "Invalid params: %pK\n", device); return -EINVAL; } if (!device->power_enabled) { @@ -1324,7 +1324,7 @@ static inline int venus_hfi_power_on(struct venus_hfi_device *device) { int rc = 0; if (!device) { - dprintk(VIDC_ERR, "Invalid params: %p\n", device); + dprintk(VIDC_ERR, "Invalid params: %pK\n", device); return -EINVAL; } @@ -1431,7 +1431,7 @@ static int venus_hfi_power_enable(void *dev) int rc = 0; struct venus_hfi_device *device = dev; if (!device) { - dprintk(VIDC_ERR, "Invalid params: %p\n", device); + dprintk(VIDC_ERR, "Invalid params: %pK\n", device); return -EINVAL; } @@ -1459,7 +1459,7 @@ static inline int venus_hfi_clk_gating_off(struct venus_hfi_device *device) { int rc = 0; if (!device) { - dprintk(VIDC_ERR, "Invalid params: %p\n", device); + dprintk(VIDC_ERR, "Invalid params: %pK\n", device); return -EINVAL; } if (device->clk_state == ENABLED_PREPARED) { @@ -3141,7 +3141,7 @@ static void venus_hfi_core_work_handler(struct work_struct *work) dprintk(VIDC_INFO, " GOT INTERRUPT () "); if (!device->callback) { - dprintk(VIDC_ERR, "No interrupt callback function: %p\n", + dprintk(VIDC_ERR, "No interrupt callback function: %pK\n", device); return; } @@ -3227,7 +3227,7 @@ static inline int venus_hfi_init_clocks(struct msm_vidc_platform_resources *res, int rc = 0; struct venus_core_clock *clock; if (!res || !device) { - dprintk(VIDC_ERR, "Invalid params: %p\n", device); + dprintk(VIDC_ERR, "Invalid params: %pK\n", device); return -EINVAL; } clock = device->resources.clock; @@ -3308,7 +3308,7 @@ static inline void venus_hfi_disable_unprepare_clks( int i; struct venus_core_clock *cl; if (!device) { - dprintk(VIDC_ERR, "Invalid params: %p\n", device); + dprintk(VIDC_ERR, "Invalid params: %pK\n", device); return; } @@ -3364,7 +3364,7 @@ static inline int venus_hfi_prepare_enable_clks(struct venus_hfi_device *device) struct venus_core_clock *cl; int rc = 0; if (!device) { - dprintk(VIDC_ERR, "Invalid params: %p\n", device); + dprintk(VIDC_ERR, "Invalid params: %pK\n", device); return -EINVAL; } WARN_ON(!mutex_is_locked(&device->clk_pwr_lock)); @@ -3429,7 +3429,7 @@ static int venus_hfi_register_iommu_domains(struct venus_hfi_device *device, domain = iommu_group_get_iommudata(iommu_map->group); if (!domain) { dprintk(VIDC_ERR, - "Failed to get domain data for group %p", + "Failed to get domain data for group %pK", iommu_map->group); rc = -EINVAL; goto fail_group; @@ -3437,7 +3437,7 @@ static int venus_hfi_register_iommu_domains(struct venus_hfi_device *device, iommu_map->domain = msm_find_domain_no(domain); if (iommu_map->domain < 0) { dprintk(VIDC_ERR, - "Failed to get domain index for domain %p", + "Failed to get domain index for domain %pK", domain); rc = -EINVAL; goto fail_group; @@ -3557,7 +3557,7 @@ static int venus_hfi_init_resources(struct venus_hfi_device *device, device->res = res; if (!res) { - dprintk(VIDC_ERR, "Invalid params: %p\n", res); + dprintk(VIDC_ERR, "Invalid params: %pK\n", res); return -ENODEV; } device->gdsc = devm_regulator_get(&res->pdev->dev, "vdd"); @@ -3612,7 +3612,7 @@ static int venus_hfi_iommu_get_domain_partition(void *dev, u32 flags, struct venus_hfi_device *device = dev; if (!device) { - dprintk(VIDC_ERR, "%s: Invalid param device: %p\n", + dprintk(VIDC_ERR, "%s: Invalid param device: %pK\n", __func__, device); return -EINVAL; } @@ -3636,7 +3636,7 @@ static int protect_cp_mem(struct venus_hfi_device *device) iommu_group_set = &device->res->iommu_group_set; if (!iommu_group_set) { - dprintk(VIDC_ERR, "invalid params: %p\n", iommu_group_set); + dprintk(VIDC_ERR, "invalid params: %pK\n", iommu_group_set); return -EINVAL; } @@ -3695,7 +3695,7 @@ static int venus_hfi_load_fw(void *dev) struct venus_hfi_device *device = dev; if (!device || !device->gdsc) { - dprintk(VIDC_ERR, "%s Invalid paramter: %p\n", + dprintk(VIDC_ERR, "%s Invalid paramter: %pK\n", __func__, device); return -EINVAL; } @@ -3760,7 +3760,7 @@ static void venus_hfi_unload_fw(void *dev) { struct venus_hfi_device *device = dev; if (!device) { - dprintk(VIDC_ERR, "%s Invalid paramter: %p\n", + dprintk(VIDC_ERR, "%s Invalid paramter: %pK\n", __func__, device); return; } @@ -3800,7 +3800,7 @@ static int venus_hfi_get_fw_info(void *dev, struct hal_fw_info *fw_info) if (!device|| !fw_info) { dprintk(VIDC_ERR, - "%s Invalid paramter: device = %p fw_info = %p\n", + "%s Invalid paramter: device = %pK fw_info = %pK\n", __func__, device, fw_info); return -EINVAL; } @@ -3953,7 +3953,7 @@ static void *venus_hfi_get_device(u32 device_id, int rc = 0; if (!res || !callback) { - dprintk(VIDC_ERR, "Invalid params: %p %p\n", res, callback); + dprintk(VIDC_ERR, "Invalid params: %pK %pK\n", res, callback); return NULL; } @@ -4047,7 +4047,7 @@ int venus_hfi_initialize(struct hfi_device *hdev, u32 device_id, int rc = 0; if (!hdev || !res || !callback) { - dprintk(VIDC_ERR, "Invalid params: %p %p %p\n", + dprintk(VIDC_ERR, "Invalid params: %pK %pK %pK\n", hdev, res, callback); rc = -EINVAL; goto err_venus_hfi_init; diff --git a/drivers/media/platform/msm/vidc/vidc_hfi.c b/drivers/media/platform/msm/vidc/vidc_hfi.c index ef0de370eb0..193b42f69aa 100644 --- a/drivers/media/platform/msm/vidc/vidc_hfi.c +++ b/drivers/media/platform/msm/vidc/vidc_hfi.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2013, The Linux Foundation. All rights reserved. +/* Copyright (c) 2012-2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -61,7 +61,7 @@ void vidc_hfi_deinitialize(enum msm_vidc_hfi_type hfi_type, struct hfi_device *hdev) { if (!hdev) { - dprintk(VIDC_ERR, "%s invalid device %p", __func__, hdev); + dprintk(VIDC_ERR, "%s invalid device %pK", __func__, hdev); return; } From af482a2ed885effba7776d959f4eaa4aeb028bb3 Mon Sep 17 00:00:00 2001 From: VijayaKumar T M Date: Mon, 14 Nov 2016 11:15:43 +0530 Subject: [PATCH 134/168] msm: camera: Restructure data handling to be more robust Use dynamic array allocation instead of static array to prevent stack overflow. User-supplied number of bytes may result in integer overflow. To fix this we check that the num_byte isn't above 8K size. CRs-Fixed: 1060554 Change-Id: I9b05b846e5cc3a62b1a0a67be529f09abc764796 Signed-off-by: VijayaKumar T M --- .../camera_v2/sensor/io/msm_camera_cci_i2c.c | 6 +++ .../camera_v2/sensor/io/msm_camera_qup_i2c.c | 37 ++++++++++++++++++- include/media/msm_cam_sensor.h | 1 + 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_cci_i2c.c b/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_cci_i2c.c index 6af0ceab5cc..afa73337380 100644 --- a/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_cci_i2c.c +++ b/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_cci_i2c.c @@ -76,6 +76,12 @@ int32_t msm_camera_cci_i2c_read_seq(struct msm_camera_i2c_client *client, || num_byte == 0) return rc; + if (num_byte > I2C_REG_DATA_MAX) { + pr_err("%s: Error num_byte:0x%x exceeds 8K max supported:0x%x\n", + __func__, num_byte, I2C_REG_DATA_MAX); + return rc; + } + buf = kzalloc(num_byte, GFP_KERNEL); if (!buf) { pr_err("%s:%d no memory\n", __func__, __LINE__); diff --git a/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_qup_i2c.c b/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_qup_i2c.c index 6f9aeecca0c..1c11f0cecf5 100644 --- a/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_qup_i2c.c +++ b/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_qup_i2c.c @@ -77,7 +77,7 @@ int32_t msm_camera_qup_i2c_read(struct msm_camera_i2c_client *client, enum msm_camera_i2c_data_type data_type) { int32_t rc = -EFAULT; - unsigned char buf[client->addr_type+data_type]; + unsigned char *buf = NULL; if ((client->addr_type != MSM_CAMERA_I2C_BYTE_ADDR && client->addr_type != MSM_CAMERA_I2C_WORD_ADDR) @@ -85,6 +85,17 @@ int32_t msm_camera_qup_i2c_read(struct msm_camera_i2c_client *client, && data_type != MSM_CAMERA_I2C_WORD_DATA)) return rc; + if (client->addr_type > UINT_MAX - data_type) { + pr_err("%s: integer overflow prevented\n", __func__); + return rc; + } + + buf = kzalloc(client->addr_type+data_type, GFP_KERNEL); + if (!buf) { + pr_err("%s:%d no memory\n", __func__, __LINE__); + return -ENOMEM; + } + if (client->addr_type == MSM_CAMERA_I2C_BYTE_ADDR) { buf[0] = addr; } else if (client->addr_type == MSM_CAMERA_I2C_WORD_ADDR) { @@ -94,6 +105,8 @@ int32_t msm_camera_qup_i2c_read(struct msm_camera_i2c_client *client, rc = msm_camera_qup_i2c_rxdata(client, buf, data_type); if (rc < 0) { S_I2C_DBG("%s fail\n", __func__); + kfree(buf); + buf = NULL; return rc; } @@ -103,6 +116,8 @@ int32_t msm_camera_qup_i2c_read(struct msm_camera_i2c_client *client, *data = buf[0] << 8 | buf[1]; S_I2C_DBG("%s addr = 0x%x data: 0x%x\n", __func__, addr, *data); + kfree(buf); + buf = NULL; return rc; } @@ -110,7 +125,7 @@ int32_t msm_camera_qup_i2c_read_seq(struct msm_camera_i2c_client *client, uint32_t addr, uint8_t *data, uint32_t num_byte) { int32_t rc = -EFAULT; - unsigned char buf[client->addr_type+num_byte]; + unsigned char *buf = NULL; int i; if ((client->addr_type != MSM_CAMERA_I2C_BYTE_ADDR @@ -118,6 +133,20 @@ int32_t msm_camera_qup_i2c_read_seq(struct msm_camera_i2c_client *client, || num_byte == 0) return rc; + if (num_byte > I2C_REG_DATA_MAX) { + pr_err("%s: Error num_byte:0x%x exceeds 8K max supported:0x%x\n", + __func__, num_byte, I2C_REG_DATA_MAX); + return rc; + } + if (client->addr_type > UINT_MAX - num_byte) { + pr_err("%s: integer overflow prevented\n", __func__); + return rc; + } + buf = kzalloc(client->addr_type+num_byte, GFP_KERNEL); + if (!buf) { + pr_err("%s:%d no memory\n", __func__, __LINE__); + return -ENOMEM; + } if (client->addr_type == MSM_CAMERA_I2C_BYTE_ADDR) { buf[0] = addr; } else if (client->addr_type == MSM_CAMERA_I2C_WORD_ADDR) { @@ -127,6 +156,8 @@ int32_t msm_camera_qup_i2c_read_seq(struct msm_camera_i2c_client *client, rc = msm_camera_qup_i2c_rxdata(client, buf, num_byte); if (rc < 0) { S_I2C_DBG("%s fail\n", __func__); + kfree(buf); + buf = NULL; return rc; } @@ -136,6 +167,8 @@ int32_t msm_camera_qup_i2c_read_seq(struct msm_camera_i2c_client *client, S_I2C_DBG("Byte %d: 0x%x\n", i, buf[i]); S_I2C_DBG("Data: 0x%x\n", data[i]); } + kfree(buf); + buf = NULL; return rc; } diff --git a/include/media/msm_cam_sensor.h b/include/media/msm_cam_sensor.h index a140d4b3a7e..20bf8547bce 100644 --- a/include/media/msm_cam_sensor.h +++ b/include/media/msm_cam_sensor.h @@ -10,6 +10,7 @@ #define I2C_SEQ_REG_SETTING_MAX 5 #define I2C_SEQ_REG_DATA_MAX 20 +#define I2C_REG_DATA_MAX (8*1024) #define MAX_CID 16 #define I2C_USER_REG_DATA_MAX 1024 From 28179d92ddb1e722f166b0b1508b55bcf2a01612 Mon Sep 17 00:00:00 2001 From: VijayaKumar T M Date: Tue, 22 Nov 2016 16:10:30 +0530 Subject: [PATCH 135/168] msm: camera: cpp: Add validation for v4l2 ioctl arguments In CPP v4l2 ioctl command is made, if _IOC_DIR(cmd) is _IOC_NONE, then the user-supplied argument arg is not checked and an information disclosure is possible. CRs-Fixed: 1042068 Change-Id: Iddb291b10cdcb5c42ab8497e06c2ce47885cd5ab Signed-off-by: Sunid Wilson Signed-off-by: VijayaKumar T M --- .../msm/camera_v2/pproc/cpp/msm_cpp.c | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/drivers/media/platform/msm/camera_v2/pproc/cpp/msm_cpp.c b/drivers/media/platform/msm/camera_v2/pproc/cpp/msm_cpp.c index a2f3c3480df..e28f6f7c6e2 100644 --- a/drivers/media/platform/msm/camera_v2/pproc/cpp/msm_cpp.c +++ b/drivers/media/platform/msm/camera_v2/pproc/cpp/msm_cpp.c @@ -1525,25 +1525,28 @@ void msm_cpp_clean_queue(struct cpp_device *cpp_dev) long msm_cpp_subdev_ioctl(struct v4l2_subdev *sd, unsigned int cmd, void *arg) { - struct cpp_device *cpp_dev = v4l2_get_subdevdata(sd); + struct cpp_device *cpp_dev = NULL; struct msm_camera_v4l2_ioctl_t *ioctl_ptr = arg; int rc = 0; - if (ioctl_ptr == NULL) { - pr_err("ioctl_ptr is null\n"); + if ((sd == NULL) || (ioctl_ptr == NULL)) { + pr_err("Wrong ioctl_ptr %p, sd %p\n", ioctl_ptr, sd); return -EINVAL; } - if (cpp_dev == NULL) { - pr_err("cpp_dev is null\n"); - return -EINVAL; - } - if ((ioctl_ptr->ioctl_ptr == NULL) || (ioctl_ptr->len == 0)) { pr_err("ioctl_ptr OR ioctl_ptr->len is NULL %p %d\n", ioctl_ptr, ioctl_ptr->len); return -EINVAL; } - + if (_IOC_DIR(cmd) == _IOC_NONE) { + pr_err("Invalid ioctl/subdev cmd %u\n", cmd); + return -EINVAL; + } + cpp_dev = v4l2_get_subdevdata(sd); + if (cpp_dev == NULL) { + pr_err("cpp_dev is null\n"); + return -EINVAL; + } mutex_lock(&cpp_dev->mutex); CPP_DBG("E cmd: %d\n", cmd); switch (cmd) { From ad4184dcd38f42889a58252236be94fd6d028077 Mon Sep 17 00:00:00 2001 From: VijayaKumar T M Date: Wed, 23 Nov 2016 11:43:56 +0530 Subject: [PATCH 136/168] msm: camera: restructure data handling to be more robust Add dynamic array allocation instead of static to prevent stack overflow. CRs-Fixed: 789704 Change-Id: Id12ed5b01809021d2b1d1d71436f2523b575d9de Signed-off-by: Vasko Kalanoski Signed-off-by: VijayaKumar T M --- .../camera_v2/sensor/io/msm_camera_cci_i2c.c | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_cci_i2c.c b/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_cci_i2c.c index afa73337380..808e240983e 100644 --- a/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_cci_i2c.c +++ b/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_cci_i2c.c @@ -148,17 +148,27 @@ int32_t msm_camera_cci_i2c_write_seq(struct msm_camera_i2c_client *client, int32_t rc = -EFAULT; uint8_t i = 0; struct msm_camera_cci_ctrl cci_ctrl; - struct msm_camera_i2c_reg_array reg_conf_tbl[num_byte]; + struct msm_camera_i2c_reg_array *reg_conf_tbl = NULL; if ((client->addr_type != MSM_CAMERA_I2C_BYTE_ADDR && client->addr_type != MSM_CAMERA_I2C_WORD_ADDR) || num_byte == 0) return rc; + if (num_byte > I2C_SEQ_REG_DATA_MAX) { + pr_err("%s: num_byte=%d clamped to max supported %d\n", + __func__, num_byte, I2C_SEQ_REG_DATA_MAX); + return rc; + } S_I2C_DBG("%s reg addr = 0x%x num bytes: %d\n", - __func__, addr, num_byte); - memset(reg_conf_tbl, 0, - num_byte * sizeof(struct msm_camera_i2c_reg_array)); + __func__, addr, num_byte); + + reg_conf_tbl = kzalloc(num_byte * + (sizeof(struct msm_camera_i2c_reg_array)), GFP_KERNEL); + if (!reg_conf_tbl) { + pr_err("%s:%d no memory\n", __func__, __LINE__); + return -ENOMEM; + } reg_conf_tbl[0].reg_addr = addr; for (i = 0; i < num_byte; i++) { reg_conf_tbl[i].reg_data = data[i]; @@ -174,6 +184,8 @@ int32_t msm_camera_cci_i2c_write_seq(struct msm_camera_i2c_client *client, core, ioctl, VIDIOC_MSM_CCI_CFG, &cci_ctrl); CDBG("%s line %d rc = %d\n", __func__, __LINE__, rc); rc = cci_ctrl.status; + kfree(reg_conf_tbl); + reg_conf_tbl = NULL; return rc; } From 6192d9c11e4ec888fed6c6a0909a45a1877085bb Mon Sep 17 00:00:00 2001 From: chengengjia Date: Wed, 10 Aug 2016 17:34:43 +0800 Subject: [PATCH 137/168] input: synaptics: Add checks of user input data Add checks of the user input count to avoid possible heap overflow Bug: 30799828 Change-Id: I896492b18c4ace6565fb9edd5cbf51f363ce157b Signed-off-by: chengengjia Signed-off-by: Andrew Chant Git-repo: https://android.googlesource.com/kernel/msm.git Git-commit: f549796fb9da58586c5cfc31d07b243c87dcfbd5 Signed-off-by: Dennis Cagle --- drivers/input/touchscreen/synaptics_fw_update.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/input/touchscreen/synaptics_fw_update.c b/drivers/input/touchscreen/synaptics_fw_update.c index dce9dee0106..92cecb1bce7 100644 --- a/drivers/input/touchscreen/synaptics_fw_update.c +++ b/drivers/input/touchscreen/synaptics_fw_update.c @@ -1684,6 +1684,13 @@ static ssize_t fwu_sysfs_store_image(struct file *data_file, struct kobject *kobj, struct bin_attribute *attributes, char *buf, loff_t pos, size_t count) { + if (count > fwu->image_size - fwu->data_pos) { + dev_err(&fwu->rmi4_data->i2c_client->dev, + "%s: Not enough space in buffer\n", + __func__); + return -EINVAL; + } + memcpy((void *)(&fwu->ext_data_source[fwu->data_pos]), (const void *)buf, count); From e940acc36b9fe5b5cf47606c8644ae80e101895a Mon Sep 17 00:00:00 2001 From: Jeremy Gebben Date: Fri, 27 Feb 2015 11:32:29 -0700 Subject: [PATCH 138/168] msm: kgsl: fix sync file error handling Reorder our sync_fence_install() calls to happen after all possible failures so that error cleanup will be correct. CRs-Fixed: 1081855 Change-Id: I0e7bb459f2acc010446ac5e5b3b72c8b16cce079 Signed-off-by: Jeremy Gebben Signed-off-by: Sudeep Yedalapure --- drivers/gpu/msm/kgsl_sync.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/msm/kgsl_sync.c b/drivers/gpu/msm/kgsl_sync.c index 0747939f525..d10429d38dd 100644 --- a/drivers/gpu/msm/kgsl_sync.c +++ b/drivers/gpu/msm/kgsl_sync.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2014, The Linux Foundation. All rights reserved. +/* Copyright (c) 2012-2014,2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -194,7 +194,6 @@ int kgsl_add_fence_event(struct kgsl_device *device, ret = priv.fence_fd; goto unlock; } - sync_fence_install(fence, priv.fence_fd); /* * If the timestamp hasn't expired yet create an event to trigger it. @@ -221,6 +220,7 @@ int kgsl_add_fence_event(struct kgsl_device *device, ret = -EFAULT; goto out; } + sync_fence_install(fence, priv.fence_fd); return 0; From 55158d2c505b1d425732b8b4a0d5249946607246 Mon Sep 17 00:00:00 2001 From: Andrew Chant Date: Thu, 15 Sep 2016 16:19:50 -0700 Subject: [PATCH 139/168] Input: synaptics: check input, prevent sysfs races concurrent sysfs calls on the fw updater can cause ugly race conditions. Return EBUSY on concurrent sysfs calls. For sysfs calls which generate deferred work, prevent the deferred work from running concurrently with other sysfs calls. Also check that ext_data_source is appropriately sized and allocated, based on a patch by Gengjia Chen (chengjia4574@gmail.com). Signed-off-by: Andrew Chant Change-Id:I5bbe4992f3fd2d23db288296eaeb61f5831098e9 Bug: 30799828 Bug: 31252388 Git-repo: https://android.googlesource.com/kernel/msm.git Git-commit: 287ce2ccfefe68067c1f9f5175b6664bf7397fe6 Signed-off-by: Srinivasa Rao Kuppala --- .../input/touchscreen/synaptics_fw_update.c | 113 ++++++++++++++---- 1 file changed, 90 insertions(+), 23 deletions(-) diff --git a/drivers/input/touchscreen/synaptics_fw_update.c b/drivers/input/touchscreen/synaptics_fw_update.c index 92cecb1bce7..5ff2ef69fc1 100644 --- a/drivers/input/touchscreen/synaptics_fw_update.c +++ b/drivers/input/touchscreen/synaptics_fw_update.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -294,6 +295,7 @@ struct synaptics_rmi4_fwu_handle { static struct synaptics_rmi4_fwu_handle *fwu; DECLARE_COMPLETION(fwu_remove_complete); +DEFINE_MUTEX(fwu_sysfs_mutex); static unsigned int extract_uint(const unsigned char *ptr) { @@ -1667,28 +1669,47 @@ static ssize_t fwu_sysfs_show_image(struct file *data_file, char *buf, loff_t pos, size_t count) { struct synaptics_rmi4_data *rmi4_data = fwu->rmi4_data; + ssize_t retval; + + if (!mutex_trylock(&fwu_sysfs_mutex)) + return -EBUSY; if (count < fwu->config_size) { dev_err(&rmi4_data->i2c_client->dev, "%s: Not enough space (%zu bytes) in buffer\n", __func__, count); - return -EINVAL; + retval = -EINVAL; + goto show_image_exit; } memcpy(buf, fwu->read_config_buf, fwu->config_size); - - return fwu->config_size; + retval = fwu->config_size; +show_image_exit: + mutex_unlock(&fwu_sysfs_mutex); + return retval; } static ssize_t fwu_sysfs_store_image(struct file *data_file, struct kobject *kobj, struct bin_attribute *attributes, char *buf, loff_t pos, size_t count) { + ssize_t retval; + + if (!mutex_trylock(&fwu_sysfs_mutex)) + return -EBUSY; + if (!fwu->ext_data_source) { + dev_err(&fwu->rmi4_data->i2c_client->dev, + "Cannot use this without setting imagesize!\n"); + retval = -EAGAIN; + goto exit; + } if (count > fwu->image_size - fwu->data_pos) { dev_err(&fwu->rmi4_data->i2c_client->dev, "%s: Not enough space in buffer\n", __func__); - return -EINVAL; + + retval = -EINVAL; + goto exit; } memcpy((void *)(&fwu->ext_data_source[fwu->data_pos]), @@ -1697,8 +1718,10 @@ static ssize_t fwu_sysfs_store_image(struct file *data_file, fwu->data_buffer = fwu->ext_data_source; fwu->data_pos += count; - - return count; + retval = count; +exit: + mutex_unlock(&fwu_sysfs_mutex); + return retval; } static ssize_t fwu_sysfs_image_name_store(struct device *dev, @@ -1721,18 +1744,29 @@ static ssize_t fwu_sysfs_image_name_store(struct device *dev, return -EINVAL; } + if (!mutex_trylock(&fwu_sysfs_mutex)) + return -EBUSY; strlcpy(rmi4_data->fw_image_name, buf, count); + mutex_unlock(&fwu_sysfs_mutex); + return count; } static ssize_t fwu_sysfs_image_name_show(struct device *dev, struct device_attribute *attr, char *buf) { + ssize_t retval; + + if (!mutex_trylock(&fwu_sysfs_mutex)) + return -EBUSY; + if (strnlen(fwu->rmi4_data->fw_image_name, NAME_BUFFER_SIZE) > 0) - return snprintf(buf, PAGE_SIZE, "%s\n", + retval = snprintf(buf, PAGE_SIZE, "%s\n", fwu->rmi4_data->fw_image_name); else - return snprintf(buf, PAGE_SIZE, "No firmware name given\n"); + retval = snprintf(buf, PAGE_SIZE, "No firmware name given\n"); + mutex_unlock(&fwu_sysfs_mutex); + return retval; } static ssize_t fwu_sysfs_force_reflash_store(struct device *dev, @@ -1742,10 +1776,12 @@ static ssize_t fwu_sysfs_force_reflash_store(struct device *dev, unsigned int input; struct synaptics_rmi4_data *rmi4_data = fwu->rmi4_data; - if (sscanf(buf, "%u", &input) != 1) { - retval = -EINVAL; + if (!mutex_trylock(&fwu_sysfs_mutex)) + return -EBUSY; + + retval = kstrtouint(buf, 10, &input); + if (retval) goto exit; - } if (input != 1) { retval = -EINVAL; @@ -1769,7 +1805,10 @@ exit: kfree(fwu->ext_data_source); fwu->ext_data_source = NULL; fwu->force_update = FORCE_UPDATE; + fwu->image_size = 0; + fwu->data_pos = 0; fwu->do_lockdown = rmi4_data->board->do_lockdown; + mutex_unlock(&fwu_sysfs_mutex); return retval; } @@ -1780,10 +1819,12 @@ static ssize_t fwu_sysfs_do_reflash_store(struct device *dev, unsigned int input; struct synaptics_rmi4_data *rmi4_data = fwu->rmi4_data; - if (sscanf(buf, "%u", &input) != 1) { - retval = -EINVAL; + if (!mutex_trylock(&fwu_sysfs_mutex)) + return -EBUSY; + + retval = kstrtouint(buf, 10, &input); + if (retval) goto exit; - } if (input & LOCKDOWN) { fwu->do_lockdown = true; @@ -1811,8 +1852,11 @@ static ssize_t fwu_sysfs_do_reflash_store(struct device *dev, exit: kfree(fwu->ext_data_source); fwu->ext_data_source = NULL; + fwu->image_size = 0; + fwu->data_pos = 0; fwu->force_update = FORCE_UPDATE; fwu->do_lockdown = rmi4_data->board->do_lockdown; + mutex_unlock(&fwu_sysfs_mutex); return retval; } @@ -1823,6 +1867,9 @@ static ssize_t fwu_sysfs_write_lockdown_store(struct device *dev, unsigned int input; struct synaptics_rmi4_data *rmi4_data = fwu->rmi4_data; + if (!mutex_trylock(&fwu_sysfs_mutex)) + return -EBUSY; + if (sscanf(buf, "%u", &input) != 1) { retval = -EINVAL; goto exit; @@ -1848,6 +1895,9 @@ exit: fwu->ext_data_source = NULL; fwu->force_update = FORCE_UPDATE; fwu->do_lockdown = rmi4_data->board->do_lockdown; + fwu->image_size = 0; + fwu->data_pos = 0; + mutex_unlock(&fwu_sysfs_mutex); return retval; } @@ -1858,10 +1908,12 @@ static ssize_t fwu_sysfs_write_config_store(struct device *dev, unsigned int input; struct synaptics_rmi4_data *rmi4_data = fwu->rmi4_data; - if (sscanf(buf, "%u", &input) != 1) { - retval = -EINVAL; + if (!mutex_trylock(&fwu_sysfs_mutex)) + return -EBUSY; + + retval = kstrtouint(buf, 10, &input); + if (retval) goto exit; - } if (input != 1) { retval = -EINVAL; @@ -1881,6 +1933,9 @@ static ssize_t fwu_sysfs_write_config_store(struct device *dev, exit: kfree(fwu->ext_data_source); fwu->ext_data_source = NULL; + fwu->image_size = 0; + fwu->data_pos = 0; + mutex_unlock(&fwu_sysfs_mutex); return retval; } @@ -1897,7 +1952,11 @@ static ssize_t fwu_sysfs_read_config_store(struct device *dev, if (input != 1) return -EINVAL; + if (!mutex_trylock(&fwu_sysfs_mutex)) + return -EBUSY; retval = fwu_do_read_config(); + mutex_unlock(&fwu_sysfs_mutex); + if (retval < 0) { dev_err(&rmi4_data->i2c_client->dev, "%s: Failed to read config\n", @@ -1926,7 +1985,10 @@ static ssize_t fwu_sysfs_config_area_store(struct device *dev, return -EINVAL; } + if (!mutex_trylock(&fwu_sysfs_mutex)) + return -EBUSY; fwu->config_area = config_area; + mutex_unlock(&fwu_sysfs_mutex); return count; } @@ -1936,25 +1998,28 @@ static ssize_t fwu_sysfs_image_size_store(struct device *dev, { int retval; unsigned long size; - struct synaptics_rmi4_data *rmi4_data = fwu->rmi4_data; retval = kstrtoul(buf, 10, &size); if (retval) return retval; + if (!mutex_trylock(&fwu_sysfs_mutex)) + return -EBUSY; + fwu->image_size = size; fwu->data_pos = 0; kfree(fwu->ext_data_source); fwu->ext_data_source = kzalloc(fwu->image_size, GFP_KERNEL); if (!fwu->ext_data_source) { - dev_err(&rmi4_data->i2c_client->dev, - "%s: Failed to alloc mem for image data\n", - __func__); - return -ENOMEM; + retval = -ENOMEM; + goto exit; } - return count; + retval = count; +exit: + mutex_unlock(&fwu_sysfs_mutex); + return retval; } static ssize_t fwu_sysfs_block_size_show(struct device *dev, @@ -2118,7 +2183,9 @@ static struct device_attribute attrs[] = { static void synaptics_rmi4_fwu_work(struct work_struct *work) { + mutex_lock(&fwu_sysfs_mutex); fwu_start_reflash(); + mutex_unlock(&fwu_sysfs_mutex); } static int synaptics_rmi4_fwu_init(struct synaptics_rmi4_data *rmi4_data) From af99b4f13a356075b408b0075090ee40943088fd Mon Sep 17 00:00:00 2001 From: VijayaKumar T M Date: Tue, 18 Oct 2016 14:55:01 +0530 Subject: [PATCH 140/168] msm: sensor: Avoid potential stack overflow Add a check to validate the user input data is not greater than expected stack buffer size to avoid out of bounds array accesses CRs-Fixed: 1056307 Change-Id: I8b31006772367a120828269243b1971d33a4d7d3 Signed-off-by: VijayaKumar T M --- .../platform/msm/camera_v2/sensor/io/msm_camera_cci_i2c.c | 8 +++++++- .../platform/msm/camera_v2/sensor/io/msm_camera_qup_i2c.c | 7 ++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_cci_i2c.c b/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_cci_i2c.c index 808e240983e..304f5c39b46 100644 --- a/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_cci_i2c.c +++ b/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_cci_i2c.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2013, The Linux Foundation. All rights reserved. +/* Copyright (c) 2011-2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -251,6 +251,12 @@ int32_t msm_camera_cci_i2c_write_seq_table( client_addr_type = client->addr_type; client->addr_type = write_setting->addr_type; + if (reg_setting->reg_data_size > I2C_SEQ_REG_DATA_MAX) { + pr_err("%s: number of bytes %u exceeding the max supported %d\n", + __func__, reg_setting->reg_data_size, I2C_SEQ_REG_DATA_MAX); + return rc; + } + for (i = 0; i < write_setting->size; i++) { rc = msm_camera_cci_i2c_write_seq(client, reg_setting->reg_addr, reg_setting->reg_data, reg_setting->reg_data_size); diff --git a/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_qup_i2c.c b/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_qup_i2c.c index 1c11f0cecf5..ae05488ff04 100644 --- a/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_qup_i2c.c +++ b/drivers/media/platform/msm/camera_v2/sensor/io/msm_camera_qup_i2c.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011, 2013, The Linux Foundation. All rights reserved. +/* Copyright (c) 2011, 2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -322,6 +322,11 @@ int32_t msm_camera_qup_i2c_write_seq_table(struct msm_camera_i2c_client *client, client_addr_type = client->addr_type; client->addr_type = write_setting->addr_type; + if (reg_setting->reg_data_size > I2C_SEQ_REG_DATA_MAX) { + pr_err("%s: number of bytes %u exceeding the max supported %d\n", + __func__, reg_setting->reg_data_size, I2C_SEQ_REG_DATA_MAX); + return rc; + } for (i = 0; i < write_setting->size; i++) { rc = msm_camera_qup_i2c_write_seq(client, reg_setting->reg_addr, reg_setting->reg_data, reg_setting->reg_data_size); From 2e8308128aad2de683a1e3db9c34044687dd8efe Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Mon, 6 Jun 2016 11:36:45 -0700 Subject: [PATCH 141/168] spmi: prevent showing the address of spmidev Creating devices with the address of the container spmidev is not indicative of the actual hardware device it represents. Instead use the slave id to indicate the device it represents. CRs-Fixed: 1024197 Change-Id: I254d89ff1058752e523eaae8e27976440858f541 Signed-off-by: Abhijeet Dharmapurikar --- drivers/spmi/spmi.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/spmi/spmi.c b/drivers/spmi/spmi.c index 43f27104061..113c3291914 100644 --- a/drivers/spmi/spmi.c +++ b/drivers/spmi/spmi.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2014, The Linux Foundation. All rights reserved. +/* Copyright (c) 2012-2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -252,7 +252,7 @@ int spmi_add_device(struct spmi_device *spmidev) } /* Set the device name */ - dev_set_name(dev, "%s-%p", spmidev->name, spmidev); + dev_set_name(dev, "%s-%02d", spmidev->name, spmidev->sid); /* Device may be bound to an active driver when this returns */ rc = device_add(dev); From 85b891e0dc12aaf16e1fc4ed42da4112e6c42bc2 Mon Sep 17 00:00:00 2001 From: Srinivasarao P Date: Wed, 17 Aug 2016 13:17:40 +0530 Subject: [PATCH 142/168] msm: dma_test: Initialize newly allocated memory The MSM_DMA_IOALLOC ioctl command allocates kernel memory and this memory can be read back using the MSM_DMA_IORBUF ioctl command. This memory is not zero-initialized and may contain sensitive data. Change-Id: I8c55d6fe500e7607690b89806715893783eecf9c Signed-off-by: Srinivasarao P --- arch/arm/mach-msm/dma_test.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm/mach-msm/dma_test.c b/arch/arm/mach-msm/dma_test.c index 3d13e4e3cdc..3867f13322e 100644 --- a/arch/arm/mach-msm/dma_test.c +++ b/arch/arm/mach-msm/dma_test.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2008-2009, The Linux Foundation. All rights reserved. +/* Copyright (c) 2008-2009,2016 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -99,7 +99,7 @@ static int buffer_req(struct msm_dma_alloc_req *req) if (i >= MAX_TEST_BUFFERS) goto error; - buffers[i] = kmalloc(req->size, GFP_KERNEL | __GFP_DMA); + buffers[i] = kzalloc(req->size, GFP_KERNEL | __GFP_DMA); if (buffers[i] == 0) goto error; sizes[i] = req->size; From 1b03354bd7d24f6c0df152916f4127615f33f57f Mon Sep 17 00:00:00 2001 From: Florian Westphal Date: Tue, 22 Mar 2016 18:02:52 +0100 Subject: [PATCH 143/168] netfilter: x_tables: fix unconditional helper Ben Hawkes says: In the mark_source_chains function (net/ipv4/netfilter/ip_tables.c) it is possible for a user-supplied ipt_entry structure to have a large next_offset field. This field is not bounds checked prior to writing a counter value at the supplied offset. Problem is that mark_source_chains should not have been called -- the rule doesn't have a next entry, so its supposed to return an absolute verdict of either ACCEPT or DROP. However, the function conditional() doesn't work as the name implies. It only checks that the rule is using wildcard address matching. However, an unconditional rule must also not be using any matches (no -m args). The underflow validator only checked the addresses, therefore passing the 'unconditional absolute verdict' test, while mark_source_chains also tested for presence of matches, and thus proceeeded to the next (not-existent) rule. Unify this so that all the callers have same idea of 'unconditional rule'. Change-Id: Icbca80abeff1811180e61195802664220b30853f Reported-by: Ben Hawkes Signed-off-by: Florian Westphal Signed-off-by: Pablo Neira Ayuso Git-commit: 54d83fc74aa9ec72794373cb47432c5f7fb1a309 Git-repo:http://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git Signed-off-by: Ravi Kumar Siddojigari --- net/ipv4/netfilter/arp_tables.c | 18 +++++++++--------- net/ipv4/netfilter/ip_tables.c | 23 +++++++++++------------ net/ipv6/netfilter/ip6_tables.c | 23 +++++++++++------------ 3 files changed, 31 insertions(+), 33 deletions(-) diff --git a/net/ipv4/netfilter/arp_tables.c b/net/ipv4/netfilter/arp_tables.c index cfaaf13285c..ee110f98e3d 100644 --- a/net/ipv4/netfilter/arp_tables.c +++ b/net/ipv4/netfilter/arp_tables.c @@ -355,11 +355,12 @@ unsigned int arpt_do_table(struct sk_buff *skb, } /* All zeroes == unconditional rule. */ -static inline bool unconditional(const struct arpt_arp *arp) +static inline bool unconditional(const struct arpt_entry *e) { static const struct arpt_arp uncond; - return memcmp(arp, &uncond, sizeof(uncond)) == 0; + return e->target_offset == sizeof(struct arpt_entry) && + memcmp(&e->arp, &uncond, sizeof(uncond)) == 0; } /* Figures out from what hook each rule can be called: returns 0 if @@ -398,11 +399,10 @@ static int mark_source_chains(const struct xt_table_info *newinfo, |= ((1 << hook) | (1 << NF_ARP_NUMHOOKS)); /* Unconditional return/END. */ - if ((e->target_offset == sizeof(struct arpt_entry) && + if ((unconditional(e) && (strcmp(t->target.u.user.name, XT_STANDARD_TARGET) == 0) && - t->verdict < 0 && unconditional(&e->arp)) || - visited) { + t->verdict < 0) || visited) { unsigned int oldpos, size; if ((strcmp(t->target.u.user.name, @@ -547,7 +547,7 @@ static bool check_underflow(const struct arpt_entry *e) const struct xt_entry_target *t; unsigned int verdict; - if (!unconditional(&e->arp)) + if (!unconditional(e)) return false; t = arpt_get_target_c(e); if (strcmp(t->u.user.name, XT_STANDARD_TARGET) != 0) @@ -588,9 +588,9 @@ static inline int check_entry_size_and_hooks(struct arpt_entry *e, newinfo->hook_entry[h] = hook_entries[h]; if ((unsigned char *)e - base == underflows[h]) { if (!check_underflow(e)) { - pr_err("Underflows must be unconditional and " - "use the STANDARD target with " - "ACCEPT/DROP\n"); + pr_debug("Underflows must be unconditional and " + "use the STANDARD target with " + "ACCEPT/DROP\n"); return -EINVAL; } newinfo->underflow[h] = underflows[h]; diff --git a/net/ipv4/netfilter/ip_tables.c b/net/ipv4/netfilter/ip_tables.c index ed1468a90b8..9012f7bfba6 100644 --- a/net/ipv4/netfilter/ip_tables.c +++ b/net/ipv4/netfilter/ip_tables.c @@ -168,11 +168,12 @@ get_entry(const void *base, unsigned int offset) /* All zeroes == unconditional rule. */ /* Mildly perf critical (only if packet tracing is on) */ -static inline bool unconditional(const struct ipt_ip *ip) +static inline bool unconditional(const struct ipt_entry *e) { static const struct ipt_ip uncond; - return memcmp(ip, &uncond, sizeof(uncond)) == 0; + return e->target_offset == sizeof(struct ipt_entry) && + memcmp(&e->ip, &uncond, sizeof(uncond)) == 0; #undef FWINV } @@ -230,11 +231,10 @@ get_chainname_rulenum(const struct ipt_entry *s, const struct ipt_entry *e, } else if (s == e) { (*rulenum)++; - if (s->target_offset == sizeof(struct ipt_entry) && + if (unconditional(s) && strcmp(t->target.u.kernel.target->name, XT_STANDARD_TARGET) == 0 && - t->verdict < 0 && - unconditional(&s->ip)) { + t->verdict < 0) { /* Tail of chains: STANDARD target (return/policy) */ *comment = *chainname == hookname ? comments[NF_IP_TRACE_COMMENT_POLICY] @@ -473,11 +473,10 @@ mark_source_chains(const struct xt_table_info *newinfo, e->comefrom |= ((1 << hook) | (1 << NF_INET_NUMHOOKS)); /* Unconditional return/END. */ - if ((e->target_offset == sizeof(struct ipt_entry) && + if ((unconditional(e) && (strcmp(t->target.u.user.name, XT_STANDARD_TARGET) == 0) && - t->verdict < 0 && unconditional(&e->ip)) || - visited) { + t->verdict < 0) || visited) { unsigned int oldpos, size; if ((strcmp(t->target.u.user.name, @@ -710,7 +709,7 @@ static bool check_underflow(const struct ipt_entry *e) const struct xt_entry_target *t; unsigned int verdict; - if (!unconditional(&e->ip)) + if (!unconditional(e)) return false; t = ipt_get_target_c(e); if (strcmp(t->u.user.name, XT_STANDARD_TARGET) != 0) @@ -752,9 +751,9 @@ check_entry_size_and_hooks(struct ipt_entry *e, newinfo->hook_entry[h] = hook_entries[h]; if ((unsigned char *)e - base == underflows[h]) { if (!check_underflow(e)) { - pr_err("Underflows must be unconditional and " - "use the STANDARD target with " - "ACCEPT/DROP\n"); + pr_debug("Underflows must be unconditional and " + "use the STANDARD target with " + "ACCEPT/DROP\n"); return -EINVAL; } newinfo->underflow[h] = underflows[h]; diff --git a/net/ipv6/netfilter/ip6_tables.c b/net/ipv6/netfilter/ip6_tables.c index c30a20cdac8..ab498dd2504 100644 --- a/net/ipv6/netfilter/ip6_tables.c +++ b/net/ipv6/netfilter/ip6_tables.c @@ -195,11 +195,12 @@ get_entry(const void *base, unsigned int offset) /* All zeroes == unconditional rule. */ /* Mildly perf critical (only if packet tracing is on) */ -static inline bool unconditional(const struct ip6t_ip6 *ipv6) +static inline bool unconditional(const struct ip6t_entry *e) { static const struct ip6t_ip6 uncond; - return memcmp(ipv6, &uncond, sizeof(uncond)) == 0; + return e->target_offset == sizeof(struct ip6t_entry) && + memcmp(&e->ipv6, &uncond, sizeof(uncond)) == 0; } static inline const struct xt_entry_target * @@ -256,11 +257,10 @@ get_chainname_rulenum(const struct ip6t_entry *s, const struct ip6t_entry *e, } else if (s == e) { (*rulenum)++; - if (s->target_offset == sizeof(struct ip6t_entry) && + if (unconditional(s) && strcmp(t->target.u.kernel.target->name, XT_STANDARD_TARGET) == 0 && - t->verdict < 0 && - unconditional(&s->ipv6)) { + t->verdict < 0) { /* Tail of chains: STANDARD target (return/policy) */ *comment = *chainname == hookname ? comments[NF_IP6_TRACE_COMMENT_POLICY] @@ -482,11 +482,10 @@ mark_source_chains(const struct xt_table_info *newinfo, e->comefrom |= ((1 << hook) | (1 << NF_INET_NUMHOOKS)); /* Unconditional return/END. */ - if ((e->target_offset == sizeof(struct ip6t_entry) && + if ((unconditional(e) && (strcmp(t->target.u.user.name, XT_STANDARD_TARGET) == 0) && - t->verdict < 0 && - unconditional(&e->ipv6)) || visited) { + t->verdict < 0) || visited) { unsigned int oldpos, size; if ((strcmp(t->target.u.user.name, @@ -720,7 +719,7 @@ static bool check_underflow(const struct ip6t_entry *e) const struct xt_entry_target *t; unsigned int verdict; - if (!unconditional(&e->ipv6)) + if (!unconditional(e)) return false; t = ip6t_get_target_c(e); if (strcmp(t->u.user.name, XT_STANDARD_TARGET) != 0) @@ -762,9 +761,9 @@ check_entry_size_and_hooks(struct ip6t_entry *e, newinfo->hook_entry[h] = hook_entries[h]; if ((unsigned char *)e - base == underflows[h]) { if (!check_underflow(e)) { - pr_err("Underflows must be unconditional and " - "use the STANDARD target with " - "ACCEPT/DROP\n"); + pr_debug("Underflows must be unconditional and " + "use the STANDARD target with " + "ACCEPT/DROP\n"); return -EINVAL; } newinfo->underflow[h] = underflows[h]; From ab6ff8bf3a4d95a4bda7e1234b358aa70c5d0d50 Mon Sep 17 00:00:00 2001 From: Guillaume Nault Date: Wed, 23 Mar 2016 16:38:55 +0100 Subject: [PATCH 144/168] ppp: take reference on channels netns Let channels hold a reference on their network namespace. Some channel types, like ppp_async and ppp_synctty, can have their userspace controller running in a different namespace. Therefore they can't rely on them to preclude their netns from being removed from under them. ================================================================== BUG: KASAN: use-after-free in ppp_unregister_channel+0x372/0x3a0 at addr ffff880064e217e0 Read of size 8 by task syz-executor/11581 ============================================================================= BUG net_namespace (Not tainted): kasan: bad access detected ----------------------------------------------------------------------------- Disabling lock debugging due to kernel taint INFO: Allocated in copy_net_ns+0x6b/0x1a0 age=92569 cpu=3 pid=6906 [< none >] ___slab_alloc+0x4c7/0x500 kernel/mm/slub.c:2440 [< none >] __slab_alloc+0x4c/0x90 kernel/mm/slub.c:2469 [< inline >] slab_alloc_node kernel/mm/slub.c:2532 [< inline >] slab_alloc kernel/mm/slub.c:2574 [< none >] kmem_cache_alloc+0x23a/0x2b0 kernel/mm/slub.c:2579 [< inline >] kmem_cache_zalloc kernel/include/linux/slab.h:597 [< inline >] net_alloc kernel/net/core/net_namespace.c:325 [< none >] copy_net_ns+0x6b/0x1a0 kernel/net/core/net_namespace.c:360 [< none >] create_new_namespaces+0x2f6/0x610 kernel/kernel/nsproxy.c:95 [< none >] copy_namespaces+0x297/0x320 kernel/kernel/nsproxy.c:150 [< none >] copy_process.part.35+0x1bf4/0x5760 kernel/kernel/fork.c:1451 [< inline >] copy_process kernel/kernel/fork.c:1274 [< none >] _do_fork+0x1bc/0xcb0 kernel/kernel/fork.c:1723 [< inline >] SYSC_clone kernel/kernel/fork.c:1832 [< none >] SyS_clone+0x37/0x50 kernel/kernel/fork.c:1826 [< none >] entry_SYSCALL_64_fastpath+0x16/0x7a kernel/arch/x86/entry/entry_64.S:185 INFO: Freed in net_drop_ns+0x67/0x80 age=575 cpu=2 pid=2631 [< none >] __slab_free+0x1fc/0x320 kernel/mm/slub.c:2650 [< inline >] slab_free kernel/mm/slub.c:2805 [< none >] kmem_cache_free+0x2a0/0x330 kernel/mm/slub.c:2814 [< inline >] net_free kernel/net/core/net_namespace.c:341 [< none >] net_drop_ns+0x67/0x80 kernel/net/core/net_namespace.c:348 [< none >] cleanup_net+0x4e5/0x600 kernel/net/core/net_namespace.c:448 [< none >] process_one_work+0x794/0x1440 kernel/kernel/workqueue.c:2036 [< none >] worker_thread+0xdb/0xfc0 kernel/kernel/workqueue.c:2170 [< none >] kthread+0x23f/0x2d0 kernel/drivers/block/aoe/aoecmd.c:1303 [< none >] ret_from_fork+0x3f/0x70 kernel/arch/x86/entry/entry_64.S:468 INFO: Slab 0xffffea0001938800 objects=3 used=0 fp=0xffff880064e20000 flags=0x5fffc0000004080 INFO: Object 0xffff880064e20000 @offset=0 fp=0xffff880064e24200 CPU: 1 PID: 11581 Comm: syz-executor Tainted: G B 4.4.0+ Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.8.2-0-g33fbe13 by qemu-project.org 04/01/2014 00000000ffffffff ffff8800662c7790 ffffffff8292049d ffff88003e36a300 ffff880064e20000 ffff880064e20000 ffff8800662c77c0 ffffffff816f2054 ffff88003e36a300 ffffea0001938800 ffff880064e20000 0000000000000000 Call Trace: [< inline >] __dump_stack kernel/lib/dump_stack.c:15 [] dump_stack+0x6f/0xa2 kernel/lib/dump_stack.c:50 [] print_trailer+0xf4/0x150 kernel/mm/slub.c:654 [] object_err+0x2f/0x40 kernel/mm/slub.c:661 [< inline >] print_address_description kernel/mm/kasan/report.c:138 [] kasan_report_error+0x215/0x530 kernel/mm/kasan/report.c:236 [< inline >] kasan_report kernel/mm/kasan/report.c:259 [] __asan_report_load8_noabort+0x3e/0x40 kernel/mm/kasan/report.c:280 [< inline >] ? ppp_pernet kernel/include/linux/compiler.h:218 [] ? ppp_unregister_channel+0x372/0x3a0 kernel/drivers/net/ppp/ppp_generic.c:2392 [< inline >] ppp_pernet kernel/include/linux/compiler.h:218 [] ppp_unregister_channel+0x372/0x3a0 kernel/drivers/net/ppp/ppp_generic.c:2392 [< inline >] ? ppp_pernet kernel/drivers/net/ppp/ppp_generic.c:293 [] ? ppp_unregister_channel+0xe6/0x3a0 kernel/drivers/net/ppp/ppp_generic.c:2392 [] ppp_asynctty_close+0xa3/0x130 kernel/drivers/net/ppp/ppp_async.c:241 [] ? async_lcp_peek+0x5b0/0x5b0 kernel/drivers/net/ppp/ppp_async.c:1000 [] tty_ldisc_close.isra.1+0x99/0xe0 kernel/drivers/tty/tty_ldisc.c:478 [] tty_ldisc_kill+0x40/0x170 kernel/drivers/tty/tty_ldisc.c:744 [] tty_ldisc_release+0x1b3/0x260 kernel/drivers/tty/tty_ldisc.c:772 [] tty_release+0xac1/0x13e0 kernel/drivers/tty/tty_io.c:1901 [] ? release_tty+0x320/0x320 kernel/drivers/tty/tty_io.c:1688 [] __fput+0x236/0x780 kernel/fs/file_table.c:208 [] ____fput+0x15/0x20 kernel/fs/file_table.c:244 [] task_work_run+0x16b/0x200 kernel/kernel/task_work.c:115 [< inline >] exit_task_work kernel/include/linux/task_work.h:21 [] do_exit+0x8b5/0x2c60 kernel/kernel/exit.c:750 [] ? debug_check_no_locks_freed+0x290/0x290 kernel/kernel/locking/lockdep.c:4123 [] ? mm_update_next_owner+0x6f0/0x6f0 kernel/kernel/exit.c:357 [] ? __dequeue_signal+0x136/0x470 kernel/kernel/signal.c:550 [] ? recalc_sigpending_tsk+0x13b/0x180 kernel/kernel/signal.c:145 [] do_group_exit+0x108/0x330 kernel/kernel/exit.c:880 [] get_signal+0x5e4/0x14f0 kernel/kernel/signal.c:2307 [< inline >] ? kretprobe_table_lock kernel/kernel/kprobes.c:1113 [] ? kprobe_flush_task+0xb5/0x450 kernel/kernel/kprobes.c:1158 [] do_signal+0x83/0x1c90 kernel/arch/x86/kernel/signal.c:712 [] ? recycle_rp_inst+0x310/0x310 kernel/include/linux/list.h:655 [] ? setup_sigcontext+0x780/0x780 kernel/arch/x86/kernel/signal.c:165 [] ? finish_task_switch+0x424/0x5f0 kernel/kernel/sched/core.c:2692 [< inline >] ? finish_lock_switch kernel/kernel/sched/sched.h:1099 [] ? finish_task_switch+0x120/0x5f0 kernel/kernel/sched/core.c:2678 [< inline >] ? context_switch kernel/kernel/sched/core.c:2807 [] ? __schedule+0x919/0x1bd0 kernel/kernel/sched/core.c:3283 [] exit_to_usermode_loop+0xf1/0x1a0 kernel/arch/x86/entry/common.c:247 [< inline >] prepare_exit_to_usermode kernel/arch/x86/entry/common.c:282 [] syscall_return_slowpath+0x19f/0x210 kernel/arch/x86/entry/common.c:344 [] int_ret_from_sys_call+0x25/0x9f kernel/arch/x86/entry/entry_64.S:281 Memory state around the buggy address: ffff880064e21680: fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb ffff880064e21700: fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb >ffff880064e21780: fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb ^ ffff880064e21800: fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb ffff880064e21880: fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb ================================================================== Change-Id: I0f07a51358271a42ec6e1f0d3b2de8913575dfae Fixes: 273ec51dd7ce ("net: ppp_generic - introduce net-namespace functionality v2") Reported-by: Baozeng Ding Signed-off-by: Guillaume Nault Reviewed-by: Cyrill Gorcunov Signed-off-by: David S. Miller Git-repo: http://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git Git-commit: 1f461dcdd296eecedaffffc6bae2bfa90bd7eb89 Signed-off-by: Ravi Kumar Siddojigari --- drivers/net/ppp/ppp_generic.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/net/ppp/ppp_generic.c b/drivers/net/ppp/ppp_generic.c index 21d7151fb0a..e0890534ae3 100644 --- a/drivers/net/ppp/ppp_generic.c +++ b/drivers/net/ppp/ppp_generic.c @@ -2177,7 +2177,7 @@ int ppp_register_net_channel(struct net *net, struct ppp_channel *chan) pch->ppp = NULL; pch->chan = chan; - pch->chan_net = net; + pch->chan_net = get_net(net); chan->ppp = pch; init_ppp_file(&pch->file, CHANNEL); pch->file.hdrlen = chan->hdrlen; @@ -2274,6 +2274,8 @@ ppp_unregister_channel(struct ppp_channel *chan) spin_lock_bh(&pn->all_channels_lock); list_del(&pch->list); spin_unlock_bh(&pn->all_channels_lock); + put_net(pch->chan_net); + pch->chan_net = NULL; pch->file.dead = 1; wake_up_interruptible(&pch->file.rwait); From ac90dc4db224b6bacb65612b75a530ffe608eda9 Mon Sep 17 00:00:00 2001 From: Alan Stern Date: Wed, 16 Dec 2015 13:32:38 -0500 Subject: [PATCH 145/168] USB: fix invalid memory access in hub_activate() Commit 8520f38099cc ("USB: change hub initialization sleeps to delayed_work") changed the hub_activate() routine to make part of it run in a workqueue. However, the commit failed to take a reference to the usb_hub structure or to lock the hub interface while doing so. As a result, if a hub is plugged in and quickly unplugged before the work routine can run, the routine will try to access memory that has been deallocated. Or, if the hub is unplugged while the routine is running, the memory may be deallocated while it is in active use. This patch fixes the problem by taking a reference to the usb_hub at the start of hub_activate() and releasing it at the end (when the work is finished), and by locking the hub interface while the work routine is running. It also adds a check at the start of the routine to see if the hub has already been disconnected, in which nothing should be done. Change-Id: I1e5a67d22a5b61898e5a2aab8ea87b0c08e1130c Signed-off-by: Alan Stern Reported-by: Alexandru Cornea Tested-by: Alexandru Cornea Fixes: 8520f38099cc ("USB: change hub initialization sleeps to delayed_work") CC: Signed-off-by: Greg Kroah-Hartman Git-commit: e50293ef9775c5f1cf3fcc093037dd6a8c5684ea Git-repo: https://git.kernel.org/cgit/linux/kernel/git/stable/linux-stable.git Signed-off-by: Ravi Kumar Siddojigari --- drivers/usb/core/hub.c | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c index 827ac9d6f1e..05d107afbb9 100644 --- a/drivers/usb/core/hub.c +++ b/drivers/usb/core/hub.c @@ -181,6 +181,7 @@ EXPORT_SYMBOL_GPL(ehci_cf_port_reset_rwsem); #define HUB_DEBOUNCE_STABLE 100 +static void hub_release(struct kref *kref); static int usb_reset_and_verify_device(struct usb_device *udev); static inline char *portspeed(struct usb_hub *hub, int portstatus) @@ -754,10 +755,20 @@ static void hub_activate(struct usb_hub *hub, enum hub_activation_type type) unsigned delay; /* Continue a partial initialization */ - if (type == HUB_INIT2) - goto init2; - if (type == HUB_INIT3) + if (type == HUB_INIT2 || type == HUB_INIT3) { + device_lock(hub->intfdev); + + /* Was the hub disconnected while we were waiting? */ + if (hub->disconnected) { + device_unlock(hub->intfdev); + kref_put(&hub->kref, hub_release); + return; + } + if (type == HUB_INIT2) + goto init2; goto init3; + } + kref_get(&hub->kref); /* The superspeed hub except for root hub has to use Hub Depth * value as an offset into the route string to locate the bits @@ -951,6 +962,7 @@ static void hub_activate(struct usb_hub *hub, enum hub_activation_type type) PREPARE_DELAYED_WORK(&hub->init_work, hub_init_func3); schedule_delayed_work(&hub->init_work, msecs_to_jiffies(delay)); + device_unlock(hub->intfdev); return; /* Continues at init3: below */ } else { msleep(delay); @@ -971,6 +983,11 @@ static void hub_activate(struct usb_hub *hub, enum hub_activation_type type) /* Allow autosuspend if it was suppressed */ if (type <= HUB_INIT3) usb_autopm_put_interface_async(to_usb_interface(hub->intfdev)); + + if (type == HUB_INIT2 || type == HUB_INIT3) + device_unlock(hub->intfdev); + + kref_put(&hub->kref, hub_release); } /* Implement the continuations for the delays above */ From c216a1c08b5778135fb2b1a833c23c4162c47f20 Mon Sep 17 00:00:00 2001 From: Kangjie Lu Date: Tue, 19 Jul 2016 13:39:29 +0530 Subject: [PATCH 146/168] USB: usbfs: fix potential infoleak in devio The stack object "ci" has a total size of 8 bytes. Its last 3 bytes are padding bytes which are not initialized and leaked to userland via "copy_to_user". Change-Id: Iedd743ab04eb30b6133fc45bb549fcbae60c12fa Signed-off-by: Kangjie Lu Signed-off-by: Greg Kroah-Hartman Git-commit: 681fef8380eb818c0b845fca5d2ab1dcbab114ee Git-repo: http://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git Signed-off-by: Ravi Kumar Siddojigari --- drivers/usb/core/devio.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/usb/core/devio.c b/drivers/usb/core/devio.c index 8df4b76465a..fe50a70d2ad 100644 --- a/drivers/usb/core/devio.c +++ b/drivers/usb/core/devio.c @@ -1035,10 +1035,11 @@ static int proc_getdriver(struct dev_state *ps, void __user *arg) static int proc_connectinfo(struct dev_state *ps, void __user *arg) { - struct usbdevfs_connectinfo ci = { - .devnum = ps->dev->devnum, - .slow = ps->dev->speed == USB_SPEED_LOW - }; + struct usbdevfs_connectinfo ci; + + memset(&ci, 0, sizeof(ci)); + ci.devnum = ps->dev->devnum; + ci.slow = ps->dev->speed == USB_SPEED_LOW; if (copy_to_user(arg, &ci, sizeof(ci))) return -EFAULT; From dfff3fc5fa9110fcac7aca621c02b7fa7b6a0bc3 Mon Sep 17 00:00:00 2001 From: Peter Hurley Date: Sun, 10 Jan 2016 22:40:55 -0800 Subject: [PATCH 147/168] tty: Fix unsafe ldisc reference via ioctl(TIOCGETD) ioctl(TIOCGETD) retrieves the line discipline id directly from the ldisc because the line discipline id (c_line) in termios is untrustworthy; userspace may have set termios via ioctl(TCSETS*) without actually changing the line discipline via ioctl(TIOCSETD). However, directly accessing the current ldisc via tty->ldisc is unsafe; the ldisc ptr dereferenced may be stale if the line discipline is changing via ioctl(TIOCSETD) or hangup. Wait for the line discipline reference (just like read() or write()) to retrieve the "current" line discipline id. Change-Id: I7b2e474af7e1885a1ca7e277ee7478a3c0ec01a7 Cc: Signed-off-by: Peter Hurley Signed-off-by: Greg Kroah-Hartman Git-Commit:5c17c861a357e9458001f021a7afa7aab9937439 Git-Repo: https://git.kernel.org/cgit/linux/kernel/git/stable/linux-stable.git Signed-off-by: Ravi Kumar Siddojigari --- drivers/tty/tty_io.c | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/drivers/tty/tty_io.c b/drivers/tty/tty_io.c index a2034723717..a20dcb9dca8 100644 --- a/drivers/tty/tty_io.c +++ b/drivers/tty/tty_io.c @@ -2474,6 +2474,28 @@ static int tiocsetd(struct tty_struct *tty, int __user *p) return ret; } +/** + * tiocgetd - get line discipline + * @tty: tty device + * @p: pointer to user data + * + * Retrieves the line discipline id directly from the ldisc. + * + * Locking: waits for ldisc reference (in case the line discipline + * is changing or the tty is being hungup) + */ + +static int tiocgetd(struct tty_struct *tty, int __user *p) +{ + struct tty_ldisc *ld; + int ret; + + ld = tty_ldisc_ref_wait(tty); + ret = put_user(ld->ops->num, p); + tty_ldisc_deref(ld); + return ret; +} + /** * send_break - performed time break * @tty: device to break on @@ -2687,7 +2709,7 @@ long tty_ioctl(struct file *file, unsigned int cmd, unsigned long arg) case TIOCGSID: return tiocgsid(tty, real_tty, p); case TIOCGETD: - return put_user(tty->ldisc->ops->num, (int __user *)p); + return tiocgetd(tty, p); case TIOCSETD: return tiocsetd(tty, p); case TIOCVHANGUP: From 7ab3f84e8dd543264321deb7c7b04dac06853d1b Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Wed, 17 Aug 2016 16:00:08 -0700 Subject: [PATCH 148/168] binder: prevent kptr leak by using %pK format specifier Works in conjunction with kptr_restrict. Bug: 30143283 Change-Id: I2b3ce22f4e206e74614d51453a1d59b7080ab05a Git-repo: https://android.googlesource.com/kernel/msm.git Git-commit: b884cbf06200b18e660514a30293931a61126ef5 Signed-off-by: Dennis Cagle --- drivers/staging/android/binder.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/staging/android/binder.c b/drivers/staging/android/binder.c index aa0754de7f7..f183e47ab0f 100644 --- a/drivers/staging/android/binder.c +++ b/drivers/staging/android/binder.c @@ -3343,7 +3343,7 @@ static void print_binder_node(struct seq_file *m, struct binder_node *node) static void print_binder_ref(struct seq_file *m, struct binder_ref *ref) { - seq_printf(m, " ref %d: desc %d %snode %d s %d w %d d %p\n", + seq_printf(m, " ref %d: desc %d %snode %d s %d w %d d %pK\n", ref->debug_id, ref->desc, ref->node->proc ? "" : "dead ", ref->node->debug_id, ref->strong, ref->weak, ref->death); } From 951440dc0845c561f860704060978d63a69d3f46 Mon Sep 17 00:00:00 2001 From: Walter Yang Date: Wed, 7 Sep 2016 16:28:50 +0800 Subject: [PATCH 149/168] ASoC: msm: initialize the params array before using it The params array is used without initialization, which may cause security issues. Initialize it as all zero after the definition. CRs-Fixed: 1062271 Change-Id: If462fe3d82f139d72547f82dc7eb564f83cb35bf Signed-off-by: Walter Yang --- sound/soc/msm/qdsp6v2/msm-compr-q6-v2.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sound/soc/msm/qdsp6v2/msm-compr-q6-v2.c b/sound/soc/msm/qdsp6v2/msm-compr-q6-v2.c index 5176bbe67e9..8c4c96e69c3 100755 --- a/sound/soc/msm/qdsp6v2/msm-compr-q6-v2.c +++ b/sound/soc/msm/qdsp6v2/msm-compr-q6-v2.c @@ -1031,6 +1031,7 @@ static int msm_compr_ioctl(struct snd_pcm_substream *substream, struct snd_dec_ddp *ddp = &compr->info.codec_param.codec.options.ddp; uint32_t params_length = 0; + memset(params_value, 0, MAX_AC3_PARAM_SIZE); /* check integer overflow */ if (ddp->params_length > UINT_MAX/sizeof(int)) { pr_err("%s: Integer overflow ddp->params_length %d\n", @@ -1075,6 +1076,7 @@ static int msm_compr_ioctl(struct snd_pcm_substream *substream, struct snd_dec_ddp *ddp = &compr->info.codec_param.codec.options.ddp; uint32_t params_length = 0; + memset(params_value, 0, MAX_AC3_PARAM_SIZE); /* check integer overflow */ if (ddp->params_length > UINT_MAX/sizeof(int)) { pr_err("%s: Integer overflow ddp->params_length %d\n", From 789835befc29de6572d8a8058a5621aea22ee84c Mon Sep 17 00:00:00 2001 From: Takashi Iwai Date: Tue, 12 Jan 2016 15:36:27 +0100 Subject: [PATCH 150/168] ALSA: seq: Fix race at timer setup and close ALSA sequencer code has an open race between the timer setup ioctl and the close of the client. This was triggered by syzkaller fuzzer, and a use-after-free was caught there as a result. This patch papers over it by adding a proper queue->timer_mutex lock around the timer-related calls in the relevant code path. Change-Id: I74a0789b15ad8c823adff8652498d95e3304b7a4 Reported-by: Dmitry Vyukov Tested-by: Dmitry Vyukov Cc: Signed-off-by: Takashi Iwai Git-commit: 3567eb6af614dac436c4b16a8d426f9faed639b3 Git-repo: http://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git Signed-off-by: Ravi Kumar Siddojigari --- sound/core/seq/seq_queue.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sound/core/seq/seq_queue.c b/sound/core/seq/seq_queue.c index f9077361c11..4c9aa462de9 100644 --- a/sound/core/seq/seq_queue.c +++ b/sound/core/seq/seq_queue.c @@ -144,8 +144,10 @@ static struct snd_seq_queue *queue_new(int owner, int locked) static void queue_delete(struct snd_seq_queue *q) { /* stop and release the timer */ + mutex_lock(&q->timer_mutex); snd_seq_timer_stop(q->timer); snd_seq_timer_close(q); + mutex_unlock(&q->timer_mutex); /* wait until access free */ snd_use_lock_sync(&q->use_lock); /* release resources... */ From 91ec8cc248db476db12fd30e6b9ec2d6882fe5b0 Mon Sep 17 00:00:00 2001 From: Ben Hutchings Date: Tue, 16 Jun 2015 22:11:06 +0100 Subject: [PATCH 151/168] pipe: iovec: Fix memory corruption when retrying atomic copy as non-atomic pipe_iov_copy_{from,to}_user() may be tried twice with the same iovec, the first time atomically and the second time not. The second attempt needs to continue from the iovec position, pipe buffer offset and remaining length where the first attempt failed, but currently the pipe buffer offset and remaining length are reset. This will corrupt the piped data (possibly also leading to an information leak between processes) and may also corrupt kernel memory. This was fixed upstream by commits f0d1bec9d58d ("new helper: copy_page_from_iter()") and 637b58c2887e ("switch pipe_read() to copy_page_to_iter()"), but those aren't suitable for stable. This fix for older kernel versions was made by Seth Jennings for RHEL and I have extracted it from their update. CVE-2015-1805 Change-Id: I4a18eab0e46f35b1d72f178138d50e1b3eb867cd References: https://bugzilla.redhat.com/show_bug.cgi?id=1202855 Signed-off-by: Ben Hutchings Signed-off-by: Greg Kroah-Hartman Git-repo: http://git.kernel.org/cgit/linux/kernel/git/stable/linux-stable.git Git-commit: 14f81062f365fa9e3839bb2a16862217b71a553c Signed-off-by: Srinivasa Rao Kuppala --- fs/pipe.c | 55 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/fs/pipe.c b/fs/pipe.c index fec5e4ad071..074d15fe3bc 100644 --- a/fs/pipe.c +++ b/fs/pipe.c @@ -104,25 +104,27 @@ void pipe_wait(struct pipe_inode_info *pipe) } static int -pipe_iov_copy_from_user(void *to, struct iovec *iov, unsigned long len, - int atomic) +pipe_iov_copy_from_user(void *addr, int *offset, struct iovec *iov, + size_t *remaining, int atomic) { unsigned long copy; - while (len > 0) { + while (*remaining > 0) { while (!iov->iov_len) iov++; - copy = min_t(unsigned long, len, iov->iov_len); + copy = min_t(unsigned long, *remaining, iov->iov_len); if (atomic) { - if (__copy_from_user_inatomic(to, iov->iov_base, copy)) + if (__copy_from_user_inatomic(addr + *offset, + iov->iov_base, copy)) return -EFAULT; } else { - if (copy_from_user(to, iov->iov_base, copy)) + if (copy_from_user(addr + *offset, + iov->iov_base, copy)) return -EFAULT; } - to += copy; - len -= copy; + *offset += copy; + *remaining -= copy; iov->iov_base += copy; iov->iov_len -= copy; } @@ -130,25 +132,27 @@ pipe_iov_copy_from_user(void *to, struct iovec *iov, unsigned long len, } static int -pipe_iov_copy_to_user(struct iovec *iov, const void *from, unsigned long len, - int atomic) +pipe_iov_copy_to_user(struct iovec *iov, void *addr, int *offset, + size_t *remaining, int atomic) { unsigned long copy; - while (len > 0) { + while (*remaining > 0) { while (!iov->iov_len) iov++; - copy = min_t(unsigned long, len, iov->iov_len); + copy = min_t(unsigned long, *remaining, iov->iov_len); if (atomic) { - if (__copy_to_user_inatomic(iov->iov_base, from, copy)) + if (__copy_to_user_inatomic(iov->iov_base, + addr + *offset, copy)) return -EFAULT; } else { - if (copy_to_user(iov->iov_base, from, copy)) + if (copy_to_user(iov->iov_base, + addr + *offset, copy)) return -EFAULT; } - from += copy; - len -= copy; + *offset += copy; + *remaining -= copy; iov->iov_base += copy; iov->iov_len -= copy; } @@ -384,7 +388,7 @@ pipe_read(struct kiocb *iocb, const struct iovec *_iov, struct pipe_buffer *buf = pipe->bufs + curbuf; const struct pipe_buf_operations *ops = buf->ops; void *addr; - size_t chars = buf->len; + size_t chars = buf->len, remaining; int error, atomic; if (chars > total_len) @@ -398,9 +402,11 @@ pipe_read(struct kiocb *iocb, const struct iovec *_iov, } atomic = !iov_fault_in_pages_write(iov, chars); + remaining = chars; redo: addr = ops->map(pipe, buf, atomic); - error = pipe_iov_copy_to_user(iov, addr + buf->offset, chars, atomic); + error = pipe_iov_copy_to_user(iov, addr, &buf->offset, + &remaining, atomic); ops->unmap(pipe, buf, addr); if (unlikely(error)) { /* @@ -415,7 +421,6 @@ redo: break; } ret += chars; - buf->offset += chars; buf->len -= chars; /* Was it a packet buffer? Clean up and exit */ @@ -522,6 +527,7 @@ pipe_write(struct kiocb *iocb, const struct iovec *_iov, if (ops->can_merge && offset + chars <= PAGE_SIZE) { int error, atomic = 1; void *addr; + size_t remaining = chars; error = ops->confirm(pipe, buf); if (error) @@ -530,8 +536,8 @@ pipe_write(struct kiocb *iocb, const struct iovec *_iov, iov_fault_in_pages_read(iov, chars); redo1: addr = ops->map(pipe, buf, atomic); - error = pipe_iov_copy_from_user(offset + addr, iov, - chars, atomic); + error = pipe_iov_copy_from_user(addr, &offset, iov, + &remaining, atomic); ops->unmap(pipe, buf, addr); ret = error; do_wakeup = 1; @@ -566,6 +572,8 @@ redo1: struct page *page = pipe->tmp_page; char *src; int error, atomic = 1; + int offset = 0; + size_t remaining; if (!page) { page = alloc_page(GFP_HIGHUSER); @@ -586,14 +594,15 @@ redo1: chars = total_len; iov_fault_in_pages_read(iov, chars); + remaining = chars; redo2: if (atomic) src = kmap_atomic(page); else src = kmap(page); - error = pipe_iov_copy_from_user(src, iov, chars, - atomic); + error = pipe_iov_copy_from_user(src, &offset, iov, + &remaining, atomic); if (atomic) kunmap_atomic(src); else From a9366e0bed1c578c93ca518a2a042b892eccaf5f Mon Sep 17 00:00:00 2001 From: Takashi Iwai Date: Wed, 13 Jan 2016 17:48:01 +0100 Subject: [PATCH 152/168] ALSA: timer: Fix race among timer ioctls ALSA timer ioctls have an open race and this may lead to a use-after-free of timer instance object. A simplistic fix is to make each ioctl exclusive. We have already tread_sem for controlling the tread, and extend this as a global mutex to be applied to each ioctl. The downside is, of course, the worse concurrency. But these ioctls aren't to be parallel accessible, in anyway, so it should be fine to serialize there. Change-Id: Ic177ce2a35996d450cd2807aa22a69c8a90b1139 Reported-by: Dmitry Vyukov Tested-by: Dmitry Vyukov Cc: Signed-off-by: Takashi Iwai Git-commit: af368027a49a751d6ff4ee9e3f9961f35bb4fede Git-repo: http://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git Signed-off-by: Ravi Kumar Siddojigari --- sound/core/timer.c | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/sound/core/timer.c b/sound/core/timer.c index 38a137d6b04..1d5461719e3 100644 --- a/sound/core/timer.c +++ b/sound/core/timer.c @@ -73,7 +73,7 @@ struct snd_timer_user { struct timespec tstamp; /* trigger tstamp */ wait_queue_head_t qchange_sleep; struct fasync_struct *fasync; - struct mutex tread_sem; + struct mutex ioctl_lock; }; /* list of timers */ @@ -1266,7 +1266,7 @@ static int snd_timer_user_open(struct inode *inode, struct file *file) return -ENOMEM; spin_lock_init(&tu->qlock); init_waitqueue_head(&tu->qchange_sleep); - mutex_init(&tu->tread_sem); + mutex_init(&tu->ioctl_lock); tu->ticks = 1; tu->queue_size = 128; tu->queue = kmalloc(tu->queue_size * sizeof(struct snd_timer_read), @@ -1286,8 +1286,10 @@ static int snd_timer_user_release(struct inode *inode, struct file *file) if (file->private_data) { tu = file->private_data; file->private_data = NULL; + mutex_lock(&tu->ioctl_lock); if (tu->timeri) snd_timer_close(tu->timeri); + mutex_unlock(&tu->ioctl_lock); kfree(tu->queue); kfree(tu->tqueue); kfree(tu); @@ -1525,7 +1527,6 @@ static int snd_timer_user_tselect(struct file *file, int err = 0; tu = file->private_data; - mutex_lock(&tu->tread_sem); if (tu->timeri) { snd_timer_close(tu->timeri); tu->timeri = NULL; @@ -1569,7 +1570,6 @@ static int snd_timer_user_tselect(struct file *file, } __err: - mutex_unlock(&tu->tread_sem); return err; } @@ -1782,7 +1782,7 @@ enum { SNDRV_TIMER_IOCTL_PAUSE_OLD = _IO('T', 0x23), }; -static long snd_timer_user_ioctl(struct file *file, unsigned int cmd, +static long __snd_timer_user_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { struct snd_timer_user *tu; @@ -1799,17 +1799,11 @@ static long snd_timer_user_ioctl(struct file *file, unsigned int cmd, { int xarg; - mutex_lock(&tu->tread_sem); - if (tu->timeri) { /* too late */ - mutex_unlock(&tu->tread_sem); + if (tu->timeri) /* too late */ return -EBUSY; - } - if (get_user(xarg, p)) { - mutex_unlock(&tu->tread_sem); + if (get_user(xarg, p)) return -EFAULT; - } tu->tread = xarg ? 1 : 0; - mutex_unlock(&tu->tread_sem); return 0; } case SNDRV_TIMER_IOCTL_GINFO: @@ -1842,6 +1836,18 @@ static long snd_timer_user_ioctl(struct file *file, unsigned int cmd, return -ENOTTY; } +static long snd_timer_user_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + struct snd_timer_user *tu = file->private_data; + long ret; + + mutex_lock(&tu->ioctl_lock); + ret = __snd_timer_user_ioctl(file, cmd, arg); + mutex_unlock(&tu->ioctl_lock); + return ret; +} + static int snd_timer_user_fasync(int fd, struct file * file, int on) { struct snd_timer_user *tu; From 531e0aea7876d50fbf8bb7f7498d37fb919afbf9 Mon Sep 17 00:00:00 2001 From: Benjamin Tissoires Date: Tue, 19 Jan 2016 12:34:58 +0100 Subject: [PATCH 153/168] HID: core: prevent out-of-bound readings Plugging a Logitech DJ receiver with KASAN activated raises a bunch of out-of-bound readings. The fields are allocated up to MAX_USAGE, meaning that potentially, we do not have enough fields to fit the incoming values. Add checks and silence KASAN. Change-Id: Iae651de2ab33e72aab6e1f16a462409b96955e36 Signed-off-by: Benjamin Tissoires Signed-off-by: Jiri Kosina Git-repo: https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git Git-commit: 50220dead1650609206efe91f0cc116132d59b3f Signed-off-by: Ravi Kumar Siddojigari --- drivers/hid/hid-core.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c index 4da66b4b977..8e7b22e190d 100644 --- a/drivers/hid/hid-core.c +++ b/drivers/hid/hid-core.c @@ -921,6 +921,7 @@ static void hid_input_field(struct hid_device *hid, struct hid_field *field, /* Ignore report if ErrorRollOver */ if (!(field->flags & HID_MAIN_ITEM_VARIABLE) && value[n] >= min && value[n] <= max && + value[n] - min < field->maxusage && field->usage[value[n] - min].hid == HID_UP_KEYBOARD + 1) goto exit; } @@ -933,11 +934,13 @@ static void hid_input_field(struct hid_device *hid, struct hid_field *field, } if (field->value[n] >= min && field->value[n] <= max + && field->value[n] - min < field->maxusage && field->usage[field->value[n] - min].hid && search(value, field->value[n], count)) hid_process_event(hid, field, &field->usage[field->value[n] - min], 0, interrupt); if (value[n] >= min && value[n] <= max + && value[n] - min < field->maxusage && field->usage[value[n] - min].hid && search(field->value, value[n], count)) hid_process_event(hid, field, &field->usage[value[n] - min], 1, interrupt); From b8d54cca5de362a6ee87d1ca302cb92ccdb4fb85 Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Fri, 1 Jul 2016 00:39:35 -0700 Subject: [PATCH 154/168] block: fix use-after-free in sys_ioprio_get() get_task_ioprio() accesses the task->io_context without holding the task lock and thus can race with exit_io_context(), leading to a use-after-free. The reproducer below hits this within a few seconds on my 4-core QEMU VM: int main(int argc, char **argv) { pid_t pid, child; long nproc, i; /* ioprio_set(IOPRIO_WHO_PROCESS, 0, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0)); */ syscall(SYS_ioprio_set, 1, 0, 0x6000); nproc = sysconf(_SC_NPROCESSORS_ONLN); for (i = 0; i < nproc; i++) { pid = fork(); assert(pid != -1); if (pid == 0) { for (;;) { pid = fork(); assert(pid != -1); if (pid == 0) { _exit(0); } else { child = wait(NULL); assert(child == pid); } } } pid = fork(); assert(pid != -1); if (pid == 0) { for (;;) { /* ioprio_get(IOPRIO_WHO_PGRP, 0); */ syscall(SYS_ioprio_get, 2, 0); } } } for (;;) { /* ioprio_get(IOPRIO_WHO_PGRP, 0); */ syscall(SYS_ioprio_get, 2, 0); } return 0; } This gets us KASAN dumps like this: [ 35.526914] ================================================================== [ 35.530009] BUG: KASAN: out-of-bounds in get_task_ioprio+0x7b/0x90 at addr ffff880066f34e6c [ 35.530009] Read of size 2 by task ioprio-gpf/363 [ 35.530009] ============================================================================= [ 35.530009] BUG blkdev_ioc (Not tainted): kasan: bad access detected [ 35.530009] ----------------------------------------------------------------------------- [ 35.530009] Disabling lock debugging due to kernel taint [ 35.530009] INFO: Allocated in create_task_io_context+0x2b/0x370 age=0 cpu=0 pid=360 [ 35.530009] ___slab_alloc+0x55d/0x5a0 [ 35.530009] __slab_alloc.isra.20+0x2b/0x40 [ 35.530009] kmem_cache_alloc_node+0x84/0x200 [ 35.530009] create_task_io_context+0x2b/0x370 [ 35.530009] get_task_io_context+0x92/0xb0 [ 35.530009] copy_process.part.8+0x5029/0x5660 [ 35.530009] _do_fork+0x155/0x7e0 [ 35.530009] SyS_clone+0x19/0x20 [ 35.530009] do_syscall_64+0x195/0x3a0 [ 35.530009] return_from_SYSCALL_64+0x0/0x6a [ 35.530009] INFO: Freed in put_io_context+0xe7/0x120 age=0 cpu=0 pid=1060 [ 35.530009] __slab_free+0x27b/0x3d0 [ 35.530009] kmem_cache_free+0x1fb/0x220 [ 35.530009] put_io_context+0xe7/0x120 [ 35.530009] put_io_context_active+0x238/0x380 [ 35.530009] exit_io_context+0x66/0x80 [ 35.530009] do_exit+0x158e/0x2b90 [ 35.530009] do_group_exit+0xe5/0x2b0 [ 35.530009] SyS_exit_group+0x1d/0x20 [ 35.530009] entry_SYSCALL_64_fastpath+0x1a/0xa4 [ 35.530009] INFO: Slab 0xffffea00019bcd00 objects=20 used=4 fp=0xffff880066f34ff0 flags=0x1fffe0000004080 [ 35.530009] INFO: Object 0xffff880066f34e58 @offset=3672 fp=0x0000000000000001 [ 35.530009] ================================================================== Fix it by grabbing the task lock while we poke at the io_context. Change-Id: Icacf9fc31e35e8914d0724dbdf0c76954bd5291e Cc: stable@vger.kernel.org Reported-by: Dmitry Vyukov Signed-off-by: Omar Sandoval Signed-off-by: Jens Axboe Git-repo: https://git.kernel.org/cgit/linux/kernel/git/stable/linux-stable.git Git-commit: 8ba8682107ee2ca3347354e018865d8e1967c5f4 Signed-off-by: Ravi Kumar Siddojigari --- fs/ioprio.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fs/ioprio.c b/fs/ioprio.c index 0f1b9515213..d792f569907 100644 --- a/fs/ioprio.c +++ b/fs/ioprio.c @@ -145,8 +145,10 @@ static int get_task_ioprio(struct task_struct *p) if (ret) goto out; ret = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, IOPRIO_NORM); + task_lock(p); if (p->io_context) ret = p->io_context->ioprio; + task_unlock(p); out: return ret; } From ac871687976eefea34ef3f531d1e769d5518f2ee Mon Sep 17 00:00:00 2001 From: Patrick Daly Date: Thu, 28 May 2015 18:32:49 -0700 Subject: [PATCH 155/168] ASoC: wcd9xxx: Fix unprotected userspace access Protect against memory faults while accessing userspace addresses. Change-Id: I1433bac73d24d428749558e530e6869c2e5ee98f Signed-off-by: Patrick Daly --- sound/soc/codecs/wcdcal-hwdep.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sound/soc/codecs/wcdcal-hwdep.c b/sound/soc/codecs/wcdcal-hwdep.c index 1132a3c6eb2..0b77d671d08 100644 --- a/sound/soc/codecs/wcdcal-hwdep.c +++ b/sound/soc/codecs/wcdcal-hwdep.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, The Linux Foundation. All rights reserved. + * Copyright (c) 2014-2015, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -83,7 +83,8 @@ static int wcdcal_hwdep_ioctl_shared(struct snd_hwdep *hw, return -EFAULT; } data = fw[fw_user.cal_type]->data; - memcpy(data, fw_user.buffer, fw_user.size); + if (copy_from_user(data, fw_user.buffer, fw_user.size)) + return -EFAULT; fw[fw_user.cal_type]->size = fw_user.size; mutex_lock(&fw_data->lock); set_bit(WCDCAL_RECIEVED, &fw_data->wcdcal_state[fw_user.cal_type]); From 33582d7ae834c5776837da6d35575b3c1a469ac2 Mon Sep 17 00:00:00 2001 From: Neeraj Soni Date: Mon, 28 Nov 2016 18:23:33 +0530 Subject: [PATCH 156/168] qcrypto: protect potential integer overflow. Adding user passed parameters without check might lead to Integer overflow and unpredictable system behaviour. Change-Id: Iaf8259e3c4a157e1790f1447b1b62a646988b7c4 Signed-off-by: Neeraj Soni --- drivers/crypto/msm/qce50.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/crypto/msm/qce50.c b/drivers/crypto/msm/qce50.c index b93a82fa69c..f4895b7d977 100644 --- a/drivers/crypto/msm/qce50.c +++ b/drivers/crypto/msm/qce50.c @@ -4489,6 +4489,12 @@ int qce_aead_req(void *handle, struct qce_req *q_req) else q_req->cryptlen = areq->cryptlen - authsize; + if ((q_req->cryptlen > UINT_MAX - areq->assoclen) || + (q_req->cryptlen + areq->assoclen > UINT_MAX - ivsize)) { + pr_err("Integer overflow on total aead req length.\n"); + return -EINVAL; + } + totallen = q_req->cryptlen + areq->assoclen + ivsize; if (pce_dev->support_cmd_dscr) { From 9d0cd63300c322988911b95d98101cca22e22238 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Wed, 17 Aug 2016 05:56:26 -0700 Subject: [PATCH 157/168] tcp: fix use after free in tcp_xmit_retransmit_queue() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When tcp_sendmsg() allocates a fresh and empty skb, it puts it at the tail of the write queue using tcp_add_write_queue_tail() Then it attempts to copy user data into this fresh skb. If the copy fails, we undo the work and remove the fresh skb. Unfortunately, this undo lacks the change done to tp->highest_sack and we can leave a dangling pointer (to a freed skb) Later, tcp_xmit_retransmit_queue() can dereference this pointer and access freed memory. For regular kernels where memory is not unmapped, this might cause SACK bugs because tcp_highest_sack_seq() is buggy, returning garbage instead of tp->snd_nxt, but with various debug features like CONFIG_DEBUG_PAGEALLOC, this can crash the kernel. This bug was found by Marco Grassi thanks to syzkaller. Change-Id: I542104d73238bb671d6141cc5bc511831903cc19 Fixes: 6859d49475d4 ("[TCP]: Abstract tp->highest_sack accessing & point to next skb") Reported-by: Marco Grassi Signed-off-by: Eric Dumazet Cc: Ilpo Jäinen Cc: Yuchung Cheng Cc: Neal Cardwell Acked-by: Neal Cardwell Reviewed-by: Cong Wang Signed-off-by: David S. Miller Git-repo: https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git Git-commit: bb1fceca22492109be12640d49f5ea5a544c6bb4 Signed-off-by: Ravi Kumar Siddojigari --- include/net/tcp.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/net/tcp.h b/include/net/tcp.h index 4f093b4f1a7..171ce22c061 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -1306,6 +1306,8 @@ static inline void tcp_check_send_head(struct sock *sk, struct sk_buff *skb_unli { if (sk->sk_send_head == skb_unlinked) sk->sk_send_head = NULL; + if (tcp_sk(sk)->highest_sack == skb_unlinked) + tcp_sk(sk)->highest_sack = NULL; } static inline void tcp_init_send_head(struct sock *sk) From 6874131e7a165acba03ade2cc2c81903c0fa7af5 Mon Sep 17 00:00:00 2001 From: Vegard Nossum Date: Fri, 29 Jul 2016 10:40:31 +0200 Subject: [PATCH 158/168] UPSTREAM: block: fix use-after-free in seq file (cherry picked from commit 77da160530dd1dc94f6ae15a981f24e5f0021e84) I got a KASAN report of use-after-free: ================================================================== BUG: KASAN: use-after-free in klist_iter_exit+0x61/0x70 at addr ffff8800b6581508 Read of size 8 by task trinity-c1/315 ============================================================================= BUG kmalloc-32 (Not tainted): kasan: bad access detected ----------------------------------------------------------------------------- Disabling lock debugging due to kernel taint INFO: Allocated in disk_seqf_start+0x66/0x110 age=144 cpu=1 pid=315 ___slab_alloc+0x4f1/0x520 __slab_alloc.isra.58+0x56/0x80 kmem_cache_alloc_trace+0x260/0x2a0 disk_seqf_start+0x66/0x110 traverse+0x176/0x860 seq_read+0x7e3/0x11a0 proc_reg_read+0xbc/0x180 do_loop_readv_writev+0x134/0x210 do_readv_writev+0x565/0x660 vfs_readv+0x67/0xa0 do_preadv+0x126/0x170 SyS_preadv+0xc/0x10 do_syscall_64+0x1a1/0x460 return_from_SYSCALL_64+0x0/0x6a INFO: Freed in disk_seqf_stop+0x42/0x50 age=160 cpu=1 pid=315 __slab_free+0x17a/0x2c0 kfree+0x20a/0x220 disk_seqf_stop+0x42/0x50 traverse+0x3b5/0x860 seq_read+0x7e3/0x11a0 proc_reg_read+0xbc/0x180 do_loop_readv_writev+0x134/0x210 do_readv_writev+0x565/0x660 vfs_readv+0x67/0xa0 do_preadv+0x126/0x170 SyS_preadv+0xc/0x10 do_syscall_64+0x1a1/0x460 return_from_SYSCALL_64+0x0/0x6a CPU: 1 PID: 315 Comm: trinity-c1 Tainted: G B 4.7.0+ #62 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Ubuntu-1.8.2-1ubuntu1 04/01/2014 ffffea0002d96000 ffff880119b9f918 ffffffff81d6ce81 ffff88011a804480 ffff8800b6581500 ffff880119b9f948 ffffffff8146c7bd ffff88011a804480 ffffea0002d96000 ffff8800b6581500 fffffffffffffff4 ffff880119b9f970 Call Trace: [] dump_stack+0x65/0x84 [] print_trailer+0x10d/0x1a0 [] object_err+0x2f/0x40 [] kasan_report_error+0x221/0x520 [] __asan_report_load8_noabort+0x3e/0x40 [] klist_iter_exit+0x61/0x70 [] class_dev_iter_exit+0x9/0x10 [] disk_seqf_stop+0x3a/0x50 [] seq_read+0x4b2/0x11a0 [] proc_reg_read+0xbc/0x180 [] do_loop_readv_writev+0x134/0x210 [] do_readv_writev+0x565/0x660 [] vfs_readv+0x67/0xa0 [] do_preadv+0x126/0x170 [] SyS_preadv+0xc/0x10 This problem can occur in the following situation: open() - pread() - .seq_start() - iter = kmalloc() // succeeds - seqf->private = iter - .seq_stop() - kfree(seqf->private) - pread() - .seq_start() - iter = kmalloc() // fails - .seq_stop() - class_dev_iter_exit(seqf->private) // boom! old pointer As the comment in disk_seqf_stop() says, stop is called even if start failed, so we need to reinitialise the private pointer to NULL when seq iteration stops. An alternative would be to set the private pointer to NULL when the kmalloc() in disk_seqf_start() fails. Cc: stable@vger.kernel.org Signed-off-by: Vegard Nossum Acked-by: Tejun Heo Signed-off-by: Jens Axboe Change-Id: I07b33f4b38341f60a37806cdd45b0a0c3ab4d84d Bug: 30942273 Git-repo: https://android.googlesource.com/kernel/common.git Git-commit: 300e00132d7aaa907c67ea30d6b6cd80e45eceb4 Signed-off-by: Ravi Kumar Siddojigari --- block/genhd.c | 1 + 1 file changed, 1 insertion(+) diff --git a/block/genhd.c b/block/genhd.c index 53c3f7e212a..ee048a19ce7 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -830,6 +830,7 @@ static void disk_seqf_stop(struct seq_file *seqf, void *v) if (iter) { class_dev_iter_exit(iter); kfree(iter); + seqf->private = NULL; } } From 91d0765f24c5ca13081373cfcc078a2495619a96 Mon Sep 17 00:00:00 2001 From: Florian Westphal Date: Tue, 22 Mar 2016 18:02:50 +0100 Subject: [PATCH 159/168] netfilter: x_tables: make sure e->next_offset covers remaining blob size Otherwise this function may read data beyond the ruleset blob. Signed-off-by: Florian Westphal Signed-off-by: Pablo Neira Ayuso Git-repo: http://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git Git-commit: 6e94e0cfb0887e4013b3b930fa6ab1fe6bb6ba91 Change-Id: I78ac6043166c21e47b83a3742e3bd95c6710c953 Signed-off-by: Ravi Kumar Siddojigari --- net/ipv4/netfilter/arp_tables.c | 6 ++++-- net/ipv4/netfilter/ip_tables.c | 6 ++++-- net/ipv6/netfilter/ip6_tables.c | 6 ++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/net/ipv4/netfilter/arp_tables.c b/net/ipv4/netfilter/arp_tables.c index ee110f98e3d..e6c851dfaf4 100644 --- a/net/ipv4/netfilter/arp_tables.c +++ b/net/ipv4/netfilter/arp_tables.c @@ -568,7 +568,8 @@ static inline int check_entry_size_and_hooks(struct arpt_entry *e, unsigned int h; if ((unsigned long)e % __alignof__(struct arpt_entry) != 0 || - (unsigned char *)e + sizeof(struct arpt_entry) >= limit) { + (unsigned char *)e + sizeof(struct arpt_entry) >= limit || + (unsigned char *)e + e->next_offset > limit) { duprintf("Bad offset %p\n", e); return -EINVAL; } @@ -1222,7 +1223,8 @@ check_compat_entry_size_and_hooks(struct compat_arpt_entry *e, duprintf("check_compat_entry_size_and_hooks %p\n", e); if ((unsigned long)e % __alignof__(struct compat_arpt_entry) != 0 || - (unsigned char *)e + sizeof(struct compat_arpt_entry) >= limit) { + (unsigned char *)e + sizeof(struct compat_arpt_entry) >= limit || + (unsigned char *)e + e->next_offset > limit) { duprintf("Bad offset %p, limit = %p\n", e, limit); return -EINVAL; } diff --git a/net/ipv4/netfilter/ip_tables.c b/net/ipv4/netfilter/ip_tables.c index 9012f7bfba6..0d9cecb0212 100644 --- a/net/ipv4/netfilter/ip_tables.c +++ b/net/ipv4/netfilter/ip_tables.c @@ -731,7 +731,8 @@ check_entry_size_and_hooks(struct ipt_entry *e, unsigned int h; if ((unsigned long)e % __alignof__(struct ipt_entry) != 0 || - (unsigned char *)e + sizeof(struct ipt_entry) >= limit) { + (unsigned char *)e + sizeof(struct ipt_entry) >= limit || + (unsigned char *)e + e->next_offset > limit) { duprintf("Bad offset %p\n", e); return -EINVAL; } @@ -1488,7 +1489,8 @@ check_compat_entry_size_and_hooks(struct compat_ipt_entry *e, duprintf("check_compat_entry_size_and_hooks %p\n", e); if ((unsigned long)e % __alignof__(struct compat_ipt_entry) != 0 || - (unsigned char *)e + sizeof(struct compat_ipt_entry) >= limit) { + (unsigned char *)e + sizeof(struct compat_ipt_entry) >= limit || + (unsigned char *)e + e->next_offset > limit) { duprintf("Bad offset %p, limit = %p\n", e, limit); return -EINVAL; } diff --git a/net/ipv6/netfilter/ip6_tables.c b/net/ipv6/netfilter/ip6_tables.c index ab498dd2504..e18d64335dc 100644 --- a/net/ipv6/netfilter/ip6_tables.c +++ b/net/ipv6/netfilter/ip6_tables.c @@ -741,7 +741,8 @@ check_entry_size_and_hooks(struct ip6t_entry *e, unsigned int h; if ((unsigned long)e % __alignof__(struct ip6t_entry) != 0 || - (unsigned char *)e + sizeof(struct ip6t_entry) >= limit) { + (unsigned char *)e + sizeof(struct ip6t_entry) >= limit || + (unsigned char *)e + e->next_offset > limit) { duprintf("Bad offset %p\n", e); return -EINVAL; } @@ -1499,7 +1500,8 @@ check_compat_entry_size_and_hooks(struct compat_ip6t_entry *e, duprintf("check_compat_entry_size_and_hooks %p\n", e); if ((unsigned long)e % __alignof__(struct compat_ip6t_entry) != 0 || - (unsigned char *)e + sizeof(struct compat_ip6t_entry) >= limit) { + (unsigned char *)e + sizeof(struct compat_ip6t_entry) >= limit || + (unsigned char *)e + e->next_offset > limit) { duprintf("Bad offset %p, limit = %p\n", e, limit); return -EINVAL; } From f1d2f51520063097e9bb4d0337a0188e00838f13 Mon Sep 17 00:00:00 2001 From: Florian Westphal Date: Tue, 22 Mar 2016 18:02:49 +0100 Subject: [PATCH 160/168] netfilter: x_tables: validate e->target_offset early We should check that e->target_offset is sane before mark_source_chains gets called since it will fetch the target entry for loop detection. Signed-off-by: Florian Westphal Signed-off-by: Pablo Neira Ayuso Signed-off-by: Dennis Cagle Git-repo: http://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git Git-commit: bdf533de6968e9686df777dc178486f600c6e617 (cherry picked from commit bdf533de6968e9686df777dc178486f600c6e617) Change-Id: Id3ec56cdc333990d62c99d6c2e59dbcce633bcc1 --- net/ipv4/netfilter/arp_tables.c | 17 ++++++++--------- net/ipv4/netfilter/ip_tables.c | 17 ++++++++--------- net/ipv6/netfilter/ip6_tables.c | 17 ++++++++--------- 3 files changed, 24 insertions(+), 27 deletions(-) diff --git a/net/ipv4/netfilter/arp_tables.c b/net/ipv4/netfilter/arp_tables.c index e6c851dfaf4..e4d9d77b9d5 100644 --- a/net/ipv4/netfilter/arp_tables.c +++ b/net/ipv4/netfilter/arp_tables.c @@ -470,14 +470,12 @@ static int mark_source_chains(const struct xt_table_info *newinfo, return 1; } -static inline int check_entry(const struct arpt_entry *e, const char *name) +static inline int check_entry(const struct arpt_entry *e) { const struct xt_entry_target *t; - if (!arp_checkentry(&e->arp)) { - duprintf("arp_tables: arp check failed %p %s.\n", e, name); + if (!arp_checkentry(&e->arp)) return -EINVAL; - } if (e->target_offset + sizeof(struct xt_entry_target) > e->next_offset) return -EINVAL; @@ -518,10 +516,6 @@ find_check_entry(struct arpt_entry *e, const char *name, unsigned int size) struct xt_target *target; int ret; - ret = check_entry(e, name); - if (ret) - return ret; - t = arpt_get_target(e); target = xt_request_find_target(NFPROTO_ARP, t->u.user.name, t->u.user.revision); @@ -566,6 +560,7 @@ static inline int check_entry_size_and_hooks(struct arpt_entry *e, unsigned int valid_hooks) { unsigned int h; + int err; if ((unsigned long)e % __alignof__(struct arpt_entry) != 0 || (unsigned char *)e + sizeof(struct arpt_entry) >= limit || @@ -581,6 +576,10 @@ static inline int check_entry_size_and_hooks(struct arpt_entry *e, return -EINVAL; } + err = check_entry(e); + if (err) + return err; + /* Check hooks & underflows */ for (h = 0; h < NF_ARP_NUMHOOKS; h++) { if (!(valid_hooks & (1 << h))) @@ -1237,7 +1236,7 @@ check_compat_entry_size_and_hooks(struct compat_arpt_entry *e, } /* For purposes of check_entry casting the compat entry is fine */ - ret = check_entry((struct arpt_entry *)e, name); + ret = check_entry((struct arpt_entry *)e); if (ret) return ret; diff --git a/net/ipv4/netfilter/ip_tables.c b/net/ipv4/netfilter/ip_tables.c index 0d9cecb0212..9b7357467d6 100644 --- a/net/ipv4/netfilter/ip_tables.c +++ b/net/ipv4/netfilter/ip_tables.c @@ -565,14 +565,12 @@ static void cleanup_match(struct xt_entry_match *m, struct net *net) } static int -check_entry(const struct ipt_entry *e, const char *name) +check_entry(const struct ipt_entry *e) { const struct xt_entry_target *t; - if (!ip_checkentry(&e->ip)) { - duprintf("ip check failed %p %s.\n", e, name); + if (!ip_checkentry(&e->ip)) return -EINVAL; - } if (e->target_offset + sizeof(struct xt_entry_target) > e->next_offset) @@ -662,10 +660,6 @@ find_check_entry(struct ipt_entry *e, struct net *net, const char *name, struct xt_mtchk_param mtpar; struct xt_entry_match *ematch; - ret = check_entry(e, name); - if (ret) - return ret; - j = 0; mtpar.net = net; mtpar.table = name; @@ -729,6 +723,7 @@ check_entry_size_and_hooks(struct ipt_entry *e, unsigned int valid_hooks) { unsigned int h; + int err; if ((unsigned long)e % __alignof__(struct ipt_entry) != 0 || (unsigned char *)e + sizeof(struct ipt_entry) >= limit || @@ -744,6 +739,10 @@ check_entry_size_and_hooks(struct ipt_entry *e, return -EINVAL; } + err = check_entry(e); + if (err) + return err; + /* Check hooks & underflows */ for (h = 0; h < NF_INET_NUMHOOKS; h++) { if (!(valid_hooks & (1 << h))) @@ -1503,7 +1502,7 @@ check_compat_entry_size_and_hooks(struct compat_ipt_entry *e, } /* For purposes of check_entry casting the compat entry is fine */ - ret = check_entry((struct ipt_entry *)e, name); + ret = check_entry((struct ipt_entry *)e); if (ret) return ret; diff --git a/net/ipv6/netfilter/ip6_tables.c b/net/ipv6/netfilter/ip6_tables.c index e18d64335dc..66d47e74a4b 100644 --- a/net/ipv6/netfilter/ip6_tables.c +++ b/net/ipv6/netfilter/ip6_tables.c @@ -574,14 +574,12 @@ static void cleanup_match(struct xt_entry_match *m, struct net *net) } static int -check_entry(const struct ip6t_entry *e, const char *name) +check_entry(const struct ip6t_entry *e) { const struct xt_entry_target *t; - if (!ip6_checkentry(&e->ipv6)) { - duprintf("ip_tables: ip check failed %p %s.\n", e, name); + if (!ip6_checkentry(&e->ipv6)) return -EINVAL; - } if (e->target_offset + sizeof(struct xt_entry_target) > e->next_offset) @@ -672,10 +670,6 @@ find_check_entry(struct ip6t_entry *e, struct net *net, const char *name, struct xt_mtchk_param mtpar; struct xt_entry_match *ematch; - ret = check_entry(e, name); - if (ret) - return ret; - j = 0; mtpar.net = net; mtpar.table = name; @@ -739,6 +733,7 @@ check_entry_size_and_hooks(struct ip6t_entry *e, unsigned int valid_hooks) { unsigned int h; + int err; if ((unsigned long)e % __alignof__(struct ip6t_entry) != 0 || (unsigned char *)e + sizeof(struct ip6t_entry) >= limit || @@ -754,6 +749,10 @@ check_entry_size_and_hooks(struct ip6t_entry *e, return -EINVAL; } + err = check_entry(e); + if (err) + return err; + /* Check hooks & underflows */ for (h = 0; h < NF_INET_NUMHOOKS; h++) { if (!(valid_hooks & (1 << h))) @@ -1514,7 +1513,7 @@ check_compat_entry_size_and_hooks(struct compat_ip6t_entry *e, } /* For purposes of check_entry casting the compat entry is fine */ - ret = check_entry((struct ip6t_entry *)e, name); + ret = check_entry((struct ip6t_entry *)e); if (ret) return ret; From ca442c224147b7b0b0a9165882b1af6f9e515d77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arve=20Hj=C3=B8nnev=C3=A5g?= Date: Fri, 12 Aug 2016 16:04:28 -0700 Subject: [PATCH 161/168] ANDROID: binder: Clear binder and cookie when setting handle in flat binder struct MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prevents leaking pointers between processes BUG: 30768347 Change-Id: Id898076926f658a1b8b27a3ccb848756b36de4ca Signed-off-by: Arve Hjønnevåg Git-repo: https://android.googlesource.com/kernel/msm.git Git-commit: 11032d745836280574827bb1db5e64a94945180e Signed-off-by: Dennis Cagle --- drivers/staging/android/binder.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/staging/android/binder.c b/drivers/staging/android/binder.c index f183e47ab0f..8c06ed55191 100644 --- a/drivers/staging/android/binder.c +++ b/drivers/staging/android/binder.c @@ -1657,7 +1657,9 @@ static void binder_transaction(struct binder_proc *proc, fp->type = BINDER_TYPE_HANDLE; else fp->type = BINDER_TYPE_WEAK_HANDLE; + fp->binder = 0; fp->handle = ref->desc; + fp->cookie = 0; binder_inc_ref(ref, fp->type == BINDER_TYPE_HANDLE, &thread->todo); @@ -1700,7 +1702,9 @@ static void binder_transaction(struct binder_proc *proc, return_error = BR_FAILED_REPLY; goto err_binder_get_ref_for_node_failed; } + fp->binder = 0; fp->handle = new_ref->desc; + fp->cookie = 0; binder_inc_ref(new_ref, fp->type == BINDER_TYPE_HANDLE, NULL); binder_debug(BINDER_DEBUG_TRANSACTION, " ref %d desc %d -> ref %d desc %d (node %d)\n", @@ -1749,6 +1753,7 @@ static void binder_transaction(struct binder_proc *proc, binder_debug(BINDER_DEBUG_TRANSACTION, " fd %ld -> %d\n", fp->handle, target_fd); /* TODO: fput? */ + fp->binder = 0; fp->handle = target_fd; } break; From a3c5b67ad682c6f18cad1334c4d56026cfc35172 Mon Sep 17 00:00:00 2001 From: Kangjie Lu Date: Tue, 3 May 2016 16:44:07 -0400 Subject: [PATCH 162/168] ALSA: timer: Fix leak in SNDRV_TIMER_IOCTL_PARAMS The stack object "tread" has a total size of 32 bytes. Its field "event" and "val" both contain 4 bytes padding. These 8 bytes padding bytes are sent to user without being initialized. Change-Id: Ief5bbada3a144447aaecc236f86aa91da968bb3f Signed-off-by: Kangjie Lu Signed-off-by: Takashi Iwai Git-commit: cec8f96e49d9be372fdb0c3836dcf31ec71e457e Git-repo: http://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git Signed-off-by: Ravi Kumar Siddojigari --- sound/core/timer.c | 1 + 1 file changed, 1 insertion(+) diff --git a/sound/core/timer.c b/sound/core/timer.c index 1d5461719e3..7e2bfb0be59 100644 --- a/sound/core/timer.c +++ b/sound/core/timer.c @@ -1683,6 +1683,7 @@ static int snd_timer_user_params(struct file *file, if (tu->timeri->flags & SNDRV_TIMER_IFLG_EARLY_EVENT) { if (tu->tread) { struct snd_timer_tread tread; + memset(&tread, 0, sizeof(tread)); tread.event = SNDRV_TIMER_EVENT_EARLY; tread.tstamp.tv_sec = 0; tread.tstamp.tv_nsec = 0; From e3f33119370954bd966d46828c15d5d0ff31fdb4 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Mon, 28 Dec 2015 20:47:08 -0500 Subject: [PATCH 163/168] [PATCH] arm: fix handling of F_OFD_... in oabi_fcntl64() Change-Id: I4809d4887caf26bcfad505d6a378369b217b5a9e Cc: stable@vger.kernel.org # 3.15+ Reviewed-by: Jeff Layton Signed-off-by: Al Viro Git-repo: https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git Git-commit: 76cc404bfdc0d419c720de4daaf2584542734f42 Signed-off-by: Dennis Cagle --- arch/arm/kernel/sys_oabi-compat.c | 77 ++++++++++++++++--------------- 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/arch/arm/kernel/sys_oabi-compat.c b/arch/arm/kernel/sys_oabi-compat.c index af0aaebf4de..76c2f34e9e6 100644 --- a/arch/arm/kernel/sys_oabi-compat.c +++ b/arch/arm/kernel/sys_oabi-compat.c @@ -193,52 +193,53 @@ struct oabi_flock64 { pid_t l_pid; } __attribute__ ((packed,aligned(4))); +static long do_locks(unsigned int fd, unsigned int cmd, + unsigned long arg) +{ + struct flock64 kernel; + struct oabi_flock64 user; + mm_segment_t fs; + long ret; + + if (copy_from_user(&user, (struct oabi_flock64 __user *)arg, + sizeof(user))) + return -EFAULT; + kernel.l_type = user.l_type; + kernel.l_whence = user.l_whence; + kernel.l_start = user.l_start; + kernel.l_len = user.l_len; + kernel.l_pid = user.l_pid; + + fs = get_fs(); + set_fs(KERNEL_DS); + ret = sys_fcntl64(fd, cmd, (unsigned long)&kernel); + set_fs(fs); + + if (!ret && (cmd == F_GETLK64 || cmd == F_OFD_GETLK)) { + user.l_type = kernel.l_type; + user.l_whence = kernel.l_whence; + user.l_start = kernel.l_start; + user.l_len = kernel.l_len; + user.l_pid = kernel.l_pid; + if (copy_to_user((struct oabi_flock64 __user *)arg, + &user, sizeof(user))) + ret = -EFAULT; + } + return ret; +} + asmlinkage long sys_oabi_fcntl64(unsigned int fd, unsigned int cmd, unsigned long arg) { - struct oabi_flock64 user; - struct flock64 kernel; - mm_segment_t fs = USER_DS; /* initialized to kill a warning */ - unsigned long local_arg = arg; - int ret; - switch (cmd) { case F_GETLK64: case F_SETLK64: case F_SETLKW64: - if (copy_from_user(&user, (struct oabi_flock64 __user *)arg, - sizeof(user))) - return -EFAULT; - kernel.l_type = user.l_type; - kernel.l_whence = user.l_whence; - kernel.l_start = user.l_start; - kernel.l_len = user.l_len; - kernel.l_pid = user.l_pid; - local_arg = (unsigned long)&kernel; - fs = get_fs(); - set_fs(KERNEL_DS); + return do_locks(fd, cmd, arg); + + default: + return sys_fcntl64(fd, cmd, arg); } - - ret = sys_fcntl64(fd, cmd, local_arg); - - switch (cmd) { - case F_GETLK64: - if (!ret) { - user.l_type = kernel.l_type; - user.l_whence = kernel.l_whence; - user.l_start = kernel.l_start; - user.l_len = kernel.l_len; - user.l_pid = kernel.l_pid; - if (copy_to_user((struct oabi_flock64 __user *)arg, - &user, sizeof(user))) - ret = -EFAULT; - } - case F_SETLK64: - case F_SETLKW64: - set_fs(fs); - } - - return ret; } struct oabi_epoll_event { From 07c021203e6ba4ee7f8adee4d72798491612a7de Mon Sep 17 00:00:00 2001 From: VijayaKumar T M Date: Fri, 4 Nov 2016 17:05:50 +0530 Subject: [PATCH 164/168] msm: camera: ispif: Add missing mutex usage in ioctl Mutex usage is added into the ispif ioctl path to ensure these paths are thread safe. CRs-Fixed: 1074310 Change-Id: Id718f83484bc4acf98ade0205328aad6ee306270 Signed-off-by: VijayaKumar T M --- drivers/media/platform/msm/camera_v2/ispif/msm_ispif.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/media/platform/msm/camera_v2/ispif/msm_ispif.c b/drivers/media/platform/msm/camera_v2/ispif/msm_ispif.c index c73127b4a51..d6bd6984dfa 100755 --- a/drivers/media/platform/msm/camera_v2/ispif/msm_ispif.c +++ b/drivers/media/platform/msm/camera_v2/ispif/msm_ispif.c @@ -1118,8 +1118,11 @@ static long msm_ispif_subdev_ioctl(struct v4l2_subdev *sd, case MSM_SD_SHUTDOWN: { struct ispif_device *ispif = (struct ispif_device *)v4l2_get_subdevdata(sd); - if (ispif && ispif->base) + if (ispif && ispif->base) { + mutex_lock(&ispif->mutex); msm_ispif_release(ispif); + mutex_unlock(&ispif->mutex); + } return 0; } default: From 0d582ec9227e73f84ae7c0de1fd0276f97c3cdab Mon Sep 17 00:00:00 2001 From: Kangjie Lu Date: Tue, 3 May 2016 16:44:20 -0400 Subject: [PATCH 165/168] ALSA: timer: Fix leak in events via snd_timer_user_ccallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stack object “r1” has a total size of 32 bytes. Its field “event” and “val” both contain 4 bytes padding. These 8 bytes padding bytes are sent to user without being initialized. b/28980217 Git-commit: 9a47e9cff994f37f7f0dbd9ae23740d0f64f9fe6 Git-repo: http://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git Signed-off-by: Kangjie Lu Signed-off-by: Takashi Iwai Signed-off-by: Dennis Cagle (cherry picked from commit 9a47e9cff994f37f7f0dbd9ae23740d0f64f9fe6) Change-Id: I12949efac2aba669d302908704005fb94ba7efd7 --- sound/core/timer.c | 1 + 1 file changed, 1 insertion(+) diff --git a/sound/core/timer.c b/sound/core/timer.c index 7e2bfb0be59..83ef6771e80 100644 --- a/sound/core/timer.c +++ b/sound/core/timer.c @@ -1184,6 +1184,7 @@ static void snd_timer_user_ccallback(struct snd_timer_instance *timeri, tu->tstamp = *tstamp; if ((tu->filter & (1 << event)) == 0 || !tu->tread) return; + memset(&r1, 0, sizeof(r1)); r1.event = event; r1.tstamp = *tstamp; r1.val = resolution; From 1e9325f5e527df994cb3dbd9ac7655fb3633722f Mon Sep 17 00:00:00 2001 From: Kangjie Lu Date: Tue, 3 May 2016 16:44:32 -0400 Subject: [PATCH 166/168] ALSA: timer: Fix leak in events via snd_timer_user_tinterrupt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stack object “r1” has a total size of 32 bytes. Its field “event” and “val” both contain 4 bytes padding. These 8 bytes padding bytes are sent to user without being initialized. b/28980217 Git-commit: e4ec8cc8039a7063e24204299b462bd1383184a5 Git-repo: http://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git Signed-off-by: Kangjie Lu Signed-off-by: Takashi Iwai Signed-off-by: Dennis Cagle (cherry picked from commit e4ec8cc8039a7063e24204299b462bd1383184a5) Change-Id: I53aa15632e941199010aae670cefb65c8fd56833 --- sound/core/timer.c | 1 + 1 file changed, 1 insertion(+) diff --git a/sound/core/timer.c b/sound/core/timer.c index 83ef6771e80..1a21148c21b 100644 --- a/sound/core/timer.c +++ b/sound/core/timer.c @@ -1219,6 +1219,7 @@ static void snd_timer_user_tinterrupt(struct snd_timer_instance *timeri, } if ((tu->filter & (1 << SNDRV_TIMER_EVENT_RESOLUTION)) && tu->last_resolution != resolution) { + memset(&r1, 0, sizeof(r1)); r1.event = SNDRV_TIMER_EVENT_RESOLUTION; r1.tstamp = tstamp; r1.val = resolution; From 8242064d85226d9c0f46e064ac18698eebe6dc5a Mon Sep 17 00:00:00 2001 From: Bharat Pawar Date: Wed, 1 Feb 2017 16:00:36 +0530 Subject: [PATCH 167/168] Revert "spmi: prevent showing the address of spmidev" This reverts commit 2e8308128aad2de683a1e3db9c34044687dd8efe. The above change, although stops displaying the address, fails to create devices with the same name on the same slave id. That causes failures to register devices like qpnp-pin and qpnp-regulator that have multiple instances on the same slave id. Change-Id: I5bad1eacbc588f0d2572099b13bbcb08a66efe52 Signed-off-by: Bharat Pawar --- drivers/spmi/spmi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/spmi/spmi.c b/drivers/spmi/spmi.c index 113c3291914..37abd26a14d 100644 --- a/drivers/spmi/spmi.c +++ b/drivers/spmi/spmi.c @@ -252,7 +252,7 @@ int spmi_add_device(struct spmi_device *spmidev) } /* Set the device name */ - dev_set_name(dev, "%s-%02d", spmidev->name, spmidev->sid); + dev_set_name(dev, "%s-%p", spmidev->name, spmidev); /* Device may be bound to an active driver when this returns */ rc = device_add(dev); From 804e9ca7214f60deead9eca3b573c497b66aad73 Mon Sep 17 00:00:00 2001 From: Abhijeet Dharmapurikar Date: Wed, 15 Jun 2016 09:46:21 -0700 Subject: [PATCH 168/168] spmi: prevent showing the address of spmidev Creating devices with the address of the container spmidev is not indicative of the actual hardware device it represents. Instead use an unique id to indicate the device it represents. CRs-Fixed: 1024197 Change-Id: Id18e2a19f4fa1249901a3f275defa8f589270d69 Signed-off-by: Abhijeet Dharmapurikar --- drivers/spmi/spmi.c | 18 +++++++++++++++--- include/linux/spmi.h | 6 +++++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/drivers/spmi/spmi.c b/drivers/spmi/spmi.c index 37abd26a14d..18faa12b790 100644 --- a/drivers/spmi/spmi.c +++ b/drivers/spmi/spmi.c @@ -32,6 +32,7 @@ struct spmii_boardinfo { static DEFINE_MUTEX(board_lock); static LIST_HEAD(board_list); static DEFINE_IDR(ctrl_idr); +static DEFINE_IDA(spmi_devid_ida); static struct device_type spmi_dev_type; static struct device_type spmi_ctrl_type; @@ -245,22 +246,32 @@ int spmi_add_device(struct spmi_device *spmidev) { int rc; struct device *dev = get_valid_device(spmidev); + int id; if (!dev) { pr_err("invalid SPMI device\n"); return -EINVAL; } + id = ida_simple_get(&spmi_devid_ida, 0, 0, GFP_KERNEL); + if (id < 0) { + pr_err("No id available status = %d\n", id); + return id; + } + /* Set the device name */ - dev_set_name(dev, "%s-%p", spmidev->name, spmidev); + spmidev->id = id; + dev_set_name(dev, "%s-%d", spmidev->name, spmidev->id); /* Device may be bound to an active driver when this returns */ rc = device_add(dev); - if (rc < 0) + if (rc < 0) { + ida_simple_remove(&spmi_devid_ida, spmidev->id); dev_err(dev, "Can't add %s, status %d\n", dev_name(dev), rc); - else + } else { dev_dbg(dev, "device %s registered\n", dev_name(dev)); + } return rc; } @@ -308,6 +319,7 @@ EXPORT_SYMBOL_GPL(spmi_new_device); void spmi_remove_device(struct spmi_device *spmi_dev) { device_unregister(&spmi_dev->dev); + ida_simple_remove(&spmi_devid_ida, spmi_dev->id); } EXPORT_SYMBOL_GPL(spmi_remove_device); diff --git a/include/linux/spmi.h b/include/linux/spmi.h index b581de80bba..5a8525d7694 100644 --- a/include/linux/spmi.h +++ b/include/linux/spmi.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2014 The Linux Foundation. All rights reserved. +/* Copyright (c) 2012-2014, 2016 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -120,6 +120,9 @@ struct spmi_resource { * @dev_node: array of SPMI resources when used with spmi-dev-container. * @num_dev_node: number of device_node structures. * @sid: Slave Identifier. + * @id: Unique identifier to differentiate from other spmi devices with + * possibly same name. + * */ struct spmi_device { struct device dev; @@ -129,6 +132,7 @@ struct spmi_device { struct spmi_resource *dev_node; u32 num_dev_node; u8 sid; + int id; }; #define to_spmi_device(d) container_of(d, struct spmi_device, dev)