commit 05a62c0a0de98a6e0bc16687da301f02cb64906b
author Brennan Shacklett <bshacklett@mozilla.com> Thu Aug 11 15:55:52 2016 -0700
committer Brennan Shacklett <bshacklett@mozilla.com> Fri Aug 12 16:54:57 2016 +0000
tree 7b12568ccba599d139a0660e804bf660da7e389b
parent de2ba4f658470cb77b7463d6b7e9396b81367ff4

Fix eobs buffer overflow caused by SSSE3 quantize_fp

The SSSE3 version of av1_quantize_fp stored the new value of eob
with a 32bit mov instruction, but the eob values are stored in an
array of uint16_t. This caused the last eob store to write 2 extra
bytes off the end of the eob array.

This doesn't seem to currently be a problem on master, but it
causes errors for my RDO refactoring under certain conditions.

Change-Id: I9f7e9c4e40b37b8a2c232b87d170d24593c1066c

av1/encoder/x86/quantize_ssse3_x86_64.asm

diff --git a/av1/encoder/x86/quantize_ssse3_x86_64.asm b/av1/encoder/x86/quantize_ssse3_x86_64.asm
index ad4ae27..3a40a44 100644
--- a/av1/encoder/x86/quantize_ssse3_x86_64.asm
+++ b/av1/encoder/x86/quantize_ssse3_x86_64.asm
@@ -174,7 +174,7 @@
   pshuflw                         m7, m8, 0x1
   pmaxsw                          m8, m7
   pextrw                          r6, m8, 0
-  mov                           [r2], r6
+  mov                           [r2], r6w
   RET
 
   ; skip-block, i.e. just write all zeroes