liuyu/pgvector
1
0
Fork 0
mirror of https://github.com/pgvector/pgvector.git synced 2026-07-14 16:46:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fixed rounding [skip ci]

Browse Source
This commit is contained in:
Andrew Kane
2024-09-24 00:24:02 -07:00
parent 724bde5020
commit 3b0976b725
1 changed files with 5 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/minivec.h
View File

@@ -82,14 +82,14 @@ Float4ToFp8Unchecked(float num)
/* NaN */ /* NaN */
result |= 0x7F; result |= 0x7F;
} }
else if (exponent > 116) else if (exponent > 114)
{ {
int m; int m;
int gr; int gr;
int s; int s;
exponent -= 127; exponent -= 127;
s = mantissa & 0x000FFFFF; s = mantissa & 0x0007FFFF;
/* Subnormal */ /* Subnormal */
if (exponent < -6) if (exponent < -6)
@@ -98,7 +98,7 @@ Float4ToFp8Unchecked(float num)
mantissa >>= diff; mantissa >>= diff;
mantissa += 1 << (23 - diff); mantissa += 1 << (23 - diff);
s |= mantissa & 0x000FFFFF; s |= mantissa & 0x0007FFFF;
} }
m = mantissa >> 20; m = mantissa >> 20;
@@ -114,14 +114,14 @@ Float4ToFp8Unchecked(float num)
exponent += 1; exponent += 1;
} }
if (exponent > 8) if (exponent > 7)
{ {
/* Infinite, which is NaN */ /* Infinite, which is NaN */
result |= 0x7F; result |= 0x7F;
} }
else else
{ {
if (exponent >= -7) if (exponent >= -6)
result |= (exponent + 7) << 3; result |= (exponent + 7) << 3;
result |= m; result |= m;