liuyu/pgvector
1
0
Fork 0
mirror of https://github.com/pgvector/pgvector.git synced 2026-07-03 11:10:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fixed rounding [skip ci]

Browse Source
This commit is contained in:
Andrew Kane
2024-09-24 00:24:02 -07:00
parent 724bde5020
commit 3b0976b725
1 changed files with 5 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/minivec.h
View File

@@ -82,14 +82,14 @@ Float4ToFp8Unchecked(float num)
/* NaN */
result |= 0x7F;
}
else if (exponent > 116)
else if (exponent > 114)
{
int m;
int gr;
int s;
exponent -= 127;
s = mantissa & 0x000FFFFF;
s = mantissa & 0x0007FFFF;
/* Subnormal */
if (exponent < -6)
@@ -98,7 +98,7 @@ Float4ToFp8Unchecked(float num)
mantissa >>= diff;
mantissa += 1 << (23 - diff);
s |= mantissa & 0x000FFFFF;
s |= mantissa & 0x0007FFFF;
}
m = mantissa >> 20;
@@ -114,14 +114,14 @@ Float4ToFp8Unchecked(float num)
exponent += 1;
}
if (exponent > 8)
if (exponent > 7)
{
/* Infinite, which is NaN */
result |= 0x7F;
}
else
{
if (exponent >= -7)
if (exponent >= -6)
result |= (exponent + 7) << 3;
result |= m;