diff --git a/src/hnswscan.c b/src/hnswscan.c
index 4ba29cf..c21175a 100644
--- a/src/hnswscan.c
+++ b/src/hnswscan.c
@@ -23,6 +23,7 @@ GetScanItems(IndexScanDesc scan, Datum q)
 	int			m;
 	HnswElement entryPoint;
 	char	   *base = NULL;
+	instr_time *start = hnsw_time_budget == -1 ? NULL : &so->start;
 
 	/* Get m and entry point */
 	HnswGetMetaPageInfo(index, &m, &entryPoint);
@@ -34,11 +35,11 @@ GetScanItems(IndexScanDesc scan, Datum q)
 
 	for (int lc = entryPoint->level; lc >= 1; lc--)
 	{
-		w = HnswSearchLayer(base, q, ep, 1, lc, index, procinfo, collation, m, false, NULL, &so->start);
+		w = HnswSearchLayer(base, q, ep, 1, lc, index, procinfo, collation, m, false, NULL, start);
 		ep = w;
 	}
 
-	return HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, procinfo, collation, m, false, NULL, &so->start);
+	return HnswSearchLayer(base, q, ep, hnsw_ef_search, 0, index, procinfo, collation, m, false, NULL, start);
 }
 
 /*
diff --git a/src/hnswutils.c b/src/hnswutils.c
index cb46446..ead40a4 100644
--- a/src/hnswutils.c
+++ b/src/hnswutils.c
@@ -852,7 +852,7 @@ HnswSearchLayer(char *base, Datum q, List *ep, int ef, int lc, Relation index, F
 		HnswElement cElement;
 
 		/* Check time budget */
-		if (hnsw_time_budget != -1)
+		if (start != NULL)
 		{
 			instr_time	duration;