fix fillna bug (#1914)

* fix fillna bug

* fix flake8 error

* fix pylint error

* update ubuntu version for action

* fix pytest error

* fix pylint error

* fix black error

* fix pylint error

* add Fillna test

* fix black error

* add  instruments

* remove code

.github/workflows/test_qlib_from_pip.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [windows-latest, ubuntu-20.04, ubuntu-22.04, macos-13, macos-14, macos-15]
+        os: [windows-latest, ubuntu-24.04, ubuntu-22.04, macos-13, macos-14, macos-15]
         # In github action, using python 3.7, pip install will not match the latest version of the package.
         # Also, python 3.7 is no longer supported from macos-14, and will be phased out from macos-13 in the near future.
         # All things considered, we have removed python 3.7.

.github/workflows/test_qlib_from_source.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [windows-latest, ubuntu-20.04, ubuntu-22.04, macos-13, macos-14, macos-15]
+        os: [windows-latest, ubuntu-24.04, ubuntu-22.04, macos-13, macos-14, macos-15]
         # In github action, using python 3.7, pip install will not match the latest version of the package.
         # Also, python 3.7 is no longer supported from macos-14, and will be phased out from macos-13 in the near future.
         # All things considered, we have removed python 3.7.
@@ -39,7 +39,7 @@ jobs:
         python -m pip install torch torchvision torchaudio
 
     - name: Installing pytorch for ubuntu
-      if: ${{ matrix.os == 'ubuntu-20.04' || matrix.os == 'ubuntu-22.04' }}
+      if: ${{ matrix.os == 'ubuntu-24.04' || matrix.os == 'ubuntu-22.04' }}
       run: |
         python -m pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu
 

.github/workflows/test_qlib_from_source_slow.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [windows-latest, ubuntu-20.04, ubuntu-22.04, macos-13, macos-14, macos-15]
+        os: [windows-latest, ubuntu-24.04, ubuntu-22.04, macos-13, macos-14, macos-15]
         # In github action, using python 3.7, pip install will not match the latest version of the package.
         # Also, python 3.7 is no longer supported from macos-14, and will be phased out from macos-13 in the near future.
         # All things considered, we have removed python 3.7.

pyproject.toml

@@ -23,9 +23,6 @@ description = "A Quantitative-research Platform"
 requires-python = ">=3.8.0"
 readme = {file = "README.md", content-type = "text/markdown"}
 
-# On 2025-04-02 osqp released version 1.0.2, osqp is used as a dependency for cvxpy. 
-# It would lead to errors installing qlib, so we limited the version of osqp.
-# refs: https://github.com/osqp/osqp/issues/728
 dependencies = [
   "pyyaml",
   "numpy",
@@ -42,7 +39,6 @@ dependencies = [
   "loguru",
   "lightgbm",
   "gym",
-  "osqp<1.0.2",
   "cvxpy",
   "joblib",
   "matplotlib",

qlib/contrib/model/pytorch_nn.py

@@ -146,8 +146,11 @@ class DNNModelPytorch(Model):
             raise NotImplementedError("optimizer {} is not supported!".format(optimizer))
 
         if scheduler == "default":
+            # In torch version 2.7.0, the verbose parameter has been removed. Reference Link:
+            # https://github.com/pytorch/pytorch/pull/147301/files#diff-036a7470d5307f13c9a6a51c3a65dd014f00ca02f476c545488cd856bea9bcf2L1313
+            if str(torch.__version__).split("+", maxsplit=1)[0] <= "2.6.0":
                 # Reduce learning rate when loss has stopped decrease
-            self.scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
+                self.scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(  # pylint: disable=E1123
                     self.train_optimizer,
                     mode="min",
                     factor=0.5,
@@ -159,6 +162,18 @@ class DNNModelPytorch(Model):
                     min_lr=0.00001,
                     eps=1e-08,
                 )
+            else:
+                self.scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
+                    self.train_optimizer,
+                    mode="min",
+                    factor=0.5,
+                    patience=10,
+                    threshold=0.0001,
+                    threshold_mode="rel",
+                    cooldown=0,
+                    min_lr=0.00001,
+                    eps=1e-08,
+                )
         elif scheduler is None:
             self.scheduler = None
         else:

qlib/data/dataset/processor.py

@@ -187,14 +187,9 @@ class Fillna(Processor):
         if self.fields_group is None:
             df.fillna(self.fill_value, inplace=True)
         else:
-            cols = get_group_columns(df, self.fields_group)
             # this implementation is extremely slow
             # df.fillna({col: self.fill_value for col in cols}, inplace=True)
-
+            df[self.fields_group] = df[self.fields_group].fillna(self.fill_value)
-            # So we use numpy to accelerate filling values
-            nan_select = np.isnan(df.values)
-            nan_select[:, ~df.columns.isin(cols)] = False
-            df.values[nan_select] = self.fill_value
         return df
 
 

tests/data_mid_layer_tests/test_dataloader.py

@@ -10,6 +10,7 @@ sys.path.append(str(Path(__file__).resolve().parent))
 from qlib.data.dataset.loader import NestedDataLoader, QlibDataLoader
 from qlib.data.dataset.handler import DataHandlerLP
 from qlib.contrib.data.loader import Alpha158DL, Alpha360DL
+from qlib.data.dataset.processor import Fillna
 from qlib.data import D
 
 
@@ -30,7 +31,7 @@ class TestDataLoader(unittest.TestCase):
         )
         # Of course you can use StaticDataLoader
 
-        dataset = nd.load(start_time="2020-01-01", end_time="2020-01-31")
+        dataset = nd.load(instruments="csi300", start_time="2020-01-01", end_time="2020-01-31")
 
         assert dataset is not None
 
@@ -45,6 +46,13 @@ class TestDataLoader(unittest.TestCase):
 
         assert "LABEL0" in columns_list
 
+        assert dataset.isna().any().any()
+
+        fn = Fillna(fields_group="feature", fill_value=0)
+        fn_dataset = fn.__call__(dataset)
+
+        assert not fn_dataset.isna().any().any()
+
         # Then you can use it wth DataHandler;
         # NOTE: please note that the data processors are missing!!!  You should add based on your requirements