From 50d74b5560d06bdea2626f9ceb2c428f51655d76 Mon Sep 17 00:00:00 2001 From: Linlang <30293408+SunsetWolf@users.noreply.github.com> Date: Tue, 28 Jun 2022 10:17:29 +0800 Subject: [PATCH] split_CI (#1141) --- .github/workflows/test_macos.yml | 94 ------------------- .github/workflows/test_qlib_from_pip.yml | 46 +++++++++ .../{test.yml => test_qlib_from_source.yml} | 87 +++++++++-------- .../workflows/test_qlib_from_source_slow.yml | 52 ++++++++++ qlib/data/storage/file_storage.py | 5 +- setup.py | 16 +++- tests/pytest.ini | 2 + tests/rolling_tests/test_update_pred.py | 3 + tests/test_all_pipeline.py | 5 + tests/test_contrib_workflow.py | 3 + tests/test_dataset.py | 2 + tests/test_pit.py | 4 +- 12 files changed, 175 insertions(+), 144 deletions(-) delete mode 100644 .github/workflows/test_macos.yml create mode 100644 .github/workflows/test_qlib_from_pip.yml rename .github/workflows/{test.yml => test_qlib_from_source.yml} (70%) create mode 100644 .github/workflows/test_qlib_from_source_slow.yml diff --git a/.github/workflows/test_macos.yml b/.github/workflows/test_macos.yml deleted file mode 100644 index 9ba7af30b..000000000 --- a/.github/workflows/test_macos.yml +++ /dev/null @@ -1,94 +0,0 @@ -# There are some issues (in the downloading data phase) on MacOS when running with other tests. So we split it into an individual config. -name: Test MacOS - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - build: - timeout-minutes: 120 - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [macos-11, macos-latest] - # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129 - python-version: [3.7, 3.8] - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Lint with Black - run: | - cd .. - python -m pip install pip --upgrade - python -m pip install wheel --upgrade - python -m pip install black - python -m black qlib -l 120 --check --diff - # Test Qlib installed with pip - - - name: Check Qlib with flake8 - run: | - pip install --upgrade pip - pip install flake8 - flake8 --ignore=E501,F541,E266,E402,W503,E731,E203 --per-file-ignores="__init__.py:F401,F403" qlib - - - name: Install Qlib with pip - run: | - python -m pip install numpy==1.19.5 - python -m pip install pyqlib --ignore-installed ruamel.yaml numpy - - name: Make html with sphnix - run: | - pip install -U sphinx - pip install sphinx_rtd_theme readthedocs_sphinx_ext - pip install --exists-action=w --no-cache-dir -r docs/requirements.txt - cd docs - sphinx-build -b html . build - cd .. - - name: Install Lightgbm for MacOS - run: | - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Microsoft/qlib/main/.github/brew_install.sh)" - HOMEBREW_NO_AUTO_UPDATE=1 brew install lightgbm - # FIX MacOS error: Segmentation fault - # reference: https://github.com/microsoft/LightGBM/issues/4229 - wget https://raw.githubusercontent.com/Homebrew/homebrew-core/fb8323f2b170bd4ae97e1bac9bf3e2983af3fdb0/Formula/libomp.rb - brew unlink libomp - brew install libomp.rb - - name: Test data downloads - run: | - python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data_simple --interval 1d --region cn - python -c "import os; userpath=os.path.expanduser('~'); os.rename(userpath + '/.qlib/qlib_data/cn_data_simple', userpath + '/.qlib/qlib_data/cn_data')" - azcopy copy https://qlibpublic.blob.core.windows.net/data /tmp/qlibpublic --recursive - mv /tmp/qlibpublic/data tests/.data - - name: Test workflow by config (install from pip) - run: | - python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml - python -m pip uninstall -y pyqlib - # Test Qlib installed from source - - name: Install Qlib from source - run: | - python -m pip install --upgrade cython - python -m pip install numpy jupyter jupyter_contrib_nbextensions - python -m pip install -U scipy scikit-learn # installing without this line will cause errors on GitHub Actions, while instsalling locally won't - python -m pip install gym tianshou torch - pip install -e . - - name: Install test dependencies - run: | - python -m pip install --upgrade pip - python -m pip install -U pyopenssl idna - python -m pip install black pytest - - name: Unit tests with Pytest - run: | - pip install -r scripts/data_collector/pit/requirements.txt - cd tests - python -m pytest . --durations=0 - - name: Test workflow by config (install from source) - run: | - python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml diff --git a/.github/workflows/test_qlib_from_pip.yml b/.github/workflows/test_qlib_from_pip.yml new file mode 100644 index 000000000..96cb60b38 --- /dev/null +++ b/.github/workflows/test_qlib_from_pip.yml @@ -0,0 +1,46 @@ +name: Test qlib from pip + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + timeout-minutes: 120 + + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [windows-latest, ubuntu-18.04, ubuntu-20.04, macos-11, macos-latest] + # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129 + python-version: [3.7, 3.8] + + steps: + - name: Test qlib from pip + uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Update pip to the latest version + run: | + python -m pip install --upgrade pip + + - name: Qlib installation test + run: | + python -m pip install pyqlib + # Specify the numpy version because the numpy upgrade caused the CI test to fail, + # and this line of code will be removed when the next version of qlib is released. + python -m pip install "numpy<1.23" + + - name: Downloads dependencies data + run: | + python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn + + - name: Test workflow by config + run: | + qrun examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml diff --git a/.github/workflows/test.yml b/.github/workflows/test_qlib_from_source.yml similarity index 70% rename from .github/workflows/test.yml rename to .github/workflows/test_qlib_from_source.yml index af1236cdd..2758e6fad 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test_qlib_from_source.yml @@ -1,4 +1,4 @@ -name: Test +name: Test qlib from source on: push: @@ -9,41 +9,60 @@ on: jobs: build: timeout-minutes: 120 + runs-on: ${{ matrix.os }} strategy: matrix: - os: [windows-latest, ubuntu-18.04, ubuntu-20.04] + os: [windows-latest, ubuntu-18.04, ubuntu-20.04, macos-11, macos-latest] # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129 python-version: [3.7, 3.8] steps: - - uses: actions/checkout@v2 + - name: Test qlib from source + uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + - name: Update pip to the latest version + run: | + python -m pip install --upgrade pip + + - name: Installing pytorch for macos + if: ${{ matrix.os == 'macos-11' || matrix.os == 'macos-latest' }} + run: | + python -m pip install torch torchvision torchaudio + brew install libomp # lightgbm dependencies + + - name: Installing pytorch for ubuntu + if: ${{ matrix.os == 'ubuntu-18.04' || matrix.os == 'ubuntu-20.04' }} + run: | + python -m pip install --upgrade pip + python -m pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu + + - name: Installing pytorch for windows + if: ${{ matrix.os == 'windows-latest' }} + run: | + python -m pip install --upgrade pip + python -m pip install torch torchvision torchaudio + + - name: Set up Python tools + run: | + python -m pip install --upgrade cython + python -m pip install -e .[dev] + - name: Lint with Black run: | - pip install --upgrade pip - pip install black wheel - black qlib -l 120 --check --diff - - - name: Install Qlib with pip - run: | - pip install numpy==1.19.5 ruamel.yaml - pip install pyqlib --ignore-installed + black . -l 120 --check --diff - name: Make html with sphinx run: | - pip install -U sphinx - pip install sphinx_rtd_theme readthedocs_sphinx_ext - pip install --exists-action=w --no-cache-dir -r docs/requirements.txt cd docs sphinx-build -b html . build cd .. - + # Check Qlib with pylint # TODO: These problems we will solve in the future. Important among them are: W0221, W0223, W0237, E1102 # C0103: invalid-name @@ -67,11 +86,9 @@ jobs: # W1309: f-string-without-interpolation # E1102: not-callable # E1136: unsubscriptable-object - # References for parameters: https://github.com/PyCQA/pylint/issues/4577#issuecomment-1000245962 + # References for parameters: https://github.com/PyCQA/pylint/issues/4577#issuecomment-1000245962 - name: Check Qlib with pylint run: | - pip install --upgrade pip - pip install pylint pylint --disable=C0104,C0114,C0115,C0116,C0301,C0302,C0411,C0413,C1802,R0401,R0801,R0902,R0903,R0911,R0912,R0913,R0914,R0915,R1720,W0105,W0123,W0201,W0511,W0613,W1113,W1514,E0401,E1121,C0103,C0209,R0402,R1705,R1710,R1725,R1735,W0102,W0212,W0221,W0223,W0231,W0237,W0612,W0621,W0622,W0703,W1309,E1102,E1136 --const-rgx='[a-z_][a-z0-9_]{2,30}$' qlib --init-hook "import astroid; astroid.context.InferenceContext.max_inferred = 500" # The following flake8 error codes were ignored: @@ -95,47 +112,29 @@ jobs: # Description: If there is whitespace before ":", it cannot pass the black check. - name: Check Qlib with flake8 run: | - pip install --upgrade pip - pip install flake8 flake8 --ignore=E501,F541,E266,E402,W503,E731,E203 --per-file-ignores="__init__.py:F401,F403" qlib # https://github.com/python/mypy/issues/10600 - name: Check Qlib with mypy run: | - pip install mypy mypy qlib --install-types --non-interactive || true - mypy qlib + mypy qlib --verbose - name: Test data downloads run: | - python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data_simple --interval 1d --region cn - python -c "import os; userpath=os.path.expanduser('~'); os.rename(userpath + '/.qlib/qlib_data/cn_data_simple', userpath + '/.qlib/qlib_data/cn_data')" + python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn azcopy copy https://qlibpublic.blob.core.windows.net/data/rl /tmp/qlibpublic/data --recursive mv /tmp/qlibpublic/data tests/.data - - name: Test workflow by config (install from pip) + - name: Test workflow by config (install from source) run: | + # Version 0.52.0 of numba must be installed manually in CI, otherwise it will cause incompatibility with the latest version of numpy. + python -m pip install numba==0.52.0 + # You must update numpy manually, because when installing python tools, it will try to uninstall numpy and cause CI to fail. + python -m pip install --upgrade numpy python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml - python -m pip uninstall -y pyqlib - - # Test Qlib installed from source - - name: Install Qlib from source - run: | - pip install --upgrade cython jupyter jupyter_contrib_nbextensions numpy scipy scikit-learn # installing without this line will cause errors on GitHub Actions, while instsalling locally won't - pip install gym tianshou torch - pip install -e . - - - name: Install test dependencies - run: | - pip install --upgrade pip - pip install black pytest - name: Unit tests with Pytest run: | - pip install -r scripts/data_collector/pit/requirements.txt cd tests - python -m pytest . --durations=10 - - - name: Test workflow by config (install from source) - run: | - python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml + python -m pytest . -m "not slow" --durations=0 diff --git a/.github/workflows/test_qlib_from_source_slow.yml b/.github/workflows/test_qlib_from_source_slow.yml new file mode 100644 index 000000000..5d97b2038 --- /dev/null +++ b/.github/workflows/test_qlib_from_source_slow.yml @@ -0,0 +1,52 @@ +name: Test qlib from source slow + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + timeout-minutes: 120 + + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [windows-latest, ubuntu-18.04, ubuntu-20.04, macos-11, macos-latest] + # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129 + python-version: [3.7, 3.8] + + steps: + - name: Test qlib from source slow + uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Set up Python tools + run: | + pip install --upgrade cython numpy pip + pip install -e .[dev] + + - name: Downloads dependencies data + run: | + python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn + + - name: Install Lightgbm for MacOS + if: ${{ matrix.os == 'macos-11' || matrix.os == 'macos-latest' }} + run: | + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Microsoft/qlib/main/.github/brew_install.sh)" + HOMEBREW_NO_AUTO_UPDATE=1 brew install lightgbm + # FIX MacOS error: Segmentation fault + # reference: https://github.com/microsoft/LightGBM/issues/4229 + wget https://raw.githubusercontent.com/Homebrew/homebrew-core/fb8323f2b170bd4ae97e1bac9bf3e2983af3fdb0/Formula/libomp.rb + brew unlink libomp + brew install libomp.rb + + - name: Unit tests with Pytest + run: | + cd tests + python -m pytest . -m "slow" --durations=0 diff --git a/qlib/data/storage/file_storage.py b/qlib/data/storage/file_storage.py index faa3b404a..cfac8d12b 100644 --- a/qlib/data/storage/file_storage.py +++ b/qlib/data/storage/file_storage.py @@ -106,10 +106,7 @@ class FileCalendarStorage(FileStorageMixin, CalendarStorage): if not self.uri.exists(): self._write_calendar(values=[]) with self.uri.open("rb") as fp: - return [ - str(x) - for x in np.loadtxt(fp, str, skiprows=skip_rows, max_rows=n_rows, delimiter="\n", encoding="utf-8") - ] + return [str(x) for x in np.loadtxt(fp, str, skiprows=skip_rows, max_rows=n_rows, encoding="utf-8")] def _write_calendar(self, values: Iterable[CalVT], mode: str = "wb"): with self.uri.open(mode=mode) as fp: diff --git a/setup.py b/setup.py index 8ee93c3eb..0ca9f26ba 100644 --- a/setup.py +++ b/setup.py @@ -137,10 +137,24 @@ setup( "sphinx", "sphinx_rtd_theme", "pre-commit", + # CI dependencies + "wheel", + "setuptools", + "black", + "pylint", + "mypy", + "flake8", + "readthedocs_sphinx_ext", + "cmake", + "lxml", + "baostock", + "yahooquery", + "beautifulsoup4", + "tianshou", + "gym>=0.24", # If you do not put gym at the end, gym will degrade causing pytest results to fail. ], "rl": [ "tianshou", - "gym", "torch", ], }, diff --git a/tests/pytest.ini b/tests/pytest.ini index 2ea41da8c..8f798f51a 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -1,4 +1,6 @@ [pytest] +markers = + slow: marks tests as slow (deselect with '-m "not slow"') filterwarnings = ignore:.*rng.randint:DeprecationWarning ignore:.*Casting input x to numpy array:UserWarning diff --git a/tests/rolling_tests/test_update_pred.py b/tests/rolling_tests/test_update_pred.py index 275c1d419..324611948 100644 --- a/tests/rolling_tests/test_update_pred.py +++ b/tests/rolling_tests/test_update_pred.py @@ -1,5 +1,6 @@ import copy import unittest +import pytest import fire import pandas as pd @@ -14,6 +15,7 @@ from qlib.workflow.online.update import LabelUpdater class TestRolling(TestAutoData): + @pytest.mark.slow def test_update_pred(self): """ This test is for testing if it will raise error if the `to_date` is out of the boundary. @@ -73,6 +75,7 @@ class TestRolling(TestAutoData): # this range is fixed now self.assertTrue((updated_pred.loc[mod_range2] == -2).all().item()) + @pytest.mark.slow def test_update_label(self): task = copy.deepcopy(CSI300_GBDT_TASK) diff --git a/tests/test_all_pipeline.py b/tests/test_all_pipeline.py index 179ab1e4a..30c0b5010 100644 --- a/tests/test_all_pipeline.py +++ b/tests/test_all_pipeline.py @@ -4,6 +4,7 @@ import sys import shutil import unittest +import pytest from pathlib import Path import qlib @@ -184,16 +185,19 @@ class TestAllFlow(TestAutoData): def tearDownClass(cls) -> None: shutil.rmtree(cls.URI_PATH.lstrip("file:")) + @pytest.mark.slow def test_0_train_with_sigana(self): TestAllFlow.PRED_SCORE, ic_ric, uri_path = train_with_sigana(self.URI_PATH) self.assertGreaterEqual(ic_ric["ic"].all(), 0, "train failed") self.assertGreaterEqual(ic_ric["ric"].all(), 0, "train failed") + @pytest.mark.slow def test_1_train(self): TestAllFlow.PRED_SCORE, ic_ric, TestAllFlow.RID = train(self.URI_PATH) self.assertGreaterEqual(ic_ric["ic"].all(), 0, "train failed") self.assertGreaterEqual(ic_ric["ric"].all(), 0, "train failed") + @pytest.mark.slow def test_2_backtest(self): analyze_df = backtest_analysis(TestAllFlow.PRED_SCORE, TestAllFlow.RID, self.URI_PATH) self.assertGreaterEqual( @@ -203,6 +207,7 @@ class TestAllFlow(TestAutoData): ) self.assertTrue(not analyze_df.isna().any().any(), "backtest failed") + @pytest.mark.slow def test_3_expmanager(self): pass_default, pass_current, uri_path = fake_experiment() self.assertTrue(pass_default, msg="default uri is incorrect") diff --git a/tests/test_contrib_workflow.py b/tests/test_contrib_workflow.py index 081cf78ea..3d250a142 100644 --- a/tests/test_contrib_workflow.py +++ b/tests/test_contrib_workflow.py @@ -4,6 +4,7 @@ from qlib.workflow.record_temp import SignalRecord import shutil import unittest +import pytest from pathlib import Path from qlib.contrib.workflow import MultiSegRecord, SignalMseRecord @@ -47,9 +48,11 @@ class TestAllFlow(TestAutoData): def tearDownClass(cls) -> None: shutil.rmtree(cls.URI_PATH.lstrip("file:")) + @pytest.mark.slow def test_0_multiseg(self): uri_path = train_multiseg(self.URI_PATH) + @pytest.mark.slow def test_1_mse(self): uri_path = train_mse(self.URI_PATH) diff --git a/tests/test_dataset.py b/tests/test_dataset.py index c5f24b178..ebb4aaa55 100755 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -2,6 +2,7 @@ # Licensed under the MIT License. import unittest +import pytest import sys from qlib.tests import TestAutoData from qlib.data.dataset import TSDatasetH @@ -11,6 +12,7 @@ from qlib.data.dataset.handler import DataHandlerLP class TestDataset(TestAutoData): + @pytest.mark.slow def testTSDataset(self): tsdh = TSDatasetH( handler={ diff --git a/tests/test_pit.py b/tests/test_pit.py index be56d0aea..b524e6b57 100644 --- a/tests/test_pit.py +++ b/tests/test_pit.py @@ -6,12 +6,13 @@ import sys import qlib import shutil import unittest +import pytest import pandas as pd import baostock as bs from pathlib import Path from qlib.data import D -from scripts.get_data import GetData +from qlib.tests.data import GetData from scripts.dump_pit import DumpPitData sys.path.append(str(Path(__file__).resolve().parent.parent.joinpath("scripts/data_collector/pit"))) @@ -119,6 +120,7 @@ class TestPIT(unittest.TestCase): """ self.check_same(data, expect) + @pytest.mark.slow def test_expr(self): fields = [ "P(Mean($$roewa_q, 1))",