diff --git a/.github/workflows/test_qlib_from_pip.yml b/.github/workflows/test_qlib_from_pip.yml deleted file mode 100644 index bde41d802..000000000 --- a/.github/workflows/test_qlib_from_pip.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: Test qlib from pip - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - build: - timeout-minutes: 120 - - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [windows-latest, ubuntu-20.04, ubuntu-22.04, macos-11, macos-latest] - # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129 - python-version: [3.7, 3.8] - - steps: - - name: Test qlib from pip - uses: actions/checkout@v3 - - # Since version 3.7 of python for MacOS is installed in CI, version 3.7.17, this version causes "_bz not found error". - # So we make the version number of python 3.7 for MacOS more specific. - # refs: https://github.com/actions/setup-python/issues/682 - - name: Set up Python ${{ matrix.python-version }} - if: (matrix.os == 'macos-latest' && matrix.python-version == '3.7') || (matrix.os == 'macos-11' && matrix.python-version == '3.7') - uses: actions/setup-python@v4 - with: - python-version: "3.7.16" - - - name: Set up Python ${{ matrix.python-version }} - if: (matrix.os != 'macos-latest' || matrix.python-version != '3.7') && (matrix.os != 'macos-11' || matrix.python-version != '3.7') - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Update pip to the latest version - run: | - python -m pip install --upgrade pip - - - name: Qlib installation test - run: | - python -m pip install pyqlib - - - name: Install Lightgbm for MacOS - if: ${{ matrix.os == 'macos-11' || matrix.os == 'macos-latest' }} - run: | - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Microsoft/qlib/main/.github/brew_install.sh)" - HOMEBREW_NO_AUTO_UPDATE=1 brew install lightgbm - # FIX MacOS error: Segmentation fault - # reference: https://github.com/microsoft/LightGBM/issues/4229 - wget https://raw.githubusercontent.com/Homebrew/homebrew-core/fb8323f2b170bd4ae97e1bac9bf3e2983af3fdb0/Formula/libomp.rb - brew unlink libomp - brew install libomp.rb - - - name: Downloads dependencies data - run: | - cd .. - python -m qlib.run.get_data qlib_data --target_dir ~/.qlib/qlib_data/cn_data --region cn - cd qlib - - - name: Test workflow by config - run: | - qrun examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml diff --git a/.github/workflows/test_qlib_from_source.yml b/.github/workflows/test_qlib_from_source.yml deleted file mode 100644 index 9205a1364..000000000 --- a/.github/workflows/test_qlib_from_source.yml +++ /dev/null @@ -1,175 +0,0 @@ -name: Test qlib from source - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - build: - timeout-minutes: 180 - # we may retry for 3 times for `Unit tests with Pytest` - - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [windows-latest, ubuntu-20.04, ubuntu-22.04, macos-11, macos-latest] - # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129 - python-version: [3.7, 3.8] - - steps: - - name: Test qlib from source - uses: actions/checkout@v3 - - # Since version 3.7 of python for MacOS is installed in CI, version 3.7.17, this version causes "_bz not found error". - # So we make the version number of python 3.7 for MacOS more specific. - # refs: https://github.com/actions/setup-python/issues/682 - - name: Set up Python ${{ matrix.python-version }} - if: (matrix.os == 'macos-latest' && matrix.python-version == '3.7') || (matrix.os == 'macos-11' && matrix.python-version == '3.7') - uses: actions/setup-python@v4 - with: - python-version: "3.7.16" - - - name: Set up Python ${{ matrix.python-version }} - if: (matrix.os != 'macos-latest' || matrix.python-version != '3.7') && (matrix.os != 'macos-11' || matrix.python-version != '3.7') - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Update pip to the latest version - run: | - python -m pip install --upgrade pip - - - name: Installing pytorch for macos - if: ${{ matrix.os == 'macos-11' || matrix.os == 'macos-latest' }} - run: | - python -m pip install torch torchvision torchaudio - - - name: Installing pytorch for ubuntu - if: ${{ matrix.os == 'ubuntu-20.04' || matrix.os == 'ubuntu-22.04' }} - run: | - python -m pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu - - - name: Installing pytorch for windows - if: ${{ matrix.os == 'windows-latest' }} - run: | - python -m pip install torch torchvision torchaudio - - - name: Set up Python tools - run: | - python -m pip install --upgrade cython - python -m pip install -e .[dev] - - - name: Lint with Black - # Python 3.7 will use a black with low level. So we use python with higher version for black check - if: (matrix.python-version != '3.7') - run: | - pip install -U black # follow the latest version of black, previous Qlib dependency will downgrade black - black . -l 120 --check --diff - - - name: Make html with sphinx - run: | - cd docs - sphinx-build -W --keep-going -b html . _build - cd .. - - # Check Qlib with pylint - # TODO: These problems we will solve in the future. Important among them are: W0221, W0223, W0237, E1102 - # C0103: invalid-name - # C0209: consider-using-f-string - # R0402: consider-using-from-import - # R1705: no-else-return - # R1710: inconsistent-return-statements - # R1725: super-with-arguments - # R1735: use-dict-literal - # W0102: dangerous-default-value - # W0212: protected-access - # W0221: arguments-differ - # W0223: abstract-method - # W0231: super-init-not-called - # W0237: arguments-renamed - # W0612: unused-variable - # W0621: redefined-outer-name - # W0622: redefined-builtin - # FIXME: specify exception type - # W0703: broad-except - # W1309: f-string-without-interpolation - # E1102: not-callable - # E1136: unsubscriptable-object - # References for parameters: https://github.com/PyCQA/pylint/issues/4577#issuecomment-1000245962 - # We use sys.setrecursionlimit(2000) to make the recursion depth larger to ensure that pylint works properly (the default recursion depth is 1000). - - name: Check Qlib with pylint - run: | - pylint --disable=C0104,C0114,C0115,C0116,C0301,C0302,C0411,C0413,C1802,R0401,R0801,R0902,R0903,R0911,R0912,R0913,R0914,R0915,R1720,W0105,W0123,W0201,W0511,W0613,W1113,W1514,E0401,E1121,C0103,C0209,R0402,R1705,R1710,R1725,R1735,W0102,W0212,W0221,W0223,W0231,W0237,W0612,W0621,W0622,W0703,W1309,E1102,E1136 --const-rgx='[a-z_][a-z0-9_]{2,30}$' qlib --init-hook "import astroid; astroid.context.InferenceContext.max_inferred = 500; import sys; sys.setrecursionlimit(2000)" - pylint --disable=C0104,C0114,C0115,C0116,C0301,C0302,C0411,C0413,C1802,R0401,R0801,R0902,R0903,R0911,R0912,R0913,R0914,R0915,R1720,W0105,W0123,W0201,W0511,W0613,W1113,W1514,E0401,E1121,C0103,C0209,R0402,R1705,R1710,R1725,R1735,W0102,W0212,W0221,W0223,W0231,W0237,W0246,W0612,W0621,W0622,W0703,W1309,E1102,E1136 --const-rgx='[a-z_][a-z0-9_]{2,30}$' scripts --init-hook "import astroid; astroid.context.InferenceContext.max_inferred = 500; import sys; sys.setrecursionlimit(2000)" - - # The following flake8 error codes were ignored: - # E501 line too long - # Description: We have used black to limit the length of each line to 120. - # F541 f-string is missing placeholders - # Description: The same thing is done when using pylint for detection. - # E266 too many leading '#' for block comment - # Description: To make the code more readable, a lot of "#" is used. - # This error code appears centrally in: - # qlib/backtest/executor.py - # qlib/data/ops.py - # qlib/utils/__init__.py - # E402 module level import not at top of file - # Description: There are times when module level import is not available at the top of the file. - # W503 line break before binary operator - # Description: Since black formats the length of each line of code, it has to perform a line break when a line of arithmetic is too long. - # E731 do not assign a lambda expression, use a def - # Description: Restricts the use of lambda expressions, but at some point lambda expressions are required. - # E203 whitespace before ':' - # Description: If there is whitespace before ":", it cannot pass the black check. - - name: Check Qlib with flake8 - run: | - flake8 --ignore=E501,F541,E266,E402,W503,E731,E203 --per-file-ignores="__init__.py:F401,F403" qlib - - # https://github.com/python/mypy/issues/10600 - - name: Check Qlib with mypy - run: | - mypy qlib --install-types --non-interactive || true - mypy qlib --verbose - - - name: Check Qlib ipynb with nbqa - run: | - nbqa black . -l 120 --check --diff - nbqa pylint . --disable=C0104,C0114,C0115,C0116,C0301,C0302,C0411,C0413,C1802,R0401,R0801,R0902,R0903,R0911,R0912,R0913,R0914,R0915,R1720,W0105,W0123,W0201,W0511,W0613,W1113,W1514,E0401,E1121,C0103,C0209,R0402,R1705,R1710,R1725,R1735,W0102,W0212,W0221,W0223,W0231,W0237,W0612,W0621,W0622,W0703,W1309,E1102,E1136,W0719,W0104,W0404,C0412,W0611,C0410 --const-rgx='[a-z_][a-z0-9_]{2,30}$' - - - name: Test data downloads - run: | - python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn - python scripts/get_data.py download_data --file_name rl_data.zip --target_dir tests/.data/rl - - - name: Install Lightgbm for MacOS - if: ${{ matrix.os == 'macos-11' || matrix.os == 'macos-latest' }} - run: | - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Microsoft/qlib/main/.github/brew_install.sh)" - HOMEBREW_NO_AUTO_UPDATE=1 brew install lightgbm - # FIX MacOS error: Segmentation fault - # reference: https://github.com/microsoft/LightGBM/issues/4229 - wget https://raw.githubusercontent.com/Homebrew/homebrew-core/fb8323f2b170bd4ae97e1bac9bf3e2983af3fdb0/Formula/libomp.rb - brew unlink libomp - brew install libomp.rb - - # Run after data downloads - - name: Check Qlib ipynb with nbconvert - run: | - # add more ipynb files in future - jupyter nbconvert --to notebook --execute examples/workflow_by_code.ipynb - - - name: Test workflow by config (install from source) - run: | - python -m pip install numba - python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml - - - name: Unit tests with Pytest - uses: nick-fields/retry@v2 - with: - timeout_minutes: 60 - max_attempts: 3 - command: | - cd tests - python -m pytest . -m "not slow" --durations=0 diff --git a/.github/workflows/test_qlib_from_source_slow.yml b/.github/workflows/test_qlib_from_source_slow.yml deleted file mode 100644 index caab6f444..000000000 --- a/.github/workflows/test_qlib_from_source_slow.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: Test qlib from source slow - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - build: - timeout-minutes: 720 - # we may retry for 3 times for `Unit tests with Pytest` - - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [windows-latest, ubuntu-20.04, ubuntu-22.04, macos-11, macos-latest] - # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129 - python-version: [3.7, 3.8] - - steps: - - name: Test qlib from source slow - uses: actions/checkout@v3 - - # Since version 3.7 of python for MacOS is installed in CI, version 3.7.17, this version causes "_bz not found error". - # So we make the version number of python 3.7 for MacOS more specific. - # refs: https://github.com/actions/setup-python/issues/682 - - name: Set up Python ${{ matrix.python-version }} - if: (matrix.os == 'macos-latest' && matrix.python-version == '3.7') || (matrix.os == 'macos-11' && matrix.python-version == '3.7') - uses: actions/setup-python@v4 - with: - python-version: "3.7.16" - - - name: Set up Python ${{ matrix.python-version }} - if: (matrix.os != 'macos-latest' || matrix.python-version != '3.7') && (matrix.os != 'macos-11' || matrix.python-version != '3.7') - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Set up Python tools - run: | - python -m pip install --upgrade pip - pip install --upgrade cython numpy - pip install -e .[dev] - - - name: Downloads dependencies data - run: | - python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn - - - name: Install Lightgbm for MacOS - if: ${{ matrix.os == 'macos-11' || matrix.os == 'macos-latest' }} - run: | - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Microsoft/qlib/main/.github/brew_install.sh)" - HOMEBREW_NO_AUTO_UPDATE=1 brew install lightgbm - # FIX MacOS error: Segmentation fault - # reference: https://github.com/microsoft/LightGBM/issues/4229 - wget https://raw.githubusercontent.com/Homebrew/homebrew-core/fb8323f2b170bd4ae97e1bac9bf3e2983af3fdb0/Formula/libomp.rb - brew unlink libomp - brew install libomp.rb - - - name: Unit tests with Pytest - uses: nick-fields/retry@v2 - with: - timeout_minutes: 240 - max_attempts: 3 - command: | - cd tests - python -m pytest . -m "slow" --durations=0 diff --git a/.github/workflows/test_upload.yml b/.github/workflows/test_upload.yml new file mode 100644 index 000000000..47e7d4b26 --- /dev/null +++ b/.github/workflows/test_upload.yml @@ -0,0 +1,83 @@ +# This workflows will upload Python Package to testpypi using Twine when a release is created + + +name: Upload qlib package to testpypi + +on: + release: + types: [published] + + +jobs: + deploy_with_bdist_wheel: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [windows-latest, macos-11] + # FIXME: macos-latest will raise error now. + # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129 + python-version: [3.7, 3.8] + steps: + - uses: actions/checkout@v2 + # This is because on macos systems you can install pyqlib using + # `pip install pyqlib` installs, it does not recognize the + # `pyqlib--cp38-cp38-macosx_11_0_x86_64.whl` and `pyqlib--cp38-cp37m-macosx_11_0_x86_64.whl`. + # So we limit the version of python, in order to generate a version of qlib that is usable for macos: `pyqlib--cp38-cp37m + # `pyqlib--cp38-cp38-macosx_10_15_x86_64.whl` and `pyqlib--cp38-cp37m-macosx_10_15_x86_64.whl`. + # Python 3.7.16, 3.8.16 can build macosx_10_15. But Python 3.7.17, 3.8.17 can build macosx_11_0 + - name: Set up Python ${{ matrix.python-version }} + if: matrix.os == 'macos-11' && matrix.python-version == '3.7' + uses: actions/setup-python@v2 + with: + python-version: "3.7.16" + - name: Set up Python ${{ matrix.python-version }} + if: matrix.os == 'macos-11' && matrix.python-version == '3.8' + uses: actions/setup-python@v2 + with: + python-version: "3.8.16" + - name: Set up Python ${{ matrix.python-version }} + if: matrix.os != 'macos-11' + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build wheel on ${{ matrix.os }} + run: | + pip install numpy + pip install cython + python setup.py bdist_wheel + - name: Build and publish + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.TESTPYPI_TOKEN }} + run: | + twine upload dist/* + + deploy_with_manylinux: + runs-on: ubuntu-latest + permissions: + id-token: write + steps: + - uses: actions/checkout@v2 + - name: Build wheel on Linux + uses: RalfG/python-wheels-manylinux-build@v0.3.1-manylinux2010_x86_64 + with: + # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129 + python-versions: 'cp37-cp37m cp38-cp38' + build-requirements: 'numpy cython' + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Install dependencies + run: | + pip install twine + - name: Build and publish + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.TESTPYPI_TOKEN }} + run: | + twine upload dist/pyqlib-*-manylinux*.whl \ No newline at end of file