1
0
mirror of https://github.com/microsoft/qlib.git synced 2026-07-06 04:20:57 +08:00

replace multi processing with joblib (#477)

* replace multi processing with joblib

* update class Parallel and data.py

* update class Parallel and data.py

* update class Parallel and data.py

* update class Parallel and data.py

* update class Parallel and data.py

* update class Parallel and data.py

* update class Parallel and data.py

* update class Parallel and data.py

* Fix Parallel support for maxtasksperchild

Co-authored-by: wangw <1666490690@qq.com>
Co-authored-by: zhupr <zhu.pengrong@foxmail.com>
This commit is contained in:
you-n-g
2021-09-14 01:16:03 +08:00
committed by GitHub
parent 6203e4c09e
commit 163e3c6266
4 changed files with 86 additions and 95 deletions

View File

@@ -0,0 +1,39 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
import unittest
import qlib
from qlib.data import D
from qlib.tests import TestAutoData
from multiprocessing import Pool
def get_features(fields):
qlib.init(provider_uri=TestAutoData.provider_uri, expression_cache=None, dataset_cache=None, joblib_backend="loky")
return D.features(D.instruments("csi300"), fields)
class TestGetData(TestAutoData):
FIELDS = "$open,$close,$high,$low,$volume,$factor,$change".split(",")
def test_multi_proc(self):
"""
For testing if it will raise error
"""
iter_n = 2
pool = Pool(iter_n)
res = []
for _ in range(iter_n):
res.append(pool.apply_async(get_features, (self.FIELDS,), {}))
for r in res:
print(r.get())
pool.close()
pool.join()
if __name__ == "__main__":
unittest.main()