From 86f08e47e8ab38338f6d662e911581800d58dc8e Mon Sep 17 00:00:00 2001 From: Di Date: Fri, 22 Jul 2022 09:24:58 +0800 Subject: [PATCH] Qlib data doc (#1207) * Explain data crawler structure * Add documentation for data and feature * Update scripts/data_collector/yahoo/README.md Co-authored-by: you-n-g * Remove some confusing wording * Add third party data source * Fix command typo * Update commands Co-authored-by: you-n-g --- docs/component/data.rst | 2 +- qlib/contrib/data/handler.py | 48 +++++++++++++++++++ scripts/README.md | 7 +++ scripts/data_collector/crowd_source/README.md | 32 +++++++++++++ scripts/data_collector/yahoo/README.md | 10 +++- 5 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 scripts/data_collector/crowd_source/README.md diff --git a/docs/component/data.rst b/docs/component/data.rst index da1842b88..b8279432e 100644 --- a/docs/component/data.rst +++ b/docs/component/data.rst @@ -51,7 +51,7 @@ Also, ``Qlib`` provides a high-frequency dataset. Users can run a high-frequency Qlib Format Dataset ------------------- -``Qlib`` has provided an off-the-shelf dataset in `.bin` format, users could use the script ``scripts/get_data.py`` to download the China-Stock dataset as follows. +``Qlib`` has provided an off-the-shelf dataset in `.bin` format, users could use the script ``scripts/get_data.py`` to download the China-Stock dataset as follows. User can also use numpy to load `.bin` file to validate data. The price volume data look different from the actual dealling price because of they are **adjusted** (`adjusted price `_). And then you may find that the adjusted price may be different from different data sources. This is because different data sources may vary in the way of adjusting prices. Qlib normalize the price on first trading day of each stock to 1 when adjusting them. Users can leverage `$factor` to get the original trading price (e.g. `$close / $factor` to get the original close price). diff --git a/qlib/contrib/data/handler.py b/qlib/contrib/data/handler.py index 1862baaa9..d9d09dbff 100644 --- a/qlib/contrib/data/handler.py +++ b/qlib/contrib/data/handler.py @@ -259,79 +259,119 @@ class Alpha158(DataHandlerLP): def use(x): return x not in exclude and (include is None or x in include) + # Some factor ref: https://guorn.com/static/upload/file/3/134065454575605.pdf if use("ROC"): + # https://www.investopedia.com/terms/r/rateofchange.asp + # Rate of change, the price change in the past d days, divided by latest close price to remove unit fields += ["Ref($close, %d)/$close" % d for d in windows] names += ["ROC%d" % d for d in windows] if use("MA"): + # https://www.investopedia.com/ask/answers/071414/whats-difference-between-moving-average-and-weighted-moving-average.asp + # Simple Moving Average, the simple moving average in the past d days, divided by latest close price to remove unit fields += ["Mean($close, %d)/$close" % d for d in windows] names += ["MA%d" % d for d in windows] if use("STD"): + # The standard diviation of close price for the past d days, divided by latest close price to remove unit fields += ["Std($close, %d)/$close" % d for d in windows] names += ["STD%d" % d for d in windows] if use("BETA"): + # The rate of close price change in the past d days, divided by latest close price to remove unit + # For example, price increase 10 dollar per day in the past d days, then Slope will be 10. fields += ["Slope($close, %d)/$close" % d for d in windows] names += ["BETA%d" % d for d in windows] if use("RSQR"): + # The R-sqaure value of linear regression for the past d days, represent the trend linearity for past d days. fields += ["Rsquare($close, %d)" % d for d in windows] names += ["RSQR%d" % d for d in windows] if use("RESI"): + # The redisdual for linear regression for the past d days, represent the trend linearity for past d days. fields += ["Resi($close, %d)/$close" % d for d in windows] names += ["RESI%d" % d for d in windows] if use("MAX"): + # The max price for past d days, divided by latest close price to remove unit fields += ["Max($high, %d)/$close" % d for d in windows] names += ["MAX%d" % d for d in windows] if use("LOW"): + # The low price for past d days, divided by latest close price to remove unit fields += ["Min($low, %d)/$close" % d for d in windows] names += ["MIN%d" % d for d in windows] if use("QTLU"): + # The 80% quantile of past d day's close price, divided by latest close price to remove unit + # Used with MIN and MAX fields += ["Quantile($close, %d, 0.8)/$close" % d for d in windows] names += ["QTLU%d" % d for d in windows] if use("QTLD"): + # The 20% quantile of past d day's close price, divided by latest close price to remove unit fields += ["Quantile($close, %d, 0.2)/$close" % d for d in windows] names += ["QTLD%d" % d for d in windows] if use("RANK"): + # Get the percentile of current close price in past d day's close price. + # Represent the current price level comparing to past N days, add additional information to moving average. fields += ["Rank($close, %d)" % d for d in windows] names += ["RANK%d" % d for d in windows] if use("RSV"): + # Represent the price position between upper and lower resistent price for past d days. fields += ["($close-Min($low, %d))/(Max($high, %d)-Min($low, %d)+1e-12)" % (d, d, d) for d in windows] names += ["RSV%d" % d for d in windows] if use("IMAX"): + # The number of days between current date and previous highest price date. + # Part of Aroon Indicator https://www.investopedia.com/terms/a/aroon.asp + # The indicator measures the time between highs and the time between lows over a time period. + # The idea is that strong uptrends will regularly see new highs, and strong downtrends will regularly see new lows. fields += ["IdxMax($high, %d)/%d" % (d, d) for d in windows] names += ["IMAX%d" % d for d in windows] if use("IMIN"): + # The number of days between current date and previous lowest price date. + # Part of Aroon Indicator https://www.investopedia.com/terms/a/aroon.asp + # The indicator measures the time between highs and the time between lows over a time period. + # The idea is that strong uptrends will regularly see new highs, and strong downtrends will regularly see new lows. fields += ["IdxMin($low, %d)/%d" % (d, d) for d in windows] names += ["IMIN%d" % d for d in windows] if use("IMXD"): + # The time period between previous lowest-price date occur after highest price date. + # Large value suggest downward momemtum. fields += ["(IdxMax($high, %d)-IdxMin($low, %d))/%d" % (d, d, d) for d in windows] names += ["IMXD%d" % d for d in windows] if use("CORR"): + # The correlation between absolute close price and log scaled trading volume fields += ["Corr($close, Log($volume+1), %d)" % d for d in windows] names += ["CORR%d" % d for d in windows] if use("CORD"): + # The correlation between price change ratio and volume change ratio fields += ["Corr($close/Ref($close,1), Log($volume/Ref($volume, 1)+1), %d)" % d for d in windows] names += ["CORD%d" % d for d in windows] if use("CNTP"): + # The percentage of days in past d days that price go up. fields += ["Mean($close>Ref($close, 1), %d)" % d for d in windows] names += ["CNTP%d" % d for d in windows] if use("CNTN"): + # The percentage of days in past d days that price go down. fields += ["Mean($closeRef($close, 1), %d)-Mean($close:/output -it --rm chenditc/investment_data bash dump_qlib_bin.sh && cp ./qlib_bin.tar.gz /output/ +``` + +## FAQ and other info +See: https://github.com/chenditc/investment_data/blob/main/README.md diff --git a/scripts/data_collector/yahoo/README.md b/scripts/data_collector/yahoo/README.md index dddfb70ea..bd852d052 100644 --- a/scripts/data_collector/yahoo/README.md +++ b/scripts/data_collector/yahoo/README.md @@ -36,7 +36,7 @@ pip install -r requirements.txt - `target_dir`: save dir, by default *~/.qlib/qlib_data/cn_data* - `version`: dataset version, value from [`v1`, `v2`], by default `v1` - `v2` end date is *2021-06*, `v1` end date is *2020-09* - - user can append data to `v2`: [automatic update of daily frequency data](#automatic-update-of-daily-frequency-datafrom-yahoo-finance) + - If users want to incrementally update data, they need to use yahoo collector to [collect data from scratch](#collector-yahoofinance-data-to-qlib). - **the [benchmarks](https://github.com/microsoft/qlib/tree/main/examples/benchmarks) for qlib use `v1`**, *due to the unstable access to historical data by YahooFinance, there are some differences between `v2` and `v1`* - `interval`: `1d` or `1min`, by default `1d` - `region`: `cn` or `us` or `in`, by default `cn` @@ -62,6 +62,8 @@ pip install -r requirements.txt > collector *YahooFinance* data and *dump* into `qlib` format. > If the above ready-made data can't meet users' requirements, users can follow this section to crawl the latest data and convert it to qlib-data. 1. download data to csv: `python scripts/data_collector/yahoo/collector.py download_data` + + This will download the raw data such as high, low, open, close, adjclose price from yahoo to a local directory. One file per symbol. - parameters: - `source_dir`: save the directory @@ -99,6 +101,10 @@ pip install -r requirements.txt ``` 2. normalize data: `python scripts/data_collector/yahoo/collector.py normalize_data` + This will: + 1. Normalize high, low, close, open price using adjclose. + 2. Normalize the high, low, close, open price so that the first valid trading date's close price is 1. + - parameters: - `source_dir`: csv directory - `normalize_dir`: result directory @@ -136,6 +142,8 @@ pip install -r requirements.txt ``` 3. dump data: `python scripts/dump_bin.py dump_all` + This will convert the normalized csv in `feature` directory as numpy array and store the normalized data one file per column and one symbol per directory. + - parameters: - `csv_path`: stock data path or directory, **normalize result(normalize_dir)** - `qlib_dir`: qlib(dump) data director